diff --git a/.gitignore b/.gitignore index 42a5b09..032e7dd 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,4 @@ cython_debug/ # VSCode .vscode .azure +test_output/ \ No newline at end of file diff --git a/data/face/enrollment_data/Jordan/Family1-Daughter3.jpg b/data/face/enrollment_data/Mary/Family1-Daughter3.jpg similarity index 100% rename from data/face/enrollment_data/Jordan/Family1-Daughter3.jpg rename to data/face/enrollment_data/Mary/Family1-Daughter3.jpg diff --git a/data/face/enrollment_data/Bill/Family1-Dad3.jpg b/data/face/new_face_image.jpg similarity index 100% rename from data/face/enrollment_data/Bill/Family1-Dad3.jpg rename to data/face/new_face_image.jpg diff --git a/notebooks/.env.sample b/notebooks/.env.sample index 05c1000..6e23e31 100644 --- a/notebooks/.env.sample +++ b/notebooks/.env.sample @@ -1 +1,37 @@ -AZURE_AI_ENDPOINT= \ No newline at end of file +# Azure Content Understanding Service Configuration +# Copy this file to /.env and update with your actual values + +# Your Azure Content Understanding service endpoint +# Example: https://your-resource-name.services.ai.azure.com/ +# If you need help to create one, please see the Prerequisites section in: +# https://learn.microsoft.com/en-us/azure/ai-services/content-understanding/quickstart/use-rest-api?tabs=document#prerequisites +# As of 2025/05, 2025-05-01-preview is only available in the regions documented in +# Content Understanding region and language support (https://learn.microsoft.com/en-us/azure/ai-services/content-understanding/language-region-support). + +# Azure Content Understanding Test Configuration + +# Required for Content Understanding SDK and testing +AZURE_CONTENT_UNDERSTANDING_ENDPOINT=https://your-resource-name.services.ai.azure.com/ + +# Authentication Options: +# Option 1: Use Azure Key (FOR TESTING ONLY - Less secure) +# Set this value if you want to use key-based authentication +# WARNING: Keys are less secure and should only be used for testing/development +# Leave empty to use DefaultAzureCredential (recommended) +AZURE_CONTENT_UNDERSTANDING_KEY= + +# Option 2: Use DefaultAzureCredential (RECOMMENDED for production and development) +# If AZURE_CONTENT_UNDERSTANDING_KEY is empty, the script will use DefaultAzureCredential +# +# Most common development scenario: +# 1. Install Azure CLI: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli +# 2. Login: az login +# 3. Run the script (no additional configuration needed) +# +# This also supports: +# - Environment variables (AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID) +# - Managed Identity (for Azure-hosted applications) +# - Visual Studio Code authentication +# - Azure PowerShell authentication +# For more info: https://learn.microsoft.com/en-us/python/api/overview/azure/identity-readme#defaultazurecredential + diff --git a/notebooks/analyzer_training.ipynb b/notebooks/analyzer_training.ipynb index 64cbc45..3109bfa 100644 --- a/notebooks/analyzer_training.ipynb +++ b/notebooks/analyzer_training.ipynb @@ -57,7 +57,6 @@ "metadata": {}, "outputs": [], "source": [ - "analyzer_template = \"../analyzer_templates/receipt.json\"\n", "training_docs_folder = \"../data/document_training\"" ] }, @@ -88,30 +87,45 @@ "import json\n", "import os\n", "import sys\n", - "from pathlib import Path\n", - "from dotenv import find_dotenv, load_dotenv\n", - "from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n", + "from datetime import datetime\n", + "import uuid\n", + "from dotenv import load_dotenv\n", + "from azure.storage.blob import ContainerSasPermissions\n", + "from azure.core.credentials import AzureKeyCredential\n", + "from azure.identity import DefaultAzureCredential\n", + "from azure.ai.contentunderstanding.aio import ContentUnderstandingClient\n", + "from azure.ai.contentunderstanding.models import (\n", + " ContentAnalyzer,\n", + " FieldSchema,\n", + " FieldDefinition,\n", + " FieldType,\n", + " GenerationMethod,\n", + " AnalysisMode,\n", + " ProcessingLocation,\n", + ")\n", "\n", - "# Import utility package from the Python samples root directory\n", - "parent_dir = Path(Path.cwd()).parent\n", - "sys.path.append(str(parent_dir))\n", - "from python.content_understanding_client import AzureContentUnderstandingClient\n", + "# Add the parent directory to the Python path to import the sample_helper module\n", + "sys.path.append(os.path.join(os.path.dirname(os.getcwd()), 'python'))\n", + "from extension.document_processor import DocumentProcessor\n", + "from extension.sample_helper import extract_operation_id_from_poller, PollerType, save_json_to_file\n", "\n", - "load_dotenv(find_dotenv())\n", + "load_dotenv()\n", "logging.basicConfig(level=logging.INFO)\n", "\n", - "credential = DefaultAzureCredential()\n", - "token_provider = get_bearer_token_provider(credential, \"https://cognitiveservices.azure.com/.default\")\n", - "\n", - "client = AzureContentUnderstandingClient(\n", - " endpoint=os.getenv(\"AZURE_AI_ENDPOINT\"),\n", - " api_version=os.getenv(\"AZURE_AI_API_VERSION\", \"2025-05-01-preview\"),\n", - " # IMPORTANT: Comment out token_provider if using subscription key\n", - " token_provider=token_provider,\n", - " # IMPORTANT: Uncomment this if using subscription key\n", - " # subscription_key=os.getenv(\"AZURE_AI_API_KEY\"),\n", - " x_ms_useragent=\"azure-ai-content-understanding-python/analyzer_training\", # This header is used for sample usage telemetry; please comment out this line if you want to opt out.\n", - ")" + "endpoint = os.environ.get(\"AZURE_CONTENT_UNDERSTANDING_ENDPOINT\")\n", + "# Return AzureKeyCredential if AZURE_CONTENT_UNDERSTANDING_KEY is set, otherwise DefaultAzureCredential\n", + "key = os.getenv(\"AZURE_CONTENT_UNDERSTANDING_KEY\")\n", + "credential = AzureKeyCredential(key) if key else DefaultAzureCredential()\n", + "# Create the ContentUnderstandingClient\n", + "client = ContentUnderstandingClient(endpoint=endpoint, credential=credential)\n", + "print(\"βœ… ContentUnderstandingClient created successfully\")\n", + "\n", + "try:\n", + " processor = DocumentProcessor(client)\n", + " print(\"βœ… DocumentProcessor created successfully\")\n", + "except Exception as e:\n", + " print(f\"❌ Failed to create DocumentProcessor: {e}\")\n", + " raise" ] }, { @@ -133,26 +147,27 @@ "metadata": {}, "outputs": [], "source": [ + "# Load reference storage configuration from environment\n", + "training_data_path = os.getenv(\"TRAINING_DATA_PATH\") or f\"training_data_{uuid.uuid4().hex[:8]}\"\n", "training_data_sas_url = os.getenv(\"TRAINING_DATA_SAS_URL\")\n", + "\n", + "if not training_data_path.endswith(\"/\"):\n", + " training_data_sas_url += \"/\"\n", + "\n", "if not training_data_sas_url:\n", - " TRAINING_DATA_STORAGE_ACCOUNT_NAME = os.getenv(\"TRAINING_DATA_STORAGE_ACCOUNT_NAME\")\n", - " TRAINING_DATA_CONTAINER_NAME = os.getenv(\"TRAINING_DATA_CONTAINER_NAME\")\n", - " if not TRAINING_DATA_STORAGE_ACCOUNT_NAME and not training_data_sas_url:\n", - " raise ValueError(\n", - " \"Please set either TRAINING_DATA_SAS_URL or both TRAINING_DATA_STORAGE_ACCOUNT_NAME and TRAINING_DATA_CONTAINER_NAME environment variables.\"\n", + " training_data_storage_account_name = os.getenv(\"TRAINING_DATA_STORAGE_ACCOUNT_NAME\")\n", + " training_data_container_name = os.getenv(\"TRAINING_DATA_CONTAINER_NAME\")\n", + "\n", + " if training_data_storage_account_name and training_data_container_name:\n", + " # We require \"Write\" permission to upload, modify, or append blobs\n", + " training_data_sas_url = processor.generate_container_sas_url(\n", + " account_name=training_data_storage_account_name,\n", + " container_name=training_data_container_name,\n", + " permissions=ContainerSasPermissions(read=True, write=True, list=True),\n", + " expiry_hours=1,\n", " )\n", - " from azure.storage.blob import ContainerSasPermissions\n", - " # Requires \"Write\" (critical for upload/modify/append) along with \"Read\" and \"List\" for viewing/listing blobs.\n", - " training_data_sas_url = AzureContentUnderstandingClient.generate_temp_container_sas_url(\n", - " account_name=TRAINING_DATA_STORAGE_ACCOUNT_NAME,\n", - " container_name=TRAINING_DATA_CONTAINER_NAME,\n", - " permissions=ContainerSasPermissions(read=True, write=True, list=True),\n", - " expiry_hours=1,\n", - " )\n", - "\n", - "training_data_path = os.getenv(\"TRAINING_DATA_PATH\")\n", - "\n", - "await client.generate_training_data_on_blob(training_docs_folder, training_data_sas_url, training_data_path)" + "\n", + "await processor.generate_training_data_on_blob(training_docs_folder, training_data_sas_url, training_data_path)" ] }, { @@ -162,7 +177,7 @@ "## Create Analyzer with Defined Schema\n", "Before creating the analyzer, fill in the constant `ANALYZER_ID` with a relevant name for your task. In this example, we generate a unique suffix so that this cell can be run multiple times to create different analyzers.\n", "\n", - "We use **training_data_sas_url** and **training_data_path** as set in the [.env](./.env) file and used in the previous step." + "We use **TRAINING_DATA_SAS_URL** and **TRAINING_DATA_PATH** as set in the [.env](./.env) file and used in the previous step." ] }, { @@ -171,24 +186,78 @@ "metadata": {}, "outputs": [], "source": [ - "import uuid\n", - "CUSTOM_ANALYZER_ID = \"train-sample-\" + str(uuid.uuid4())\n", + "analyzer_id = f\"analyzer-training-sample-{datetime.now().strftime('%Y%m%d-%H%M%S')}-{uuid.uuid4().hex[:8]}\"\n", + "\n", + "content_analyzer = ContentAnalyzer(\n", + " base_analyzer_id=\"prebuilt-documentAnalyzer\",\n", + " description=\"Extract useful information from receipt\",\n", + " field_schema=FieldSchema(\n", + " name=\"receipt schema\",\n", + " description=\"Schema for receipt\",\n", + " fields={\n", + " \"MerchantName\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.EXTRACT,\n", + " description=\"\"\n", + " ),\n", + " \"Items\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"\",\n", + " items_property={\n", + " \"type\": \"object\",\n", + " \"method\": \"extract\",\n", + " \"properties\": {\n", + " \"Quantity\": {\n", + " \"type\": \"string\",\n", + " \"method\": \"extract\",\n", + " \"description\": \"\"\n", + " },\n", + " \"Name\": {\n", + " \"type\": \"string\",\n", + " \"method\": \"extract\",\n", + " \"description\": \"\"\n", + " },\n", + " \"Price\": {\n", + " \"type\": \"string\",\n", + " \"method\": \"extract\",\n", + " \"description\": \"\"\n", + " }\n", + " }\n", + " }\n", + " ),\n", + " \"TotalPrice\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.EXTRACT,\n", + " description=\"\"\n", + " )\n", + " }\n", + " ),\n", + " mode=AnalysisMode.STANDARD,\n", + " processing_location=ProcessingLocation.GEOGRAPHY,\n", + " tags={\"demo_type\": \"get_result\"},\n", + " training_data={\n", + " \"kind\": \"blob\",\n", + " \"containerUrl\": training_data_sas_url,\n", + " \"prefix\": training_data_path\n", + " },\n", + ")\n", + "print(f\"πŸ”§ Creating custom analyzer '{analyzer_id}'...\")\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=analyzer_id,\n", + " resource=content_analyzer,\n", + ")\n", "\n", - "response = client.begin_create_analyzer(\n", - " CUSTOM_ANALYZER_ID,\n", - " analyzer_template_path=analyzer_template,\n", - " training_storage_container_sas_url=training_data_sas_url,\n", - " training_storage_container_path_prefix=training_data_path,\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", ")\n", - "result = client.poll_result(response)\n", - "if result is not None and \"status\" in result and result[\"status\"] == \"Succeeded\":\n", - " logging.info(f\"Analyzer details for {result['result']['analyzerId']}\")\n", - " logging.info(json.dumps(result, indent=2))\n", - "else:\n", - " logging.warning(\n", - " \"An issue was encountered when trying to create the analyzer. \"\n", - " \"Please double-check your deployment and configurations for potential problems.\"\n", - " )" + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{analyzer_id}' created successfully!\")" ] }, { @@ -205,10 +274,53 @@ "metadata": {}, "outputs": [], "source": [ - "response = client.begin_analyze(CUSTOM_ANALYZER_ID, file_location='../data/receipt.png')\n", - "result_json = client.poll_result(response)\n", + "file_path = \"../data/receipt.png\"\n", + "print(f\"πŸ“„ Reading document file: {file_path}\")\n", + "with open(file_path, \"rb\") as f:\n", + " data_content = f.read()\n", + "\n", + "# Begin document analysis operation\n", + "print(f\"πŸ” Starting document analysis with analyzer '{analyzer_id}'...\")\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=analyzer_id, \n", + " input=data_content,\n", + " content_type=\"application/octet-stream\")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for document analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Document analysis completed successfully!\")\n", + "\n", + " # Extract operation ID for get_result\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the analysis result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "print(f\"βœ… Analysis result retrieved successfully!\")\n", + "print(f\" Operation ID: {operation_status.id}\")\n", + "print(f\" Status: {operation_status.status}\")\n", "\n", - "logging.info(json.dumps(result_json, indent=2))" + "# The actual analysis result is in operation_status.result\n", + "operation_result = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + "\n", + "print(f\"πŸ“„ Analysis Result: {json.dumps(operation_result.as_dict())}\")\n", + "\n", + "# Save the analysis result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"analyzer_training_get_result\",\n", + ")" ] }, { @@ -225,13 +337,15 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(CUSTOM_ANALYZER_ID)" + "print(f\"πŸ—‘οΈ Deleting analyzer '{analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=analyzer_id)\n", + "print(f\"βœ… Analyzer '{analyzer_id}' deleted successfully!\")" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "py312", "language": "python", "name": "python3" }, @@ -245,7 +359,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.12" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/notebooks/build_person_directory.ipynb b/notebooks/build_person_directory.ipynb index 4840de4..8df83f2 100644 --- a/notebooks/build_person_directory.ipynb +++ b/notebooks/build_person_directory.ipynb @@ -47,31 +47,30 @@ "source": [ "import logging\n", "import os\n", + "import uuid\n", "import sys\n", - "from pathlib import Path\n", - "from dotenv import find_dotenv, load_dotenv\n", - "from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n", + "from dotenv import load_dotenv\n", + "from azure.core.credentials import AzureKeyCredential\n", + "from azure.identity.aio import DefaultAzureCredential\n", + "from azure.ai.contentunderstanding.aio import ContentUnderstandingClient\n", + "from azure.ai.contentunderstanding.models import PersonDirectory, FaceSource, PersonDirectoryPerson\n", + "sys.path.append(os.path.join(os.path.dirname(os.getcwd()), 'python'))\n", + "from extension.sample_helper import (\n", + " read_image_to_base64\n", + ")\n", "\n", - "# import utility package from python samples root directory\n", - "parent_dir = Path.cwd().parent\n", - "sys.path.append(str(parent_dir))\n", - "from python.content_understanding_face_client import AzureContentUnderstandingFaceClient\n", + "# Add the parent directory to the Python path to import the sample_helper module\n", + "sys.path.append(os.path.join(os.path.dirname(os.getcwd()), 'python'))\n", "\n", - "load_dotenv(find_dotenv())\n", + "load_dotenv()\n", "logging.basicConfig(level=logging.INFO)\n", "\n", - "credential = DefaultAzureCredential()\n", - "token_provider = get_bearer_token_provider(credential, \"https://cognitiveservices.azure.com/.default\")\n", - "\n", - "client = AzureContentUnderstandingFaceClient(\n", - " endpoint=os.getenv(\"AZURE_AI_ENDPOINT\"),\n", - " api_version=os.getenv(\"AZURE_AI_API_VERSION\", \"2025-05-01-preview\"),\n", - " # IMPORTANT: Comment out token_provider if using subscription key\n", - " token_provider=token_provider,\n", - " # IMPORTANT: Uncomment this if using subscription key\n", - " # subscription_key=os.getenv(\"AZURE_AI_API_KEY\"),\n", - " x_ms_useragent=\"azure-ai-content-understanding-python/build_person_directory\", # This header is used for sample usage telemetry, please comment out this line if you want to opt out.\n", - ")" + "endpoint = os.environ.get(\"AZURE_CONTENT_UNDERSTANDING_ENDPOINT\")\n", + "# Return AzureKeyCredential if AZURE_CONTENT_UNDERSTANDING_KEY is set, otherwise DefaultAzureCredential\n", + "key = os.getenv(\"AZURE_CONTENT_UNDERSTANDING_KEY\")\n", + "credential = AzureKeyCredential(key) if key else DefaultAzureCredential()\n", + "# Create the ContentUnderstandingClient\n", + "client = ContentUnderstandingClient(endpoint=endpoint, credential=credential)" ] }, { @@ -89,38 +88,68 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "import uuid\n", "folder_path = \"../data/face/enrollment_data\" # Replace with the path to your folder containing subfolders of images\n", "\n", "# Create a person directory\n", "person_directory_id = f\"person_directory_id_{uuid.uuid4().hex[:8]}\"\n", - "client.create_person_directory(person_directory_id)\n", + "\n", + "# Create a person directory first\n", + "print(f\"πŸ”§ Creating person directory '{person_directory_id}'...\")\n", + "\n", + "person_directory = PersonDirectory(\n", + " description=f\"Sample person directory for delete person demo: {person_directory_id}\",\n", + " tags={\"demo_type\": \"delete_person\"},\n", + " )\n", + "person_directory = await client.person_directories.create(person_directory_id, resource=person_directory)\n", "logging.info(f\"Created person directory with ID: {person_directory_id}\")\n", "\n", + "# Initialize persons list\n", + "persons: list = []\n", + "\n", "# Iterate through all subfolders in the folder_path\n", "for subfolder_name in os.listdir(folder_path):\n", " subfolder_path = os.path.join(folder_path, subfolder_name)\n", " if os.path.isdir(subfolder_path):\n", " person_name = subfolder_name\n", " # Add a person for each subfolder\n", - " person = client.add_person(person_directory_id, tags={\"name\": person_name})\n", + " person = await client.person_directories.add_person(person_directory_id, tags={\"name\": person_name})\n", + " print(f\"πŸ”§ Creating person '{person_name}'...\")\n", " logging.info(f\"Created person {person_name} with person_id: {person['personId']}\")\n", " if person:\n", + " # Initialize person entry in persons list\n", + " person_entry = {\n", + " 'personId': person['personId'],\n", + " 'name': person_name,\n", + " 'faceIds': []\n", + " }\n", + "\n", " # Iterate through all images in the subfolder\n", " for filename in os.listdir(subfolder_path):\n", " if filename.lower().endswith(('.png', '.jpg', '.jpeg')):\n", " image_path = os.path.join(subfolder_path, filename)\n", " # Convert image to base64\n", - " image_data = AzureContentUnderstandingFaceClient.read_file_to_base64(image_path)\n", + " image_data_base64 = read_image_to_base64(image_path)\n", " # Add a face to the Person Directory and associate it to the added person\n", - " face = client.add_face(person_directory_id, image_data, person['personId'])\n", + " print(f\"πŸ”§ Adding face from image '{image_path}' to person '{person_name}'...\")\n", + " print(f\"Image Data: \", image_data_base64)\n", + " face = await client.person_directories.add_face(\n", + " person_directory_id=person_directory_id, \n", + " face_source=FaceSource(data=image_data_base64),\n", + " person_id=person['personId']\n", + " )\n", " if face:\n", + " person_entry['faceIds'].append(face['faceId'])\n", " logging.info(f\"Added face from {filename} with face_id: {face['faceId']} to person_id: {person['personId']}\")\n", " else:\n", " logging.warning(f\"Failed to add face from {filename} to person_id: {person['personId']}\")\n", "\n", - "logging.info(\"Done\")" + " # Add person entry to persons list\n", + " persons.append(person_entry)\n", + "\n", + "logging.info(\"Done\")\n", + "logging.info(f\"Created {len(persons)} persons:\")\n", + "for person in persons:\n", + " logging.info(f\"Person: {person['name']} (ID: {person['personId']}) with {len(person['faceIds'])} faces\")" ] }, { @@ -142,12 +171,15 @@ "test_image_path = \"../data/face/family.jpg\" # Path to the test image\n", "\n", "# Detect faces in the test image\n", - "image_data = AzureContentUnderstandingFaceClient.read_file_to_base64(test_image_path)\n", - "detected_faces = client.detect_faces(data=image_data)\n", + "image_data_base64 = read_image_to_base64(test_image_path)\n", + "detected_faces = await client.faces.detect(data=image_data_base64)\n", "for face in detected_faces['detectedFaces']:\n", - " identified_persons = client.identify_person(person_directory_id, image_data, face['boundingBox'])\n", - " if identified_persons.get(\"personCandidates\"):\n", - " person = identified_persons[\"personCandidates\"][0]\n", + " identified_persons = await client.person_directories.identify_person(\n", + " person_directory_id=person_directory_id, \n", + " face_source=FaceSource(data=image_data_base64), \n", + " max_person_candidates=5)\n", + " if identified_persons.get(\"person_candidates\"):\n", + " person = identified_persons[\"person_candidates\"][0]\n", " name = person.get(\"tags\", {}).get(\"name\", \"Unknown\")\n", " logging.info(f\"Detected person: {name} with confidence: {person.get('confidence', 0)} at bounding box: {face['boundingBox']}\")\n", "\n", @@ -170,15 +202,20 @@ "metadata": {}, "outputs": [], "source": [ - "new_face_image_path = \"new_face_image_path\" # The path to the face image you want to add.\n", - "existing_person_id = \"existing_person_id\" # The unique ID of the person to whom the face should be associated.\n", + "person_bill = next(person for person in persons if person['name'] == 'Bill')\n", + "new_face_image_path = \"../data/face/new_face_image.jpg\" # The path to the face image you want to add.\n", + "existing_person_id = person_bill['personId'] # The unique ID of the person to whom the face should be associated.\n", "\n", "# Convert the new face image to base64\n", - "image_data = AzureContentUnderstandingFaceClient.read_file_to_base64(new_face_image_path)\n", + "image_data_base64 = read_image_to_base64(new_face_image_path)\n", "# Add the new face to the person directory and associate it with the existing person\n", - "face = client.add_face(person_directory_id, image_data, existing_person_id)\n", + "face = await client.person_directories.add_face(\n", + " person_directory_id=person_directory_id, \n", + " face_source=FaceSource(data=image_data_base64), \n", + " person_id=existing_person_id)\n", "if face:\n", " logging.info(f\"Added face from {new_face_image_path} with face_id: {face['faceId']} to person_id: {existing_person_id}\")\n", + " person_bill['faceIds'].append(face['faceId'])\n", "else:\n", " logging.warning(f\"Failed to add face from {new_face_image_path} to person_id: {existing_person_id}\")" ] @@ -200,11 +237,16 @@ "metadata": {}, "outputs": [], "source": [ - "existing_person_id = \"existing_person_id\" # The unique ID of the person to whom the face should be associated.\n", - "existing_face_id_list = [\"existing_face_id_1\", \"existing_face_id_2\"] # The list of face IDs to be associated.\n", + "existing_person_id = person_bill['personId'] # The unique ID of the person to whom the face should be associated.\n", + "existing_face_id_list: list = [person_bill['faceIds'][0], person_bill['faceIds'][1], person_bill['faceIds'][2]] # The list of face IDs to be associated.\n", "\n", "# Associate the existing face IDs with the existing person\n", - "client.update_person(person_directory_id, existing_person_id, face_ids=existing_face_id_list)" + "await client.person_directories.update_person(\n", + " person_directory_id=person_directory_id, \n", + " person_id=existing_person_id, \n", + " resource={\"faceIds\": existing_face_id_list},\n", + " content_type=\"application/json\"\n", + ")" ] }, { @@ -223,18 +265,30 @@ "metadata": {}, "outputs": [], "source": [ - "existing_face_id = \"existing_face_id\" # The unique ID of the face.\n", + "person_mary = next(person for person in persons if person['name'] == 'Mary')\n", + "existing_face_id = person_mary['faceIds'][0] # The unique ID of the face.\n", "\n", "# Remove the association of the existing face ID from the person\n", - "client.update_face(person_directory_id, existing_face_id, person_id=\"\") # The person_id is set to \"\" to remove the association\n", + "await client.person_directories.update_face(\n", + " person_directory_id=person_directory_id, \n", + " face_id=existing_face_id,\n", + " resource={'personId': None},\n", + " content_type=\"application/json\"\n", + ")\n", "logging.info(f\"Removed association of face_id: {existing_face_id} from the existing person_id\")\n", - "logging.info(client.get_face(person_directory_id, existing_face_id)) # This will return the face information without the person association\n", + "logging.info(await client.person_directories.get_face(person_directory_id, existing_face_id)) # This will return the face information without the person association\n", "\n", "# Associate the existing face ID with a person\n", - "existing_person_id = \"existing_person_id\" # The unique ID of the person to be associated with the face.\n", - "client.update_face(person_directory_id, existing_face_id, person_id=existing_person_id)\n", + "person_jordan = next(person for person in persons if person['name'] == 'Jordan')\n", + "existing_person_id = person_jordan['personId'] # The unique ID of the person to be associated with the face.\n", + "await client.person_directories.update_face(\n", + " person_directory_id=person_directory_id, \n", + " face_id=existing_face_id, \n", + " resource={'personId': existing_person_id},\n", + " content_type=\"application/json\"\n", + ")\n", "logging.info(f\"Associated face_id: {existing_face_id} with person_id: {existing_person_id}\")\n", - "logging.info(client.get_face(person_directory_id, existing_face_id)) # This will return the face information with the new person association" + "logging.info(await client.person_directories.get_face(person_directory_id, existing_face_id)) # This will return the face information with the new person association" ] }, { @@ -257,25 +311,31 @@ "person_directory_description = \"This is a sample person directory for managing faces.\"\n", "person_directory_tags = {\"project\": \"face_management\", \"version\": \"1.0\"}\n", "\n", - "client.update_person_directory(\n", - " person_directory_id,\n", - " description=person_directory_description,\n", - " tags=person_directory_tags\n", + "await client.person_directories.update(\n", + " person_directory_id=person_directory_id,\n", + " resource=PersonDirectory(\n", + " description=person_directory_description,\n", + " tags=person_directory_tags\n", + " ),\n", + " content_type=\"application/json\",\n", ")\n", "logging.info(f\"Updated Person Directory with description: '{person_directory_description}' and tags: {person_directory_tags}\")\n", - "logging.info(client.get_person_directory(person_directory_id)) # This will return the updated person directory information\n", + "logging.info(await client.person_directories.get(person_directory_id)) # This will return the updated person directory information\n", "\n", "# Update the tags for an individual person\n", - "existing_person_id = \"existing_person_id\" # The unique ID of the person to update.\n", + "existing_person_id = person_bill['personId'] # The unique ID of the person to update.\n", "person_tags = {\"role\": \"tester\", \"department\": \"engineering\", \"name\": \"\"} # This will remove the name tag from the person.\n", "\n", - "client.update_person(\n", - " person_directory_id,\n", - " existing_person_id,\n", - " tags=person_tags\n", + "await client.person_directories.update_person(\n", + " person_directory_id=person_directory_id,\n", + " person_id=existing_person_id,\n", + " resource=PersonDirectoryPerson(\n", + " tags=person_tags\n", + " ),\n", + " content_type=\"application/json\",\n", ")\n", "logging.info(f\"Updated person with person_id: {existing_person_id} with tags: {person_tags}\")\n", - "logging.info(client.get_person(person_directory_id, existing_person_id)) # This will return the updated person information" + "logging.info(await client.person_directories.get_person(person_directory_id, existing_person_id)) # This will return the updated person information" ] }, { @@ -294,9 +354,9 @@ "metadata": {}, "outputs": [], "source": [ - "existing_face_id = \"existing_face_id\" # The unique ID of the face to delete.\n", + "existing_face_id = person_mary['faceIds'][0] # The unique ID of the face to delete.\n", "\n", - "client.delete_face(person_directory_id, existing_face_id)\n", + "await client.person_directories.delete_face(person_directory_id, existing_face_id)\n", "logging.info(f\"Deleted face with face_id: {existing_face_id}\")" ] }, @@ -317,9 +377,9 @@ "metadata": {}, "outputs": [], "source": [ - "existing_person_id = \"existing_person_id\" # The unique ID of the person to delete.\n", + "existing_person_id = person_mary['personId'] # The unique ID of the person to delete.\n", "\n", - "client.delete_person(person_directory_id, existing_person_id)\n", + "await client.person_directories.delete_person(person_directory_id, existing_person_id)\n", "logging.info(f\"Deleted person with person_id: {existing_person_id}\")" ] }, @@ -340,26 +400,26 @@ "metadata": {}, "outputs": [], "source": [ - "existing_person_id = \"existing_person_id\" # The unique ID of the person to delete.\n", + "existing_person_id = person_bill['personId'] # The unique ID of the person to delete.\n", "\n", "# Get the list of face IDs associated with the person\n", - "response = client.get_person(person_directory_id, existing_person_id)\n", + "response = await client.person_directories.get_person(person_directory_id, existing_person_id)\n", "face_ids = response.get('faceIds', [])\n", "\n", "# Delete each face associated with the person\n", "for face_id in face_ids:\n", " logging.info(f\"Deleting face with face_id: {face_id} from person_id: {existing_person_id}\")\n", - " client.delete_face(person_directory_id, face_id)\n", + " await client.person_directories.delete_face(person_directory_id, face_id)\n", "\n", "# Delete the person after deleting all associated faces\n", - "client.delete_person(person_directory_id, existing_person_id)\n", + "await client.person_directories.delete_person(person_directory_id, existing_person_id)\n", "logging.info(f\"Deleted person with person_id: {existing_person_id} and all associated faces.\")" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "py312", "language": "python", "name": "python3" }, @@ -373,7 +433,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.12" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/notebooks/classifier.ipynb b/notebooks/classifier.ipynb index 2a640a2..338dc2f 100644 --- a/notebooks/classifier.ipynb +++ b/notebooks/classifier.ipynb @@ -32,7 +32,18 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 1. Import Required Libraries" + "## Create Azure AI Content Understanding Client\n", + "\n", + "> The [AzureContentUnderstandingClient](../python/content_understanding_client.py) is a utility class that provides functions to interact with the Content Understanding API. Prior to the official release of the Content Understanding SDK, it serves as a lightweight SDK.\n", + ">\n", + "> Fill in the constants **AZURE_AI_ENDPOINT**, **AZURE_AI_API_VERSION**, and **AZURE_AI_API_KEY** with the details from your Azure AI Service.\n", + "\n", + "> ⚠️ Important:\n", + "You must update the code below to use your preferred Azure authentication method.\n", + "Look for the `# IMPORTANT` comments in the code and modify those sections accordingly.\n", + "Skipping this step may cause the sample to not run correctly.\n", + "\n", + "> ⚠️ Note: While using a subscription key is supported, it is strongly recommended to use a token provider with Azure Active Directory (AAD) for enhanced security in production environments." ] }, { @@ -41,29 +52,57 @@ "metadata": {}, "outputs": [], "source": [ - "import json\n", + "%pip install python-dotenv azure-ai-contentunderstanding azure-identity\n", + "\n", "import logging\n", + "import json\n", "import os\n", "import sys\n", + "from datetime import datetime\n", "import uuid\n", - "from pathlib import Path\n", + "from dotenv import load_dotenv\n", + "from azure.core.credentials import AzureKeyCredential\n", + "from azure.identity.aio import DefaultAzureCredential\n", + "from azure.ai.contentunderstanding.aio import ContentUnderstandingClient\n", + "from azure.ai.contentunderstanding.models import (\n", + " ContentClassifier,\n", + " ContentAnalyzer,\n", + " ClassifierCategory,\n", + " DocumentContent,\n", + " FieldSchema,\n", + " FieldDefinition,\n", + " FieldType,\n", + " ContentAnalyzerConfig,\n", + ")\n", "\n", - "from dotenv import find_dotenv, load_dotenv\n", - "from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n", + "# Add the parent directory to the Python path to import the sample_helper module\n", + "sys.path.append(os.path.join(os.path.dirname(os.getcwd()), 'python'))\n", + "from extension.sample_helper import extract_operation_id_from_poller, save_json_to_file, PollerType\n", + "from typing import Dict, Optional\n", "\n", - "load_dotenv(find_dotenv())\n", + "load_dotenv()\n", "logging.basicConfig(level=logging.INFO)\n", "\n", - "print(\"βœ… Libraries imported successfully!\")" + "endpoint = os.environ.get(\"AZURE_CONTENT_UNDERSTANDING_ENDPOINT\")\n", + "# Return AzureKeyCredential if AZURE_CONTENT_UNDERSTANDING_KEY is set, otherwise DefaultAzureCredential\n", + "key = os.getenv(\"AZURE_CONTENT_UNDERSTANDING_KEY\")\n", + "credential = AzureKeyCredential(key) if key else DefaultAzureCredential()\n", + "# Create the ContentUnderstandingClient\n", + "client = ContentUnderstandingClient(endpoint=endpoint, credential=credential)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 2. Import Azure Content Understanding Client\n", + "## Create a Basic Classifier\n", + "Classify document from URL using begin_classify API.\n", "\n", - "The `AzureContentUnderstandingClient` class manages all API interactions with the Azure AI service." + "High-level steps:\n", + "1. Create a custom classifier\n", + "2. Classify a document from a remote URL\n", + "3. Save the classification result to a file\n", + "4. Clean up the created classifier" ] }, { @@ -72,74 +111,68 @@ "metadata": {}, "outputs": [], "source": [ - "# Add the parent directory to the system path to access shared modules\n", - "parent_dir = Path(Path.cwd()).parent\n", - "sys.path.append(str(parent_dir))\n", - "try:\n", - " from python.content_understanding_client import AzureContentUnderstandingClient\n", - " print(\"βœ… Azure Content Understanding Client imported successfully!\")\n", - "except ImportError:\n", - " print(\"❌ Error: Ensure 'AzureContentUnderstandingClient.py' exists in the same directory as this notebook.\")\n", - " raise" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Configure Azure AI Service Settings and Prepare the Sample\n", + "# Create a simple ContentClassifier object with default configuration.\n", "\n", - "Update the following settings to match your Azure environment:\n", + "# Args:\n", + "# classifier_id: The classifier ID\n", + "# description: Optional description for the classifier\n", + "# tags: Optional tags for the classifier\n", "\n", - "- **AZURE_AI_ENDPOINT**: Your Azure AI service endpoint URL, or configure it in the \".env\" file\n", - "- **AZURE_AI_API_VERSION**: Azure AI API version to use. Defaults to \"2025-05-01-preview\"\n", - "- **AZURE_AI_API_KEY**: Your Azure AI API key (optional if using token-based authentication)\n", - "- **ANALYZER_SAMPLE_FILE**: Path to the PDF document you want to process" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Authentication supports either token-based or subscription key methods; only one is required\n", - "AZURE_AI_ENDPOINT = os.getenv(\"AZURE_AI_ENDPOINT\")\n", - "# IMPORTANT: Substitute with your subscription key or configure in \".env\" if not using token auth\n", - "AZURE_AI_API_KEY = os.getenv(\"AZURE_AI_API_KEY\")\n", - "AZURE_AI_API_VERSION = os.getenv(\"AZURE_AI_API_VERSION\", \"2025-05-01-preview\")\n", - "ANALYZER_SAMPLE_FILE = \"../data/mixed_financial_docs.pdf\" # Update this path to your PDF file\n", - "\n", - "# Use DefaultAzureCredential for token-based authentication\n", - "credential = DefaultAzureCredential()\n", - "token_provider = get_bearer_token_provider(credential, \"https://cognitiveservices.azure.com/.default\")\n", - "\n", - "file_location = Path(ANALYZER_SAMPLE_FILE)\n", - "\n", - "print(\"πŸ“‹ Configuration Summary:\")\n", - "print(f\" Endpoint: {AZURE_AI_ENDPOINT}\")\n", - "print(f\" API Version: {AZURE_AI_API_VERSION}\")\n", - "print(f\" Document: {file_location.name if file_location.exists() else '❌ File not found'}\")" + "# Returns:\n", + "# ContentClassifier: A configured ContentClassifier object\n", + "\n", + "def create_classifier_schema(description: Optional[str] = None, tags: Optional[Dict[str, str]] = None) -> ContentClassifier:\n", + " categories = {\n", + " \"Loan application\": ClassifierCategory(\n", + " description=\"Documents submitted by individuals or businesses to request funding, typically including personal or business details, financial history, loan amount, purpose, and supporting documentation.\"\n", + " ),\n", + " \"Invoice\": ClassifierCategory(\n", + " description=\"Billing documents issued by sellers or service providers to request payment for goods or services, detailing items, prices, taxes, totals, and payment terms.\"\n", + " ),\n", + " \"Bank_Statement\": ClassifierCategory(\n", + " description=\"Official statements issued by banks that summarize account activity over a period, including deposits, withdrawals, fees, and balances.\"\n", + " ),\n", + " }\n", + "\n", + " classifier = ContentClassifier(\n", + " categories=categories,\n", + " split_mode=\"auto\",\n", + " description=description,\n", + " tags=tags,\n", + " )\n", + "\n", + " return classifier\n", + "\n", + "# Generate a unique classifier ID\n", + "classifier_id = f\"classifier-sample-{datetime.now().strftime('%Y%m%d')}-{datetime.now().strftime('%H%M%S')}-{uuid.uuid4().hex[:8]}\"\n", + "\n", + "# Create a custom classifier using object model\n", + "print(f\"πŸ”§ Creating custom classifier '{classifier_id}'...\")\n", + "\n", + "classifier_schema: ContentClassifier = create_classifier_schema(\n", + " description=f\"Custom classifier for URL classification demo: {classifier_id}\",\n", + " tags={\"demo_type\": \"url_classification\"},\n", + ")\n", + "\n", + "# Start the classifier creation operation\n", + "poller = await client.content_classifiers.begin_create_or_replace(\n", + " classifier_id=classifier_id,\n", + " resource=classifier_schema,\n", + ")\n", + "\n", + "# Wait for the classifier to be created\n", + "print(f\"⏳ Waiting for classifier creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Classifier '{classifier_id}' created successfully!\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 4. Define Classifier Schema\n", - "\n", - "The classifier schema defines:\n", - "- **Categories**: Document types to classify (e.g., Legal, Medical)\n", - " - **description (Optional)**: Provides additional context or hints for categorizing or splitting documents. Useful when the category name alone is not sufficiently descriptive. Omit if the category name is self-explanatory.\n", - "- **splitMode Options**: Determines how multi-page documents are split before classification or analysis.\n", - " - `\"auto\"`: Automatically split based on content. \n", - " For example, given categories β€œinvoice” and β€œapplication form”:\n", - " - A PDF with one invoice will be classified as a single document.\n", - " - A PDF containing two invoices and one application form will be automatically split into three classified sections.\n", - " - `\"none\"`: No splitting. \n", - " The entire multi-page document is treated as one unit for classification and analysis.\n", - " - `\"perPage\"`: Split by page. \n", - " Treats each page as a separate document, useful if custom analyzers designed to operate at the page level." + "## Classify Your Document\n", + "\n", + "Now, use the classifier to categorize your document." ] }, { @@ -148,73 +181,67 @@ "metadata": {}, "outputs": [], "source": [ - "# Define document categories and their descriptions\n", - "classifier_schema = {\n", - " \"categories\": {\n", - " \"Loan application\": { # Both spaces and underscores are supported in category names\n", - " \"description\": \"Documents submitted by individuals or businesses to request funding, typically including personal or business details, financial history, loan amount, purpose, and supporting documentation.\"\n", - " },\n", - " \"Invoice\": {\n", - " \"description\": \"Billing documents issued by sellers or service providers to request payment for goods or services, detailing items, prices, taxes, totals, and payment terms.\"\n", - " },\n", - " \"Bank_Statement\": { # Both spaces and underscores are supported\n", - " \"description\": \"Official statements issued by banks summarizing account activity over a period, including deposits, withdrawals, fees, and balances.\"\n", - " },\n", - " },\n", - " \"splitMode\": \"auto\" # IMPORTANT: Automatically detect document boundaries; adjust as needed.\n", - "}\n", + "# Read the mixed financial docs PDF file\n", + "pdf_path = \"../data/mixed_financial_docs.pdf\"\n", + "print(f\"πŸ“„ Reading document file: {pdf_path}\")\n", + "with open(pdf_path, \"rb\") as pdf_file:\n", + " pdf_content = pdf_file.read()\n", "\n", - "print(\"πŸ“„ Classifier Categories:\")\n", - "for category, details in classifier_schema[\"categories\"].items():\n", - " print(f\" β€’ {category}: {details['description'][:60]}...\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Initialize Content Understanding Client\n", + "# Begin binary classification operation\n", + "print(f\"πŸ” Starting binary classification with classifier '{classifier_id}'...\")\n", + "classification_poller = await client.content_classifiers.begin_classify_binary(\n", + " classifier_id=classifier_id,\n", + " input=pdf_content,\n", + " content_type=\"application/pdf\",\n", + ")\n", "\n", - "Create the client to interact with Azure AI services.\n", + "# Wait for classification completion\n", + "print(f\"⏳ Waiting for classification to complete...\")\n", + "classification_result = await classification_poller.result()\n", + "print(f\"βœ… Classification completed successfully!\")\n", "\n", - "⚠️ Important:\n", - "Please update the authentication details below to match your Azure setup.\n", - "Look for the `# IMPORTANT` comments and modify those sections accordingly.\n", - "Skipping this step may result in runtime errors.\n", + "# Extract operation ID for get_result\n", + "classification_operation_id = extract_operation_id_from_poller(\n", + " classification_poller, PollerType.CLASSIFY_CALL\n", + ")\n", + "print(\n", + " f\"πŸ“‹ Extracted classification operation ID: {classification_operation_id}\"\n", + ")\n", "\n", - "⚠️ Note: While subscription key authentication works, using Azure Active Directory (AAD) token provider is more secure and recommended for production." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize the Azure Content Understanding client\n", - "try:\n", - " content_understanding_client = AzureContentUnderstandingClient(\n", - " endpoint=AZURE_AI_ENDPOINT,\n", - " api_version=AZURE_AI_API_VERSION,\n", - " # IMPORTANT: Comment out token_provider if using subscription key\n", - " token_provider=token_provider,\n", - " # IMPORTANT: Uncomment this if using subscription key\n", - " # subscription_key=AZURE_AI_API_KEY,\n", + "# Get the classification result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting classification result using operation ID '{classification_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_classifiers.get_result(\n", + " operation_id=classification_operation_id,\n", + ")\n", + "\n", + "print(f\"βœ… Classification result retrieved successfully!\")\n", + "print(f\" Operation ID: {getattr(operation_status, 'id', 'N/A')}\")\n", + "print(f\" Status: {getattr(operation_status, 'status', 'N/A')}\")\n", + "\n", + "# The actual classification result is in operation_status.result\n", + "operation_result = getattr(operation_status, \"result\", None)\n", + "if operation_result is not None:\n", + " print(\n", + " f\" Result contains {len(getattr(operation_result, 'contents', []))} contents\"\n", " )\n", - " print(\"βœ… Content Understanding client initialized successfully!\")\n", - " print(\" Ready to create classifiers and analyzers.\")\n", - "except Exception as e:\n", - " print(f\"❌ Failed to initialize client: {e}\")\n", - " raise" + "\n", + "# Save the classification result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_status.as_dict(),\n", + " filename_prefix=\"content_classifiers_get_result\",\n", + ")\n", + "print(f\"πŸ’Ύ Classification result saved to: {saved_file_path}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 6. Create a Basic Classifier\n", + "## View Classification Results\n", "\n", - "First, create a simple classifier that categorizes documents without performing additional analysis." + "Review the classification results generated for your document." ] }, { @@ -223,37 +250,21 @@ "metadata": {}, "outputs": [], "source": [ - "# Generate a unique classifier ID\n", - "classifier_id = \"classifier-sample-\" + str(uuid.uuid4())\n", - "\n", - "try:\n", - " # Create the classifier\n", - " print(f\"πŸ”¨ Creating classifier: {classifier_id}\")\n", - " print(\" This may take a few seconds...\")\n", - " \n", - " response = content_understanding_client.begin_create_classifier(classifier_id, classifier_schema)\n", - " result = content_understanding_client.poll_result(response)\n", - " \n", - " print(\"\\nβœ… Classifier created successfully!\")\n", - " print(f\" Status: {result.get('status')}\")\n", - " print(f\" Resource Location: {result.get('resourceLocation')}\")\n", - " \n", - "except Exception as e:\n", - " print(f\"\\n❌ Error creating classifier: {e}\")\n", - " if \"already exists\" in str(e):\n", - " print(\"\\nπŸ’‘ Tip: The classifier already exists. You can:\")\n", - " print(\" 1. Use a different classifier ID\")\n", - " print(\" 2. Delete the existing classifier first\")\n", - " print(\" 3. Skip to document classification\")" + "# Display classification results\n", + "print(f\"πŸ“Š Classification Results:\")\n", + "for content in classification_result.contents:\n", + " document_content: DocumentContent = content\n", + " print(f\" Category: {document_content.category}\")\n", + " print(f\" Start Page Number: {document_content.start_page_number}\")\n", + " print(f\" End Page Number: {document_content.end_page_number}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 7. Classify Your Document\n", - "\n", - "Now, use the classifier to categorize your document." + "## Saving Classification Results\n", + "The classification result is saved to a JSON file for later analysis." ] }, { @@ -262,34 +273,21 @@ "metadata": {}, "outputs": [], "source": [ - "try:\n", - " # Verify that the document exists\n", - " if not file_location.exists():\n", - " raise FileNotFoundError(f\"Document not found at {file_location}\")\n", - " \n", - " # Classify the document\n", - " print(f\"πŸ“„ Classifying document: {file_location.name}\")\n", - " print(\"\\n⏳ Processing... This may take several minutes for large documents.\")\n", - " \n", - " response = content_understanding_client.begin_classify(classifier_id, file_location=str(file_location))\n", - " result = content_understanding_client.poll_result(response, timeout_seconds=360)\n", - " \n", - " print(\"\\nβœ… Classification completed successfully!\")\n", - " \n", - "except FileNotFoundError:\n", - " print(f\"\\n❌ Document not found: {file_location}\")\n", - " print(\" Please update 'file_location' to point to your PDF file.\")\n", - "except Exception as e:\n", - " print(f\"\\n❌ Error classifying document: {e}\")" + "# Save the classification result to a file\n", + "\n", + "saved_file_path = save_json_to_file(\n", + " result=classification_result.as_dict(),\n", + " filename_prefix=\"content_classifiers_classify\",\n", + ")\n", + "print(f\"πŸ’Ύ Classification result saved to: {saved_file_path}\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 8. View Classification Results\n", - "\n", - "Review the classification results generated for your document." + "## Clean up the created analyzer \n", + "After the demo completes, the classifier is automatically deleted to prevent resource accumulation." ] }, { @@ -298,33 +296,17 @@ "metadata": {}, "outputs": [], "source": [ - "# Display classification results\n", - "if 'result' in locals() and result:\n", - " result_data = result.get(\"result\", {})\n", - " contents = result_data.get(\"contents\", [])\n", - " \n", - " print(\"πŸ“Š CLASSIFICATION RESULTS\")\n", - " print(\"=\" * 50)\n", - " print(f\"\\nTotal sections found: {len(contents)}\")\n", - " \n", - " # Summarize each classified section\n", - " print(\"\\nπŸ“‹ Document Sections:\")\n", - " for i, content in enumerate(contents, 1):\n", - " print(f\"\\n Section {i}:\")\n", - " print(f\" β€’ Category: {content.get('category', 'Unknown')}\")\n", - " print(f\" β€’ Pages: {content.get('startPageNumber', '?')} - {content.get('endPageNumber', '?')}\")\n", - " \n", - " print(\"\\nFull result output:\")\n", - " print(json.dumps(result, indent=2))\n", - "else:\n", - " print(\"❌ No results available. Please run the classification step first.\")" + "# Clean up the created classifier (demo cleanup)\n", + "print(f\"πŸ—‘οΈ Deleting classifier '{classifier_id}' (demo cleanup)...\")\n", + "await client.content_classifiers.delete(classifier_id=classifier_id)\n", + "print(f\"βœ… Classifier '{classifier_id}' deleted successfully!\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 9. Create a Custom Analyzer (Advanced)\n", + "## Create a Custom Analyzer (Advanced)\n", "\n", "Create a custom analyzer to extract specific fields from documents.\n", "This example extracts common fields from loan application documents and generates document excerpts." @@ -336,80 +318,74 @@ "metadata": {}, "outputs": [], "source": [ - "# Define the analyzer schema with custom fields\n", - "analyzer_schema = {\n", - " \"description\": \"Loan application analyzer - extracts key information from loan applications\",\n", - " \"baseAnalyzerId\": \"prebuilt-documentAnalyzer\", # Built on top of the general document analyzer\n", - " \"config\": {\n", - " \"returnDetails\": True,\n", - " \"enableLayout\": True, # Extract layout details\n", - " \"enableBarcode\": False, # Disable barcode detection\n", - " \"enableFormula\": False, # Disable formula detection\n", - " \"estimateFieldSourceAndConfidence\": True, # Enable estimation of field location and confidence\n", - " \"disableContentFiltering\": False\n", - " },\n", - " \"fieldSchema\": {\n", - " \"fields\": {\n", - " \"ApplicationDate\": {\n", - " \"type\": \"date\",\n", - " \"method\": \"generate\",\n", - " \"description\": \"The date when the loan application was submitted.\"\n", - " },\n", - " \"ApplicantName\": {\n", - " \"type\": \"string\",\n", - " \"method\": \"generate\",\n", - " \"description\": \"Full name of the loan applicant or company.\"\n", - " },\n", - " \"LoanAmountRequested\": {\n", - " \"type\": \"number\",\n", - " \"method\": \"generate\",\n", - " \"description\": \"The total loan amount requested by the applicant.\"\n", - " },\n", - " \"LoanPurpose\": {\n", - " \"type\": \"string\",\n", - " \"method\": \"generate\",\n", - " \"description\": \"The stated purpose or reason for the loan.\"\n", - " },\n", - " \"CreditScore\": {\n", - " \"type\": \"number\",\n", - " \"method\": \"generate\",\n", - " \"description\": \"Credit score of the applicant, if available.\"\n", - " },\n", - " \"Summary\": {\n", - " \"type\": \"string\",\n", - " \"method\": \"generate\",\n", - " \"description\": \"A brief summary overview of the loan application details.\"\n", - " }\n", + "import asyncio\n", + "\n", + "# Define fields schema\n", + "custom_analyzer = ContentAnalyzer(\n", + " base_analyzer_id=\"prebuilt-documentAnalyzer\", # Built on top of the general document analyzer\n", + " description=\"Loan application analyzer - extracts key information from loan applications\",\n", + " config=ContentAnalyzerConfig(\n", + " return_details=True,\n", + " enable_layout=True, # Extract layout details\n", + " enable_formula=False, # Disable formula detection\n", + " estimate_field_source_and_confidence=True, # Enable estimation of field location and confidence\n", + " disable_content_filtering=False\n", + " ),\n", + " field_schema=FieldSchema(\n", + " fields={\n", + " \"ApplicationDate\": FieldDefinition(\n", + " type=FieldType.DATE,\n", + " method=\"generate\",\n", + " description=\"The date when the loan application was submitted.\"\n", + " ),\n", + " \"ApplicantName\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=\"generate\",\n", + " description=\"Full name of the loan applicant or company.\"\n", + " ),\n", + " \"LoanAmountRequested\": FieldDefinition(\n", + " type=FieldType.NUMBER,\n", + " method=\"generate\",\n", + " description=\"The total loan amount requested by the applicant.\"\n", + " ),\n", + " \"LoanPurpose\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=\"generate\",\n", + " description=\"The stated purpose or reason for the loan.\"\n", + " ),\n", + " \"CreditScore\": FieldDefinition(\n", + " type=FieldType.NUMBER,\n", + " method=\"generate\",\n", + " description=\"Credit score of the applicant, if available.\"\n", + " ),\n", + " \"Summary\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=\"generate\",\n", + " description=\"A brief summary overview of the loan application details.\"\n", + " )\n", " }\n", - " }\n", - "}\n", + " ),\n", + " tags={\"demo\": \"loan-application\"}\n", + ")\n", "\n", "# Generate a unique analyzer ID\n", - "analyzer_id = \"analyzer-loan-application-\" + str(uuid.uuid4())\n", + "analyzer_id = f\"classifier-sample-{datetime.now().strftime('%Y%m%d')}-{datetime.now().strftime('%H%M%S')}-{uuid.uuid4().hex[:8]}\"\n", "\n", "# Create the custom analyzer\n", - "try:\n", - " print(f\"πŸ”¨ Creating custom analyzer: {analyzer_id}\")\n", - " print(\"\\nπŸ“‹ The analyzer will extract the following fields:\")\n", - " for field_name, field_info in analyzer_schema[\"fieldSchema\"][\"fields\"].items():\n", - " print(f\" β€’ {field_name}: {field_info['description']}\")\n", - " \n", - " response = content_understanding_client.begin_create_analyzer(analyzer_id, analyzer_schema)\n", - " result = content_understanding_client.poll_result(response)\n", - " \n", - " print(\"\\nβœ… Analyzer created successfully!\")\n", - " print(f\" Analyzer ID: {analyzer_id}\")\n", - " \n", - "except Exception as e:\n", - " print(f\"\\n❌ Error creating analyzer: {e}\")\n", - " analyzer_id = None # Set to None if creation failed" + "print(f\"πŸ”§ Creating custom analyzer '{analyzer_id}'...\")\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=analyzer_id,\n", + " resource=custom_analyzer,\n", + ")\n", + "result = await poller.result()\n", + "print(f\"βœ… Analyzer '{analyzer_id}' created successfully!\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 10. Create an Enhanced Classifier with Custom Analyzer\n", + "## Create an Enhanced Classifier with Custom Analyzer\n", "\n", "Now create a new classifier that uses the prebuilt invoice analyzer for invoices and the custom analyzer for loan application documents.\n", "This combines document classification with field extraction in one operation." @@ -421,12 +397,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Generate a unique enhanced classifier ID\n", - "enhanced_classifier_id = \"classifier-enhanced-\" + str(uuid.uuid4())\n", - "\n", - "# Define the enhanced classifier schema\n", - "enhanced_classifier_schema = {\n", - " \"categories\": {\n", + "def create_enhanced_classifier_schema(analyzer_id: str, description: Optional[str] = None, tags: Optional[Dict[str, str]] = None) -> ContentClassifier:\n", + " categories = {\n", " \"Loan application\": { # Both spaces and underscores allowed\n", " \"description\": \"Documents submitted by individuals or businesses requesting funding, including personal/business details, financial history, and supporting documents.\",\n", " \"analyzerId\": analyzer_id # IMPORTANT: Use the custom analyzer created previously for loan applications\n", @@ -439,35 +411,45 @@ " \"description\": \"Official bank statements summarizing account activity over a period, including deposits, withdrawals, fees, and balances.\"\n", " # No analyzer specified - uses default processing\n", " }\n", - " },\n", - " \"splitMode\": \"auto\"\n", - "}\n", + " }\n", + "\n", + " classifier = ContentClassifier(\n", + " categories=categories,\n", + " split_mode=\"auto\",\n", + " description=description,\n", + " tags=tags,\n", + " )\n", + "\n", + " return classifier\n", + "\n", + "# Generate a unique enhanced classifier ID\n", + "classifier_id = f\"enhanced-classifier-sample-{datetime.now().strftime('%Y%m%d')}-{datetime.now().strftime('%H%M%S')}-{uuid.uuid4().hex[:8]}\"\n", + "\n", + "# Create the enhanced classifier schema\n", + "enhanced_classifier_schema = create_enhanced_classifier_schema(\n", + " analyzer_id=analyzer_id,\n", + " description=f\"Custom classifier for URL classification demo: {classifier_id}\",\n", + " tags={\"demo_type\": \"url_classification\"}\n", + ")\n", "\n", "# Create the enhanced classifier only if the custom analyzer was created successfully\n", "if analyzer_id:\n", - " try:\n", - " print(f\"πŸ”¨ Creating enhanced classifier: {enhanced_classifier_id}\")\n", - " print(\"\\nπŸ“‹ Configuration:\")\n", - " print(\" β€’ Loan application documents β†’ Custom analyzer with field extraction\")\n", - " print(\" β€’ Invoice documents β†’ Prebuilt invoice analyzer\")\n", - " print(\" β€’ Bank_Statement documents β†’ Standard processing\")\n", - " \n", - " response = content_understanding_client.begin_create_classifier(enhanced_classifier_id, enhanced_classifier_schema)\n", - " result = content_understanding_client.poll_result(response)\n", - " \n", - " print(\"\\nβœ… Enhanced classifier created successfully!\")\n", - " \n", - " except Exception as e:\n", - " print(f\"\\n❌ Error creating enhanced classifier: {e}\")\n", - "else:\n", - " print(\"⚠️ Skipping enhanced classifier creation - custom analyzer was not created successfully.\")" + " poller = await client.content_classifiers.begin_create_or_replace(\n", + " classifier_id=classifier_id,\n", + " resource=enhanced_classifier_schema\n", + " )\n", + "\n", + " # Wait for the classifier to be created\n", + " print(f\"⏳ Waiting for classifier creation to complete...\")\n", + " await poller.result()\n", + " print(f\"βœ… Classifier '{classifier_id}' created successfully!\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 11. Process Document with Enhanced Classifier\n", + "## Process Document with Enhanced Classifier\n", "\n", "Process the document again using the enhanced classifier.\n", "Invoices and loan applications will now have additional fields extracted." @@ -479,24 +461,24 @@ "metadata": {}, "outputs": [], "source": [ - "if 'enhanced_classifier_id' in locals() and analyzer_id:\n", - " try:\n", - " # Verify the document exists\n", - " if not file_location.exists():\n", - " raise FileNotFoundError(f\"Document not found at {file_location}\")\n", - " \n", - " # Process document with enhanced classifier\n", - " print(\"πŸ“„ Processing document with enhanced classifier\")\n", - " print(f\" Document: {file_location.name}\")\n", - " print(\"\\n⏳ Processing with classification and field extraction...\")\n", - " \n", - " response = content_understanding_client.begin_classify(enhanced_classifier_id, file_location=str(file_location))\n", - " enhanced_result = content_understanding_client.poll_result(response, timeout_seconds=360)\n", - " \n", - " print(\"\\nβœ… Enhanced processing completed!\")\n", - " \n", - " except Exception as e:\n", - " print(f\"\\n❌ Error processing document: {e}\")\n", + "if classifier_id and analyzer_id:\n", + " pdf_path = \"../data/mixed_financial_docs.pdf\"\n", + " print(f\"πŸ“„ Reading document file: {pdf_path}\")\n", + " with open(pdf_path, \"rb\") as pdf_file:\n", + " pdf_content = pdf_file.read()\n", + "\n", + " # Begin binary classification operation\n", + " print(f\"πŸ” Starting binary classification with classifier '{classifier_id}'...\")\n", + " classification_poller = await client.content_classifiers.begin_classify_binary(\n", + " classifier_id=classifier_id,\n", + " input=pdf_content,\n", + " content_type=\"application/pdf\",\n", + " )\n", + "\n", + " # Wait for classification completion\n", + " print(f\"⏳ Waiting for classification to complete...\")\n", + " classification_result = await classification_poller.result()\n", + " print(f\"βœ… Classification completed successfully!\")\n", "else:\n", " print(\"⚠️ Skipping enhanced classification - enhanced classifier was not created.\")" ] @@ -505,7 +487,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 12. View Enhanced Results with Extracted Fields\n", + "## View Enhanced Results with Extracted Fields\n", "\n", "Review the classification results alongside extracted fields from loan application documents." ] @@ -516,41 +498,22 @@ "metadata": {}, "outputs": [], "source": [ - "# Display enhanced classification results\n", - "if 'enhanced_result' in locals() and enhanced_result:\n", - " result_data = enhanced_result.get(\"result\", {})\n", - " contents = result_data.get(\"contents\", [])\n", - " \n", - " print(\"πŸ“Š ENHANCED CLASSIFICATION RESULTS\")\n", - " print(\"=\" * 70)\n", - " print(f\"\\nTotal sections found: {len(contents)}\")\n", - " \n", - " # Iterate through each document section\n", - " for i, content in enumerate(contents, 1):\n", - " print(f\"\\n{'='*70}\")\n", - " print(f\"SECTION {i}\")\n", - " print(f\"{'='*70}\")\n", - " \n", - " category = content.get('category', 'Unknown')\n", - " print(f\"\\nπŸ“ Category: {category}\")\n", - " print(f\"πŸ“„ Pages: {content.get('startPageNumber', '?')} - {content.get('endPageNumber', '?')}\")\n", - " \n", - " # Display extracted fields if available\n", - " fields = content.get('fields', {})\n", - " if fields:\n", - " print(\"\\nπŸ” Extracted Information:\")\n", - " for field_name, field_data in fields.items():\n", - " print(f\"\\n {field_name}:\")\n", - " print(f\" β€’ Value: {field_data}\")\n", - "else:\n", - " print(\"❌ No enhanced results available. Please run the enhanced classification step first.\")" + "# Display classification results\n", + "print(f\"πŸ“Š Classification Results: {json.dumps(classification_result.as_dict(), indent=2)}\")\n", + "for content in classification_result.contents:\n", + " if hasattr(content, \"classifications\") and content.classifications:\n", + " for classification in content.classifications:\n", + " print(f\" Category: {classification.category}\")\n", + " print(f\" Confidence: {classification.confidence}\")\n", + " print(f\" Score: {classification.score}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "You can also view the full JSON result below." + "## Saving Classification Results\n", + "The classification result is saved to a JSON file for later analysis." ] }, { @@ -559,25 +522,58 @@ "metadata": {}, "outputs": [], "source": [ - "print(json.dumps(enhanced_result, indent=2))" + "# Save the classification result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=classification_result.as_dict(),\n", + " filename_prefix=\"content_classifiers_classify_binary\",\n", + ")\n", + "print(f\"πŸ’Ύ Classification result saved to: {saved_file_path}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Summary and Next Steps\n", - "\n", - "Congratulations! You have successfully:\n", - "1. βœ… Created a basic classifier to categorize documents\n", - "2. βœ… Created a custom analyzer to extract specific fields\n", - "3. βœ… Combined them into an enhanced classifier for intelligent document processing" + "## Clean up the created analyzer\n", + "After the demo completes, the analyzer is automatically deleted to prevent resource accumulation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Clean up the created analyzer (demo cleanup)\n", + "print(f\"πŸ—‘οΈ Deleting analyzer '{analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=analyzer_id)\n", + "print(f\"βœ… Analyzer '{analyzer_id}' deleted successfully!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Clean up the created classifier\n", + "After the demo completes, the classifier is automatically deleted to prevent resource accumulation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Clean up the created classifier (demo cleanup)\n", + "print(f\"πŸ—‘οΈ Deleting classifier '{classifier_id}' (demo cleanup)...\")\n", + "await client.content_classifiers.delete(classifier_id=classifier_id)\n", + "print(f\"βœ… Classifier '{classifier_id}' deleted successfully!\")" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "py312", "language": "python", "name": "python3" }, @@ -591,7 +587,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.12" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/notebooks/content_extraction.ipynb b/notebooks/content_extraction.ipynb index a06b81c..e2b89e2 100644 --- a/notebooks/content_extraction.ipynb +++ b/notebooks/content_extraction.ipynb @@ -59,53 +59,42 @@ "import logging\n", "import json\n", "import os\n", + "from pathlib import Path\n", "import sys\n", + "from dotenv import load_dotenv\n", + "from azure.core.credentials import AzureKeyCredential\n", + "from azure.identity.aio import DefaultAzureCredential\n", + "from azure.ai.contentunderstanding.aio import ContentUnderstandingClient\n", + "from azure.ai.contentunderstanding.models import (\n", + " AnalyzeResult,\n", + " ContentAnalyzer,\n", + " ContentAnalyzerConfig,\n", + " AnalysisMode,\n", + " ProcessingLocation,\n", + " AudioVisualContent,\n", + ")\n", + "from datetime import datetime\n", + "from typing import Any\n", "import uuid\n", - "from pathlib import Path\n", - "from dotenv import find_dotenv, load_dotenv\n", - "from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n", "\n", - "load_dotenv(find_dotenv())\n", - "logging.basicConfig(level=logging.INFO)\n", - "\n", - "# For authentication, you can use either token-based auth or subscription key; only one is required\n", - "AZURE_AI_ENDPOINT = os.getenv(\"AZURE_AI_ENDPOINT\")\n", - "# IMPORTANT: Replace with your actual subscription key or set it in your \".env\" file if not using token authentication\n", - "AZURE_AI_API_KEY = os.getenv(\"AZURE_AI_API_KEY\")\n", - "AZURE_AI_API_VERSION = os.getenv(\"AZURE_AI_API_VERSION\", \"2025-05-01-preview\")\n", - "\n", - "# Add the parent directory to the path to use shared modules\n", - "parent_dir = Path(Path.cwd()).parent\n", - "sys.path.append(str(parent_dir))\n", - "from python.content_understanding_client import AzureContentUnderstandingClient\n", - "\n", - "credential = DefaultAzureCredential()\n", - "token_provider = get_bearer_token_provider(credential, \"https://cognitiveservices.azure.com/.default\")\n", - "\n", - "client = AzureContentUnderstandingClient(\n", - " endpoint=AZURE_AI_ENDPOINT,\n", - " api_version=AZURE_AI_API_VERSION,\n", - " # IMPORTANT: Comment out token_provider if using subscription key\n", - " token_provider=token_provider,\n", - " # IMPORTANT: Uncomment the following line if using subscription key\n", - " # subscription_key=AZURE_AI_API_KEY,\n", - " x_ms_useragent=\"azure-ai-content-understanding-python/content_extraction\", # This header is used for sample usage telemetry; please comment out this line if you want to opt out.\n", + "# Add the parent directory to the Python path to import the sample_helper module\n", + "sys.path.append(os.path.join(os.path.dirname(os.getcwd()), 'python'))\n", + "from extension.sample_helper import (\n", + " extract_operation_id_from_poller,\n", + " PollerType,\n", + " save_json_to_file,\n", + " save_keyframe_image_to_file,\n", ")\n", "\n", - "# Utility function to save images\n", - "from PIL import Image\n", - "from io import BytesIO\n", - "import re\n", + "load_dotenv()\n", + "logging.basicConfig(level=logging.INFO)\n", "\n", - "def save_image(image_id: str, response):\n", - " raw_image = client.get_image_from_analyze_operation(analyze_response=response,\n", - " image_id=image_id\n", - " )\n", - " image = Image.open(BytesIO(raw_image))\n", - " # To display the image, uncomment the following line:\n", - " # image.show()\n", - " Path(\".cache\").mkdir(exist_ok=True)\n", - " image.save(f\".cache/{image_id}.jpg\", \"JPEG\")\n" + "endpoint = os.environ.get(\"AZURE_CONTENT_UNDERSTANDING_ENDPOINT\")\n", + "# Return AzureKeyCredential if AZURE_CONTENT_UNDERSTANDING_KEY is set, otherwise DefaultAzureCredential\n", + "key = os.getenv(\"AZURE_CONTENT_UNDERSTANDING_KEY\")\n", + "credential = AzureKeyCredential(key) if key else DefaultAzureCredential()\n", + "# Create the ContentUnderstandingClient\n", + "client = ContentUnderstandingClient(endpoint=endpoint, credential=credential)" ] }, { @@ -123,14 +112,19 @@ "metadata": {}, "outputs": [], "source": [ - "ANALYZER_SAMPLE_FILE = '../data/invoice.pdf'\n", - "ANALYZER_ID = 'prebuilt-documentAnalyzer'\n", + "analyzer_sample_file = '../data/invoice.pdf'\n", + "analyzer_id = 'prebuilt-documentAnalyzer'\n", "\n", - "# Analyze document file\n", - "response = client.begin_analyze(ANALYZER_ID, file_location=ANALYZER_SAMPLE_FILE)\n", - "result_json = client.poll_result(response)\n", + "with open(analyzer_sample_file, \"rb\") as f:\n", + " pdf_bytes = f.read()\n", "\n", - "print(json.dumps(result_json, indent=2))" + "print(f\"πŸ” Analyzing {analyzer_sample_file} with prebuilt-documentAnalyzer...\")\n", + "poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=analyzer_id,\n", + " input=pdf_bytes,\n", + " content_type=\"application/pdf\"\n", + ")\n", + "result: AnalyzeResult = await poller.result()" ] }, { @@ -146,7 +140,11 @@ "metadata": {}, "outputs": [], "source": [ - "print(result_json[\"result\"][\"contents\"][0][\"markdown\"])\n" + "print(\"\\nπŸ“„ Markdown Content:\")\n", + "print(\"=\" * 50)\n", + "content = result.contents[0]\n", + "print(content.markdown)\n", + "print(\"=\" * 50)" ] }, { @@ -162,23 +160,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(json.dumps(result_json[\"result\"][\"contents\"][0], indent=2))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> This output helps you retrieve structural information about the tables embedded within the document." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(json.dumps(result_json[\"result\"][\"contents\"][0][\"tables\"], indent=2))" + "print(json.dumps(result.as_dict(), indent=2))" ] }, { @@ -206,14 +188,62 @@ "metadata": {}, "outputs": [], "source": [ - "ANALYZER_SAMPLE_FILE = '../data/audio.wav'\n", - "ANALYZER_ID = 'prebuilt-audioAnalyzer'\n", + "analyzer_id = f\"audio-sample-{datetime.now().strftime('%Y%m%d')}-{datetime.now().strftime('%H%M%S')}-{uuid.uuid4().hex[:8]}\"\n", + "\n", + "# Create a marketing video analyzer using object model\n", + "print(f\"πŸ”§ Creating marketing video analyzer '{analyzer_id}'...\")\n", "\n", - "# Analyze audio file\n", - "response = client.begin_analyze(ANALYZER_ID, file_location=ANALYZER_SAMPLE_FILE)\n", - "result_json = client.poll_result(response)\n", + "audio_analyzer = ContentAnalyzer(\n", + " base_analyzer_id=\"prebuilt-audioAnalyzer\",\n", + " config=ContentAnalyzerConfig(return_details=True),\n", + " description=\"Marketing audio analyzer for result file demo\",\n", + " mode=AnalysisMode.STANDARD,\n", + " processing_location=ProcessingLocation.GLOBAL,\n", + " tags={\"demo_type\": \"audio_analysis\"},\n", + ")\n", + "\n", + " # Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=analyzer_id,\n", + " resource=audio_analyzer,\n", + ")\n", + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{analyzer_id}' created successfully!\")\n", "\n", - "print(json.dumps(result_json, indent=2))" + "# Analyze audio file with the created analyzer\n", + "audio_file_path = \"../data/audio.wav\"\n", + "print(f\"πŸ” Analyzing audio file from path: {audio_file_path} with analyzer '{analyzer_id}'...\")\n", + "\n", + "with open(audio_file_path, \"rb\") as f:\n", + " audio_data = f.read()\n", + "\n", + "# Begin audio analysis operation\n", + "print(f\"🎬 Starting audio analysis with analyzer '{analyzer_id}'...\")\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=analyzer_id,\n", + " input=audio_data,\n", + " content_type=\"application/octet-stream\",\n", + ")\n", + "\n", + " # Wait for analysis completion\n", + "print(f\"⏳ Waiting for audio analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Audio analysis completed successfully!\")\n", + "print(f\"πŸ“Š Analysis Results: {json.dumps(analysis_result.as_dict(), indent=2)}\")\n", + "\n", + "# Clean up the created analyzer (demo cleanup)\n", + "print(f\"πŸ—‘οΈ Deleting analyzer '{analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=analyzer_id)\n", + "print(f\"βœ… Analyzer '{analyzer_id}' deleted successfully!\")" ] }, { @@ -237,32 +267,146 @@ "metadata": {}, "outputs": [], "source": [ - "ANALYZER_SAMPLE_FILE = '../data/FlightSimulator.mp4'\n", - "ANALYZER_ID = 'prebuilt-videoAnalyzer'\n", + "analyzer_id = f\"video-sample-{datetime.now().strftime('%Y%m%d')}-{datetime.now().strftime('%H%M%S')}-{uuid.uuid4().hex[:8]}\"\n", "\n", - "# Analyze video file\n", - "response = client.begin_analyze(ANALYZER_ID, file_location=ANALYZER_SAMPLE_FILE)\n", - "result_json = client.poll_result(response)\n", + "video_analyzer = ContentAnalyzer(\n", + " base_analyzer_id='prebuilt-videoAnalyzer', \n", + " config=ContentAnalyzerConfig(return_details=True), \n", + " description=\"Marketing video analyzer for result file demo\", \n", + " mode=AnalysisMode.STANDARD,\n", + " processing_location=ProcessingLocation.GLOBAL,\n", + " tags={\"demo_type\": \"video_analysis\"}\n", + ")\n", "\n", - "print(json.dumps(result_json, indent=2))\n", + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=analyzer_id,\n", + " resource=video_analyzer,\n", + ")\n", + "\n", + " # Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", "\n", - "# Save keyframes (optional)\n", - "keyframe_ids = set()\n", - "result_data = result_json.get(\"result\", {})\n", - "contents = result_data.get(\"contents\", [])\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{analyzer_id}' created successfully!\")\n", "\n", - "# Extract keyframe IDs from markdown content\n", - "for content in contents:\n", - " markdown_content = content.get(\"markdown\", \"\")\n", - " if isinstance(markdown_content, str):\n", - " keyframe_ids.update(re.findall(r\"(keyFrame\\.\\d+)\\.jpg\", markdown_content))\n", + "# Use the FlightSimulator.mp4 video file from remote location\n", + "video_file_path = \"../data/FlightSimulator.mp4\"\n", + "print(f\"πŸ“Ή Using video file from URL: {video_file_path}\")\n", + "\n", + "with open(video_file_path, \"rb\") as f:\n", + " video_data = f.read()\n", + "\n", + "# Begin video analysis operation\n", + "print(f\"🎬 Starting video analysis with analyzer '{analyzer_id}'...\")\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=analyzer_id,\n", + " input=video_data,\n", + " content_type=\"application/octet-stream\"\n", + ")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for video analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(json.dumps(analysis_result.as_dict(), indent=2))\n", + "print(f\"βœ… Video analysis completed successfully!\")\n", + "\n", + "# Extract operation ID for get_result_file\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the result to see what files are available\n", + "print(f\"πŸ” Getting analysis result to find available files...\")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", "\n", - "# Output unique keyframe IDs\n", - "print(\"Unique Keyframe IDs:\", keyframe_ids)\n", + "# The actual analysis result is in operation_status.result\n", + "operation_result: Any = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + "else:\n", + " print(f\"βœ… Analysis result contains {len(operation_result.contents)} contents\")\n", "\n", - "# Save all keyframe images\n", - "for keyframe_id in keyframe_ids:\n", - " save_image(keyframe_id, response)" + "# Look for keyframe times in the analysis result\n", + "keyframe_times_ms: list[int] = []\n", + "for content in operation_result.contents:\n", + " if isinstance(content, AudioVisualContent):\n", + " video_content: AudioVisualContent = content\n", + " print(f\"KeyFrameTimesMs: {video_content.key_frame_times_ms}\")\n", + " print(video_content)\n", + " keyframe_times_ms.extend(video_content.key_frame_times_ms or [])\n", + " print(f\"πŸ“Ή Found {len(keyframe_times_ms)} keyframes in video content\")\n", + " break\n", + " else:\n", + " print(f\"Content is not an AudioVisualContent: {content}\")\n", + "\n", + "if not keyframe_times_ms:\n", + " print(\"⚠️ No keyframe times found in the analysis result\")\n", + "else:\n", + " print(f\"πŸ–ΌοΈ Found {len(keyframe_times_ms)} keyframe times in milliseconds\")\n", + "\n", + "# Build keyframe filenames using the time values\n", + "keyframe_files = [f\"keyFrame.{time_ms}\" for time_ms in keyframe_times_ms]\n", + "\n", + "# Download and save a few keyframe images as examples (first, middle, last)\n", + "if len(keyframe_files) >= 3:\n", + " frames_to_download = {\n", + " keyframe_files[0],\n", + " keyframe_files[-1],\n", + " keyframe_files[len(keyframe_files) // 2],\n", + " }\n", + "else:\n", + " frames_to_download = set(keyframe_files)\n", + "\n", + "files_to_download = list(frames_to_download)\n", + "print(\n", + " f\"πŸ“₯ Downloading {len(files_to_download)} keyframe images as examples: {files_to_download}\"\n", + ")\n", + "\n", + "for keyframe_id in files_to_download:\n", + " print(f\"πŸ“₯ Getting result file: {keyframe_id}\")\n", + "\n", + " # Get the result file (keyframe image)\n", + " response: Any = await client.content_analyzers.get_result_file(\n", + " operation_id=analysis_operation_id,\n", + " path=keyframe_id,\n", + " )\n", + "\n", + " # Handle the response which may be bytes or an async iterator of bytes\n", + " if isinstance(response, (bytes, bytearray)):\n", + " image_content = bytes(response)\n", + " else:\n", + " chunks: list[bytes] = []\n", + " async for chunk in response:\n", + " chunks.append(chunk)\n", + " image_content = b\"\".join(chunks)\n", + "\n", + " print(\n", + " f\"βœ… Retrieved image file for {keyframe_id} ({len(image_content)} bytes)\"\n", + " )\n", + "\n", + " # Save the image file\n", + " saved_file_path = save_keyframe_image_to_file(\n", + " image_content=image_content,\n", + " keyframe_id=keyframe_id,\n", + " test_name=\"content_analyzers_get_result_file\",\n", + " test_py_file_dir=os.getcwd(),\n", + " identifier=analyzer_id,\n", + " )\n", + " print(f\"πŸ’Ύ Keyframe image saved to: {saved_file_path}\")\n", + "\n", + "# Clean up the created analyzer (demo cleanup)\n", + "print(f\"πŸ—‘οΈ Deleting analyzer '{analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=analyzer_id)\n", + "print(f\"βœ… Analyzer '{analyzer_id}' deleted successfully!\")" ] }, { @@ -281,14 +425,78 @@ "metadata": {}, "outputs": [], "source": [ - "ANALYZER_SAMPLE_FILE = '../data/FlightSimulator.mp4'\n", - "ANALYZER_ID = 'prebuilt-videoAnalyzer'\n", + "analyzer_id = f\"video-sample-{datetime.now().strftime('%Y%m%d')}-{datetime.now().strftime('%H%M%S')}-{uuid.uuid4().hex[:8]}\"\n", + "\n", + "# Create a marketing video analyzer using object model\n", + "print(f\"πŸ”§ Creating marketing video analyzer '{analyzer_id}'...\")\n", + "\n", + "video_analyzer = ContentAnalyzer(\n", + " base_analyzer_id='prebuilt-videoAnalyzer',\n", + " config=ContentAnalyzerConfig(\n", + " return_details=True,\n", + " ),\n", + " description=\"Marketing video analyzer for result file demo\",\n", + " mode=AnalysisMode.STANDARD,\n", + " processing_location=ProcessingLocation.GLOBAL,\n", + " tags={\"demo_type\": \"video_analysis\"},\n", + ")\n", + "\n", + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=analyzer_id,\n", + " resource=video_analyzer,\n", + ")\n", + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{analyzer_id}' created successfully!\")\n", + "\n", + "# Use the FlightSimulator.mp4 video file from remote location\n", + "video_file_path = \"../data/FlightSimulator.mp4\"\n", + "print(f\"πŸ“Ή Using video file from URL: {video_file_path}\")\n", "\n", - "# Analyze video file with face recognition\n", - "response = client.begin_analyze(ANALYZER_ID, file_location=ANALYZER_SAMPLE_FILE)\n", - "result_json = client.poll_result(response)\n", + "with open(video_file_path, \"rb\") as f:\n", + " video_data = f.read()\n", "\n", - "print(json.dumps(result_json, indent=2))" + "# Begin video analysis operation\n", + "print(f\"🎬 Starting video analysis with analyzer '{analyzer_id}'...\")\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=analyzer_id,\n", + " input=video_data,\n", + " content_type=\"application/octet-stream\"\n", + ")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for video analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(\"result: \", json.dumps(analysis_result.as_dict(), indent=2))\n", + "print(f\"βœ… Video analysis completed successfully!\")\n", + "\n", + "# Extract operation ID for get_result_file\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the result to see what files are available\n", + "print(f\"πŸ” Getting analysis result to find available files...\")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "# The actual analysis result is in operation_status.result\n", + "operation_result: Any = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + "else:\n", + " print(f\"βœ… Analysis result contains {len(operation_result.contents)} contents\")" ] }, { @@ -304,45 +512,93 @@ "metadata": {}, "outputs": [], "source": [ - "# Initialize sets to store unique face IDs and keyframe IDs\n", + "# Initialize sets to store unique face IDs\n", "face_ids = set()\n", - "keyframe_ids = set()\n", - "\n", - "# Safely extract face IDs and keyframe IDs from content\n", - "result_data = result_json.get(\"result\", {})\n", - "contents = result_data.get(\"contents\", [])\n", - "\n", - "for content in contents:\n", - " # Extract face IDs if \"faces\" field exists and is a list\n", - " faces = content.get(\"faces\", [])\n", - " if isinstance(faces, list):\n", - " for face in faces:\n", - " face_id = face.get(\"faceId\")\n", - " if face_id:\n", - " face_ids.add(f\"face.{face_id}\")\n", - "\n", - " # Extract keyframe IDs from \"markdown\" if present and a string\n", - " markdown_content = content.get(\"markdown\", \"\")\n", - " if isinstance(markdown_content, str):\n", - " keyframe_ids.update(re.findall(r\"(keyFrame\\.\\d+)\\.jpg\", markdown_content))\n", - "\n", - "# Display unique face and keyframe IDs\n", - "print(\"Unique Face IDs:\", face_ids)\n", - "print(\"Unique Keyframe IDs:\", keyframe_ids)\n", - "\n", - "# Save all face images\n", - "for face_id in face_ids:\n", - " save_image(face_id, response)\n", - "\n", - "# Save all keyframe images\n", - "for keyframe_id in keyframe_ids:\n", - " save_image(keyframe_id, response)" + "\n", + "# Look for keyframe times in the analysis result\n", + "keyframe_times_ms: list[int] = []\n", + "for content in operation_result.contents:\n", + " if isinstance(content, AudioVisualContent):\n", + " video_content: AudioVisualContent = content\n", + " print(f\"KeyFrameTimesMs: {video_content.key_frame_times_ms}\")\n", + " print(video_content)\n", + " keyframe_times_ms.extend(video_content.key_frame_times_ms or [])\n", + " print(f\"πŸ“Ή Found {len(keyframe_times_ms)} keyframes in video content\")\n", + " faces = content.get(\"faces\", [])\n", + " if isinstance(faces, list):\n", + " for face in faces:\n", + " face_id = face.get(\"faceId\")\n", + " if face_id:\n", + " face_ids.add(f\"face.{face_id}\")\n", + " break\n", + " else:\n", + " print(f\"Content is not an AudioVisualContent: {content}\")\n", + "\n", + "if not keyframe_times_ms:\n", + " print(\"⚠️ No keyframe times found in the analysis result\")\n", + "else:\n", + " print(f\"πŸ–ΌοΈ Found {len(keyframe_times_ms)} keyframe times in milliseconds\")\n", + "\n", + "# Build keyframe filenames using the time values\n", + "keyframe_files = [f\"keyFrame.{time_ms}\" for time_ms in keyframe_times_ms]\n", + "\n", + "# Download and save a few keyframe images as examples (first, middle, last)\n", + "if len(keyframe_files) >= 3:\n", + " frames_to_download = {\n", + " keyframe_files[0],\n", + " keyframe_files[-1],\n", + " keyframe_files[len(keyframe_files) // 2],\n", + " }\n", + "else:\n", + " frames_to_download = set(keyframe_files)\n", + "\n", + "files_to_download = list(frames_to_download)\n", + "print(\n", + " f\"πŸ“₯ Downloading {len(files_to_download)} keyframe images as examples: {files_to_download}\"\n", + ")\n", + "\n", + "for keyframe_id in files_to_download:\n", + " print(f\"πŸ“₯ Getting result file: {keyframe_id}\")\n", + "\n", + " # Get the result file (keyframe image)\n", + " response: Any = await client.content_analyzers.get_result_file(\n", + " operation_id=analysis_operation_id,\n", + " path=keyframe_id,\n", + " )\n", + "\n", + " # Handle the response which may be bytes or an async iterator of bytes\n", + " if isinstance(response, (bytes, bytearray)):\n", + " image_content = bytes(response)\n", + " else:\n", + " chunks: list[bytes] = []\n", + " async for chunk in response:\n", + " chunks.append(chunk)\n", + " image_content = b\"\".join(chunks)\n", + "\n", + " print(\n", + " f\"βœ… Retrieved image file for {keyframe_id} ({len(image_content)} bytes)\"\n", + " )\n", + "\n", + " # Save the image file\n", + " saved_file_path = save_keyframe_image_to_file(\n", + " image_content=image_content,\n", + " keyframe_id=keyframe_id,\n", + " test_name=\"content_analyzers_get_result_file\",\n", + " test_py_file_dir=os.getcwd(),\n", + " identifier=analyzer_id,\n", + " )\n", + " print(f\"πŸ’Ύ Keyframe image saved to: {saved_file_path}\")\n", + "\n", + "# Clean up the created analyzer (demo cleanup)\n", + "print(f\"πŸ—‘οΈ Deleting analyzer '{analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=analyzer_id)\n", + "print(f\"βœ… Analyzer '{analyzer_id}' deleted successfully!\")" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "py312", "language": "python", "name": "python3" }, @@ -356,7 +612,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.12" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/notebooks/conversational_field_extraction.ipynb b/notebooks/conversational_field_extraction.ipynb index 7872f0c..d36f2dd 100644 --- a/notebooks/conversational_field_extraction.ipynb +++ b/notebooks/conversational_field_extraction.ipynb @@ -74,12 +74,8 @@ "metadata": {}, "outputs": [], "source": [ - "import uuid\n", - "\n", - "ANALYZER_TEMPLATE = \"call_recording_pretranscribe_batch\"\n", - "CUSTOM_ANALYZER_ID = \"field-extraction-sample-\" + str(uuid.uuid4())\n", - "\n", - "(analyzer_template_path, analyzer_sample_file_path) = extraction_templates[ANALYZER_TEMPLATE]" + "analyzer_template = \"call_recording_pretranscribe_batch\"\n", + "(analyzer_template_path, analyzer_sample_file_path) = extraction_templates[analyzer_template]" ] }, { @@ -110,36 +106,46 @@ "import json\n", "import os\n", "import sys\n", - "from pathlib import Path\n", - "from dotenv import find_dotenv, load_dotenv\n", - "from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n", - "\n", - "load_dotenv(find_dotenv())\n", - "logging.basicConfig(level=logging.INFO)\n", + "import uuid\n", + "from dotenv import load_dotenv\n", + "from azure.storage.blob import ContainerSasPermissions\n", + "from azure.core.credentials import AzureKeyCredential\n", + "from azure.identity import DefaultAzureCredential\n", + "from azure.ai.contentunderstanding.aio import ContentUnderstandingClient\n", + "from azure.ai.contentunderstanding.models import (\n", + " ContentAnalyzer,\n", + " ContentAnalyzerConfig,\n", + " FieldSchema,\n", + " FieldDefinition,\n", + " FieldType,\n", + " GenerationMethod,\n", + " AnalysisMode,\n", + " ProcessingLocation,\n", + ")\n", + "from datetime import datetime\n", "\n", - "# For authentication, you may use either token-based auth or a subscription key; only one is required.\n", - "AZURE_AI_ENDPOINT = os.getenv(\"AZURE_AI_ENDPOINT\")\n", - "# IMPORTANT: Replace with your actual subscription key or configure it in the \".env\" file if not using token authentication.\n", - "AZURE_AI_API_KEY = os.getenv(\"AZURE_AI_API_KEY\")\n", - "AZURE_AI_API_VERSION = os.getenv(\"AZURE_AI_API_VERSION\", \"2025-05-01-preview\")\n", + "# Add the parent directory to the Python path to import the sample_helper module\n", + "sys.path.append(os.path.join(os.path.dirname(os.getcwd()), 'python'))\n", + "from extension.document_processor import DocumentProcessor\n", + "from extension.sample_helper import extract_operation_id_from_poller, PollerType, save_json_to_file\n", "\n", - "# Add the parent directory to the system path to access shared modules\n", - "parent_dir = Path(Path.cwd()).parent\n", - "sys.path.append(str(parent_dir))\n", - "from python.content_understanding_client import AzureContentUnderstandingClient\n", + "load_dotenv()\n", + "logging.basicConfig(level=logging.INFO)\n", "\n", - "credential = DefaultAzureCredential()\n", - "token_provider = get_bearer_token_provider(credential, \"https://cognitiveservices.azure.com/.default\")\n", + "endpoint = os.environ.get(\"AZURE_CONTENT_UNDERSTANDING_ENDPOINT\")\n", + "# Return AzureKeyCredential if AZURE_CONTENT_UNDERSTANDING_KEY is set, otherwise DefaultAzureCredential\n", + "key = os.getenv(\"AZURE_CONTENT_UNDERSTANDING_KEY\")\n", + "credential = AzureKeyCredential(key) if key else DefaultAzureCredential()\n", + "# Create the ContentUnderstandingClient\n", + "client = ContentUnderstandingClient(endpoint=endpoint, credential=credential)\n", + "print(\"βœ… ContentUnderstandingClient created successfully\")\n", "\n", - "client = AzureContentUnderstandingClient(\n", - " endpoint=AZURE_AI_ENDPOINT,\n", - " api_version=AZURE_AI_API_VERSION,\n", - " # IMPORTANT: Comment out token_provider if using subscription key\n", - " token_provider=token_provider,\n", - " # IMPORTANT: Uncomment the following line if using subscription key\n", - " # subscription_key=AZURE_AI_API_KEY,\n", - " # x_ms_useragent=\"azure-ai-content-understanding-python/field_extraction\", # This header is used for sample usage telemetry. Please comment out if you want to opt out.\n", - ")" + "try:\n", + " processor = DocumentProcessor(client)\n", + " print(\"βœ… DocumentProcessor created successfully\")\n", + "except Exception as e:\n", + " print(f\"❌ Failed to create DocumentProcessor: {e}\")\n", + " raise" ] }, { @@ -155,10 +161,109 @@ "metadata": {}, "outputs": [], "source": [ - "response = client.begin_create_analyzer(CUSTOM_ANALYZER_ID, analyzer_template_path=analyzer_template_path)\n", - "result = client.poll_result(response)\n", + "analyzer_id = f\"conversational_field_extraction-sample-{datetime.now().strftime('%Y%m%d')}-{datetime.now().strftime('%H%M%S')}-{uuid.uuid4().hex[:8]}\"\n", + "\n", + "# Create a custom analyzer using object model\n", + "print(f\"πŸ”§ Creating custom analyzer '{analyzer_id}'...\")\n", + "\n", + "content_analyzer = ContentAnalyzer(\n", + " base_analyzer_id=\"prebuilt-audioAnalyzer\",\n", + " config=ContentAnalyzerConfig(\n", + " return_details=True,\n", + " ),\n", + " description=\"Sample call recording analytics\",\n", + " field_schema=FieldSchema(\n", + " fields={\n", + " \"Summary\": FieldDefinition(\n", + " description=\"A one-paragraph summary\",\n", + " method=GenerationMethod.GENERATE,\n", + " type=FieldType.STRING,\n", + " ),\n", + " \"Topics\": FieldDefinition(\n", + " description=\"Top 5 topics mentioned\",\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " items_property={\n", + " \"type\": \"string\",\n", + " }\n", + " ),\n", + " \"Companies\": FieldDefinition(\n", + " description=\"List of companies mentioned\",\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " items_property={\n", + " \"type\": \"string\"\n", + " }\n", + " ),\n", + " \"People\": FieldDefinition(\n", + " description=\"List of people mentioned\",\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " properties={\n", + " \"Name\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Person's name\"\n", + " ),\n", + " \"Role\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Person's title/role\"\n", + " )\n", + " }\n", + " )\n", + " ),\n", + " \"Sentiment\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.CLASSIFY,\n", + " description=\"Overall sentiment\",\n", + " enum=[\n", + " \"Positive\",\n", + " \"Neutral\",\n", + " \"Negative\"\n", + " ]\n", + " ),\n", + " \"Categories\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.CLASSIFY,\n", + " description=\"List of relevant categories\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.STRING,\n", + " enum=[\n", + " \"Agriculture\",\n", + " \"Business\",\n", + " \"Finance\",\n", + " \"Health\",\n", + " \"Insurance\",\n", + " \"Mining\",\n", + " \"Pharmaceutical\",\n", + " \"Retail\",\n", + " \"Technology\",\n", + " \"Transportation\"\n", + " ]\n", + " )\n", + " )\n", + " }\n", + " )\n", + ")\n", "\n", - "print(json.dumps(result, indent=2))" + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=analyzer_id,\n", + " resource=content_analyzer,\n", + " content_type=\"application/json\"\n", + ")\n", + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{analyzer_id}' created successfully!\")" ] }, { @@ -181,7 +286,7 @@ "metadata": {}, "outputs": [], "source": [ - "from python.extension.transcripts_processor import TranscriptsProcessor\n", + "from extension.transcripts_processor import TranscriptsProcessor\n", "\n", "test_file_path = analyzer_sample_file_path\n", "\n", @@ -191,11 +296,55 @@ "if \"WEBVTT\" not in webvtt_output:\n", " print(\"Error: The output is not in WebVTT format.\")\n", "else:\n", - " response = client.begin_analyze(CUSTOM_ANALYZER_ID, file_location=webvtt_output_file_path)\n", - " print(\"Response:\", response)\n", - " result_json = client.poll_result(response)\n", + " # Read the sample invoice PDF file\n", + " with open(webvtt_output_file_path, 'r', encoding='utf-8') as f:\n", + " webvtt_content = f.read()\n", + "\n", + " print(f\"βœ… Sample WebVTT file read successfully from {webvtt_output_file_path}\")\n", + " # Begin document analysis operation\n", + " print(f\"πŸ” Starting document analysis with analyzer '{analyzer_id}'...\")\n", + " analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=analyzer_id,\n", + " input=webvtt_content,\n", + " content_type=\"application/octet-stream\",\n", + " )\n", + "\n", + " # Wait for analysis completion\n", + " print(f\"⏳ Waiting for document analysis to complete...\")\n", + " analysis_result = await analysis_poller.result()\n", + " print(f\"βœ… Document analysis completed successfully!\")\n", + "\n", + " # Extract operation ID for get_result\n", + " analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + " )\n", + " print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + " # Get the analysis result using the operation ID\n", + " print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + " )\n", + " operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + " )\n", + "\n", + " print(f\"βœ… Analysis result retrieved successfully!\")\n", + " print(f\" Operation ID: {operation_status.id}\")\n", + " print(f\" Status: {operation_status.status}\")\n", + "\n", + " # The actual analysis result is in operation_status.result\n", + " operation_result = operation_status.result\n", + " if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + " \n", + " print(f\" Result contains {len(operation_result.contents)} contents\")\n", "\n", - "print(json.dumps(result_json, indent=2))\n" + " # Save the analysis result to a file\n", + " saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"conversational_field_extraction_get_result\",\n", + " )\n", + " print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")\n" ] }, { @@ -212,13 +361,16 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(CUSTOM_ANALYZER_ID)" + "# Clean up the created analyzer (demo cleanup)\n", + "print(f\"πŸ—‘οΈ Deleting analyzer '{analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=analyzer_id)\n", + "print(f\"βœ… Analyzer '{analyzer_id}' deleted successfully!\")" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "py312", "language": "python", "name": "python3" }, @@ -232,7 +384,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.12" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/notebooks/field_extraction.ipynb b/notebooks/field_extraction.ipynb index 71a0b31..fc555aa 100644 --- a/notebooks/field_extraction.ipynb +++ b/notebooks/field_extraction.ipynb @@ -59,36 +59,45 @@ "import os\n", "import sys\n", "import uuid\n", - "from pathlib import Path\n", - "from dotenv import find_dotenv, load_dotenv\n", - "from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n", - "\n", - "load_dotenv(find_dotenv())\n", + "from dotenv import load_dotenv\n", + "from azure.storage.blob import ContainerSasPermissions\n", + "from azure.core.credentials import AzureKeyCredential\n", + "from azure.identity import DefaultAzureCredential\n", + "from azure.ai.contentunderstanding.aio import ContentUnderstandingClient\n", + "from azure.ai.contentunderstanding.models import (\n", + " ContentAnalyzer,\n", + " ContentAnalyzerConfig,\n", + " FieldSchema,\n", + " FieldDefinition,\n", + " FieldType,\n", + " GenerationMethod,\n", + " AnalysisMode,\n", + " ProcessingLocation,\n", + " SegmentationMode\n", + ")\n", + "\n", + "# Add the parent directory to the Python path to import the sample_helper module\n", + "sys.path.append(os.path.join(os.path.dirname(os.getcwd()), 'python'))\n", + "from extension.document_processor import DocumentProcessor\n", + "from extension.sample_helper import extract_operation_id_from_poller, PollerType, save_json_to_file\n", + "\n", + "load_dotenv()\n", "logging.basicConfig(level=logging.INFO)\n", "\n", - "# For authentication, you can use either token-based auth or subscription key, and only one of them is required\n", - "AZURE_AI_ENDPOINT = os.getenv(\"AZURE_AI_ENDPOINT\")\n", - "# IMPORTANT: Replace with your actual subscription key or set up in \".env\" file if not using token auth\n", - "AZURE_AI_API_KEY = os.getenv(\"AZURE_AI_API_KEY\")\n", - "AZURE_AI_API_VERSION = os.getenv(\"AZURE_AI_API_VERSION\", \"2025-05-01-preview\")\n", - "\n", - "# Add the parent directory to the path to use shared modules\n", - "parent_dir = Path(Path.cwd()).parent\n", - "sys.path.append(str(parent_dir))\n", - "from python.content_understanding_client import AzureContentUnderstandingClient\n", - "\n", - "credential = DefaultAzureCredential()\n", - "token_provider = get_bearer_token_provider(credential, \"https://cognitiveservices.azure.com/.default\")\n", + "endpoint = os.environ.get(\"AZURE_CONTENT_UNDERSTANDING_ENDPOINT\")\n", + "# Return AzureKeyCredential if AZURE_CONTENT_UNDERSTANDING_KEY is set, otherwise DefaultAzureCredential\n", + "key = os.getenv(\"AZURE_CONTENT_UNDERSTANDING_KEY\")\n", + "credential = AzureKeyCredential(key) if key else DefaultAzureCredential()\n", + "# Create the ContentUnderstandingClient\n", + "client = ContentUnderstandingClient(endpoint=endpoint, credential=credential)\n", + "print(\"βœ… ContentUnderstandingClient created successfully\")\n", "\n", - "client = AzureContentUnderstandingClient(\n", - " endpoint=AZURE_AI_ENDPOINT,\n", - " api_version=AZURE_AI_API_VERSION,\n", - " # IMPORTANT: Comment out token_provider if using subscription key\n", - " token_provider=token_provider,\n", - " # IMPORTANT: Uncomment this if using subscription key\n", - " # subscription_key=AZURE_AI_API_KEY,\n", - " x_ms_useragent=\"azure-ai-content-understanding-python/field_extraction\", # This header is used for sample usage telemetry, please comment out this line if you want to opt out.\n", - ")" + "try:\n", + " processor = DocumentProcessor(client)\n", + " print(\"βœ… DocumentProcessor created successfully\")\n", + "except Exception as e:\n", + " print(f\"❌ Failed to create DocumentProcessor: {e}\")\n", + " raise" ] }, { @@ -213,6 +222,65 @@ "Now let's create the invoice analyzer and process our sample invoice:" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "invoice_analyzer_id = \"invoice-extraction-sample-\" + str(uuid.uuid4())\n", + "\n", + "invoice_analyzer = ContentAnalyzer(\n", + " base_analyzer_id=\"prebuilt-documentAnalyzer\",\n", + " description=\"Sample invoice analyzer\",\n", + " field_schema=FieldSchema(\n", + " fields={\n", + " \"VendorName\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.EXTRACT,\n", + " description=\"Vendor issuing the invoice\"\n", + " ),\n", + " \"Items\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.EXTRACT,\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " properties={\n", + " \"Description\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.EXTRACT,\n", + " description=\"Description of the item\"\n", + " ),\n", + " \"Amount\": FieldDefinition(\n", + " type=FieldType.NUMBER,\n", + " method=GenerationMethod.EXTRACT,\n", + " description=\"Amount of the item\"\n", + " )\n", + " }\n", + " )\n", + " )\n", + " }\n", + " ),\n", + ")\n", + "print(f\"{json.dumps(invoice_analyzer.as_dict(), indent=2)}\")\n", + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=invoice_analyzer_id,\n", + " resource=invoice_analyzer,\n", + ")\n", + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{invoice_analyzer_id}' created successfully!\")" + ] + }, { "cell_type": "code", "execution_count": null, @@ -220,16 +288,54 @@ "outputs": [], "source": [ "sample_file_path = '../data/invoice.pdf'\n", - "invoice_analyzer_id = \"invoice-extraction-\" + str(uuid.uuid4())\n", "\n", - "print(f\"Creating invoice analyzer: {invoice_analyzer_id}\")\n", - "response = client.begin_create_analyzer(invoice_analyzer_id, analyzer_template_path=analyzer_template_path)\n", - "result = client.poll_result(response)\n", - "print(\"βœ… Invoice analyzer created successfully!\")\n", + "with open(sample_file_path, 'rb') as f:\n", + " invoice_content = f.read()\n", "\n", - "print(f\"Analyzing invoice: {sample_file_path}\")\n", - "response = client.begin_analyze(invoice_analyzer_id, file_location=sample_file_path)\n", - "result_json = client.poll_result(response)" + "# Begin document analysis operation\n", + "print(f\"πŸ” Starting document analysis with analyzer '{invoice_analyzer_id}'...\")\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=invoice_analyzer_id,\n", + " input=invoice_content,\n", + " content_type=\"application/pdf\",\n", + ")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for document analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Document analysis completed successfully!\")\n", + "\n", + "# Extract operation ID for get_result\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the analysis result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "print(f\"βœ… Analysis result retrieved successfully!\")\n", + "print(f\" Operation ID: {operation_status.id}\")\n", + "print(f\" Status: {operation_status.status}\")\n", + "\n", + "# The actual analysis result is in operation_status.result\n", + "operation_result = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + "\n", + "print(f\" Result contains {len(operation_result.contents)} contents\")\n", + "\n", + "# Save the analysis result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"invoice_analyzers_get_result\",\n", + ")\n", + "print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")" ] }, { @@ -247,7 +353,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(json.dumps(result_json, indent=2))" + "print(json.dumps(operation_result.as_dict(), indent=2))" ] }, { @@ -265,7 +371,7 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(invoice_analyzer_id)" + "await client.content_analyzers.delete(invoice_analyzer_id)" ] }, { @@ -307,6 +413,65 @@ "Now let's create the enhanced analyzer and process the invoice with source grounding:" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "invoice_source_analyzer_id = \"invoice-source-extraction-sample-\" + str(uuid.uuid4())\n", + "\n", + "invoice_source_analyzer = ContentAnalyzer(\n", + " base_analyzer_id=\"prebuilt-documentAnalyzer\",\n", + " description=\"Sample invoice analyzer\",\n", + " field_schema=FieldSchema(\n", + " fields={\n", + " \"VendorName\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.EXTRACT,\n", + " description=\"Vendor issuing the invoice\"\n", + " ),\n", + " \"Items\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.EXTRACT,\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " properties={\n", + " \"Description\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.EXTRACT,\n", + " description=\"Description of the item\"\n", + " ),\n", + " \"Amount\": FieldDefinition(\n", + " type=FieldType.NUMBER,\n", + " method=GenerationMethod.EXTRACT,\n", + " description=\"Amount of the item\"\n", + " )\n", + " }\n", + " )\n", + " )\n", + " }\n", + " ),\n", + ")\n", + "print(f\"{json.dumps(invoice_source_analyzer.as_dict(), indent=2)}\")\n", + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=invoice_source_analyzer_id,\n", + " resource=invoice_source_analyzer,\n", + ")\n", + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{invoice_source_analyzer_id}' created successfully!\")" + ] + }, { "cell_type": "code", "execution_count": null, @@ -314,16 +479,54 @@ "outputs": [], "source": [ "sample_file_path = '../data/invoice.pdf'\n", - "invoice_source_analyzer_id = \"invoice-field-source-\" + str(uuid.uuid4())\n", "\n", - "print(f\"Creating invoice field source analyzer: {invoice_source_analyzer_id}\")\n", - "response = client.begin_create_analyzer(invoice_source_analyzer_id, analyzer_template_path=analyzer_template_path)\n", - "result = client.poll_result(response)\n", - "print(\"βœ… Invoice field source analyzer created successfully!\")\n", + "with open(sample_file_path, 'rb') as f:\n", + " invoice_source_content = f.read()\n", + "\n", + "# Begin document analysis operation\n", + "print(f\"πŸ” Starting document analysis with analyzer '{invoice_source_analyzer_id}'...\")\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=invoice_source_analyzer_id,\n", + " input=invoice_source_content,\n", + " content_type=\"application/pdf\",\n", + ")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for document analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Document analysis completed successfully!\")\n", + "\n", + "# Extract operation ID for get_result\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the analysis result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "print(f\"βœ… Analysis result retrieved successfully!\")\n", + "print(f\" Operation ID: {operation_status.id}\")\n", + "print(f\" Status: {operation_status.status}\")\n", "\n", - "print(f\"Analyzing invoice with field source: {sample_file_path}\")\n", - "response = client.begin_analyze(invoice_source_analyzer_id, file_location=sample_file_path)\n", - "result_json = client.poll_result(response)" + "# The actual analysis result is in operation_status.result\n", + "operation_result = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + "\n", + "print(f\" Result contains {len(operation_result.contents)} contents\")\n", + "\n", + "# Save the analysis result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"invoice_source_content_analyzers_get_result\",\n", + ")\n", + "print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")" ] }, { @@ -341,7 +544,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(json.dumps(result_json, indent=2))" + "print(json.dumps(operation_result.as_dict(), indent=2))" ] }, { @@ -359,7 +562,9 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(invoice_source_analyzer_id)" + "print(f\"πŸ—‘οΈ Deleting analyzer '{invoice_source_analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=invoice_source_analyzer_id)\n", + "print(f\"βœ… Analyzer '{invoice_source_analyzer_id}' deleted successfully!\")" ] }, { @@ -402,17 +607,88 @@ "metadata": {}, "outputs": [], "source": [ - "sample_file_path = '../data/receipt.png'\n", "receipt_analyzer_id = \"receipt-extraction-\" + str(uuid.uuid4())\n", "\n", - "print(f\"Creating receipt analyzer: {receipt_analyzer_id}\")\n", - "response = client.begin_create_analyzer(receipt_analyzer_id, analyzer_template_path=analyzer_template_path)\n", - "result = client.poll_result(response)\n", - "print(\"βœ… Receipt analyzer created successfully!\")\n", + "image_analyzer = ContentAnalyzer(\n", + " base_analyzer_id=\"prebuilt-documentAnalyzer\",\n", + " description=\"Sample receipt analyzer\",\n", + " mode=AnalysisMode.STANDARD,\n", + " processing_location=ProcessingLocation.GLOBAL,\n", + " tags={\"demo_type\": \"image_analysis\"},\n", + ")\n", + "\n", + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=receipt_analyzer_id,\n", + " resource=image_analyzer,\n", + ")\n", + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", "\n", - "print(f\"Analyzing receipt: {sample_file_path}\")\n", - "response = client.begin_analyze(receipt_analyzer_id, file_location=sample_file_path)\n", - "result_json = client.poll_result(response)" + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{receipt_analyzer_id}' created successfully!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sample_file_path = '../data/receipt.png'\n", + "\n", + "with open(sample_file_path, 'rb') as f:\n", + " image_bytes: bytes = f.read()\n", + "\n", + "print(f\"πŸ” Analyzing {sample_file_path} with prebuilt-documentAnalyzer...\")\n", + "\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=receipt_analyzer_id, \n", + " input=image_bytes,\n", + " content_type=\"application/octet-stream\")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for document analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Document analysis completed successfully!\")\n", + "\n", + "# Extract operation ID for get_result\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the analysis result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "print(f\"βœ… Analysis result retrieved successfully!\")\n", + "print(f\" Operation ID: {operation_status.id}\")\n", + "print(f\" Status: {operation_status.status}\")\n", + "\n", + "# The actual analysis result is in operation_status.result\n", + "operation_result = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + " \n", + "print(f\" Result contains {len(operation_result.contents)} contents\")\n", + "\n", + "# Save the analysis result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"image_analyzer_get_result\",\n", + ")\n", + "print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")" ] }, { @@ -430,7 +706,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(json.dumps(result_json, indent=2))" + "print(json.dumps(operation_result.as_dict(), indent=2))" ] }, { @@ -448,7 +724,9 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(receipt_analyzer_id)" + "print(f\"πŸ—‘οΈ Deleting analyzer '{receipt_analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=receipt_analyzer_id)\n", + "print(f\"βœ… Analyzer '{receipt_analyzer_id}' deleted successfully!\")" ] }, { @@ -479,10 +757,108 @@ "metadata": {}, "outputs": [], "source": [ - "analyzer_template_path = '../analyzer_templates/call_recording_analytics.json'\n", - "with open(analyzer_template_path, 'r') as f:\n", - " template_content = json.load(f)\n", - " print(json.dumps(template_content, indent=2))" + "call_analyzer_id = \"call-recording-analytics-\" + str(uuid.uuid4())\n", + "# Create a custom analyzer using object model\n", + "print(f\"πŸ”§ Creating custom analyzer '{call_analyzer_id}'...\")\n", + "\n", + "call_analyzer = ContentAnalyzer(\n", + " base_analyzer_id=\"prebuilt-callCenter\",\n", + " description=\"Sample call recording analytics\",\n", + " config=ContentAnalyzerConfig(\n", + " return_details=True,\n", + " locales=[\"en-US\"]\n", + " ),\n", + " field_schema=FieldSchema(\n", + " fields={\n", + " \"Summary\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"A one-paragraph summary\"\n", + " ),\n", + " \"Topics\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Top 5 topics mentioned\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.STRING\n", + " )\n", + " ),\n", + " \"Companies\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"List of companies mentioned\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.STRING\n", + " )\n", + " ),\n", + " \"People\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"List of people mentioned\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " properties={\n", + " \"Name\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Person's name\"\n", + " ),\n", + " \"Role\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Person's title/role\"\n", + " )\n", + " }\n", + " )\n", + " ),\n", + " \"Sentiment\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.CLASSIFY,\n", + " description=\"Overall sentiment\",\n", + " enum=[\n", + " \"Positive\",\n", + " \"Neutral\",\n", + " \"Negative\"\n", + " ]\n", + " ),\n", + " \"Categories\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.CLASSIFY,\n", + " description=\"List of relevant categories\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.STRING,\n", + " enum=[\n", + " \"Agriculture\",\n", + " \"Business\",\n", + " \"Finance\",\n", + " \"Health\",\n", + " \"Insurance\",\n", + " \"Mining\",\n", + " \"Pharmaceutical\",\n", + " \"Retail\",\n", + " \"Technology\",\n", + " \"Transportation\"\n", + " ]\n", + " )\n", + " )\n", + " }\n", + " )\n", + ")\n", + "\n", + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=call_analyzer_id,\n", + " resource=call_analyzer,\n", + ")\n", + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{call_analyzer_id}' created successfully!\")" ] }, { @@ -498,18 +874,56 @@ "metadata": {}, "outputs": [], "source": [ + "# Read the sample MP3 file\n", "sample_file_path = '../data/callCenterRecording.mp3'\n", - "call_analyzer_id = \"call-recording-analytics-\" + str(uuid.uuid4())\n", - "\n", - "print(f\"Creating call recording analyzer: {call_analyzer_id}\")\n", - "response = client.begin_create_analyzer(call_analyzer_id, analyzer_template_path=analyzer_template_path)\n", - "result = client.poll_result(response)\n", - "print(\"βœ… Call recording analyzer created successfully!\")\n", - "\n", - "print(f\"Analyzing call recording: {sample_file_path}\")\n", - "print(\"⏳ Note: Audio analysis may take longer than document analysis...\")\n", - "response = client.begin_analyze(call_analyzer_id, file_location=sample_file_path)\n", - "result_json = client.poll_result(response)" + "print(f\"πŸ“„ Reading audio file: {sample_file_path}\")\n", + "with open(sample_file_path, 'rb') as f:\n", + " audio_data = f.read()\n", + "\n", + "# Begin document analysis operation\n", + "print(f\"πŸ” Starting document analysis with analyzer '{call_analyzer_id}'...\")\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=call_analyzer_id,\n", + " input=audio_data,\n", + " content_type=\"application/octet-stream\",\n", + ")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for audio analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Audio analysis completed successfully!\")\n", + "\n", + "# Extract operation ID for get_result\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the analysis result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "print(f\"βœ… Analysis result retrieved successfully!\")\n", + "print(f\" Operation ID: {operation_status.id}\")\n", + "print(f\" Status: {operation_status.status}\")\n", + "\n", + "# The actual analysis result is in operation_status.result\n", + "operation_result = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + "\n", + "print(f\" Result contains {len(operation_result.contents)} contents\")\n", + "\n", + "# Save the analysis result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"call_analyzer_get_result\",\n", + ")\n", + "print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")" ] }, { @@ -525,7 +939,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(json.dumps(result_json, indent=2))" + "print(json.dumps(operation_result.as_dict(), indent=2))" ] }, { @@ -543,7 +957,9 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(call_analyzer_id)" + "print(f\"πŸ—‘οΈ Deleting analyzer '{call_analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=call_analyzer_id)\n", + "print(f\"βœ… Analyzer '{call_analyzer_id}' deleted successfully!\")" ] }, { @@ -552,28 +968,65 @@ "source": [ "## 5. Conversational Audio Analytics\n", "\n", - "Let's analyze the same audio file but with a focus on conversational aspects like sentiment analysis and dialogue understanding.\n", - "\n", - "Conversational audio analytics template:" + "Let's analyze the same audio file but with a focus on conversational aspects like sentiment analysis and dialogue understanding." ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "analyzer_template_path = '../analyzer_templates/conversational_audio_analytics.json'\n", - "with open(analyzer_template_path, 'r') as f:\n", - " template_content = json.load(f)\n", - " print(json.dumps(template_content, indent=2))" + "Create and run conversational audio analyzer." ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "Create and run conversational audio analyzer." + "conversation_analyzer_id = \"conversational-audio-analytics-\" + str(uuid.uuid4())\n", + "# Create a custom analyzer using object model\n", + "print(f\"πŸ”§ Creating custom analyzer '{conversation_analyzer_id}'...\")\n", + "\n", + "conversation_analyzer = ContentAnalyzer(\n", + " description=\"Sample conversational audio analytics\",\n", + " base_analyzer_id=\"prebuilt-audioAnalyzer\",\n", + " config={\n", + " \"returnDetails\": True,\n", + " \"locales\": [\"en-US\"]\n", + " },\n", + " field_schema=FieldSchema(\n", + " fields={\n", + " \"Summary\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"A one-paragraph summary\"\n", + " ),\n", + " \"Sentiment\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.CLASSIFY,\n", + " description=\"Overall sentiment\",\n", + " enum=[\n", + " \"Positive\",\n", + " \"Neutral\",\n", + " \"Negative\"\n", + " ]\n", + " )\n", + " }\n", + " )\n", + ")\n", + "\n", + "# Create the analyzer\n", + "print(f\"πŸ”§ Creating custom analyzer '{conversation_analyzer_id}'...\")\n", + "response = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=conversation_analyzer_id,\n", + " resource=conversation_analyzer\n", + ")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await response.result()\n", + "print(f\"βœ… Analyzer '{conversation_analyzer_id}' created successfully!\")" ] }, { @@ -582,18 +1035,56 @@ "metadata": {}, "outputs": [], "source": [ - "sample_file_path = '../data/callCenterRecording.mp3'\n", - "conversation_analyzer_id = \"conversational-audio-analytics-\" + str(uuid.uuid4())\n", - "\n", - "print(f\"Creating conversational audio analyzer: {conversation_analyzer_id}\")\n", - "response = client.begin_create_analyzer(conversation_analyzer_id, analyzer_template_path=analyzer_template_path)\n", - "result = client.poll_result(response)\n", - "print(\"βœ… Conversational audio analyzer created successfully!\")\n", - "\n", - "print(f\"Analyzing conversational audio: {sample_file_path}\")\n", - "print(\"⏳ Note: Audio analysis may take longer than document analysis...\")\n", - "response = client.begin_analyze(conversation_analyzer_id, file_location=sample_file_path)\n", - "result_json = client.poll_result(response)" + "# Read the sample file\n", + "audio_path = sample_file_path = '../data/callCenterRecording.mp3'\n", + "print(f\"πŸ“„ Reading audio file: {audio_path}\")\n", + "with open(audio_path, \"rb\") as audio_file:\n", + " audio_content = audio_file.read()\n", + "\n", + "# Begin audio analysis operation\n", + "print(f\"πŸ” Starting audio analysis with analyzer '{conversation_analyzer_id}'...\")\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=conversation_analyzer_id,\n", + " input=audio_content,\n", + " content_type=\"application/octet-stream\",\n", + ")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for audio analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Audio analysis completed successfully!\")\n", + "\n", + "# Extract operation ID for get_result\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the analysis result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "print(f\"βœ… Analysis result retrieved successfully!\")\n", + "print(f\" Operation ID: {operation_status.id}\")\n", + "print(f\" Status: {operation_status.status}\")\n", + "\n", + "# The actual analysis result is in operation_status.result\n", + "operation_result = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + "\n", + "print(f\" Result contains {len(operation_result.contents)} contents\")\n", + "\n", + "# Save the analysis result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"conversation_analyzers_get_result\",\n", + ")\n", + "print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")" ] }, { @@ -609,7 +1100,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(json.dumps(result_json, indent=2))" + "print(json.dumps(operation_result.as_dict(), indent=2))" ] }, { @@ -627,7 +1118,9 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(conversation_analyzer_id)" + "print(f\"πŸ—‘οΈ Deleting analyzer '{conversation_analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=conversation_analyzer_id)\n", + "print(f\"βœ… Analyzer '{conversation_analyzer_id}' deleted successfully!\")" ] }, { @@ -672,22 +1165,64 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "analyzer_template_path = '../analyzer_templates/marketing_video.json'\n", - "with open(analyzer_template_path, 'r') as f:\n", - " template_content = json.load(f)\n", - " print(json.dumps(template_content, indent=2))" + "Create and run marketing video analyzer" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "Create and run marketing video analyzer" + "video_analyzer_id = \"marketing-video-analytics-\" + str(uuid.uuid4())\n", + "\n", + "# Create a custom analyzer using object model\n", + "print(f\"πŸ”§ Creating custom analyzer '{video_analyzer_id}'...\")\n", + "video_content_analyzer = ContentAnalyzer(\n", + " description=\"Sample marketing video analytics\",\n", + " base_analyzer_id=\"prebuilt-videoAnalyzer\",\n", + " config=ContentAnalyzerConfig(\n", + " return_details=True,\n", + " segmentation_mode=SegmentationMode.NO_SEGMENTATION\n", + " ),\n", + " field_schema=FieldSchema(\n", + " fields={\n", + " \"Description\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Detailed summary of the video segment, focusing on product characteristics, lighting, and color palette.\"\n", + " ),\n", + " \"Sentiment\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.CLASSIFY,\n", + " enum=[\n", + " \"Positive\",\n", + " \"Neutral\",\n", + " \"Negative\"\n", + " ]\n", + " )\n", + " }\n", + " )\n", + ")\n", + "\n", + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=video_analyzer_id,\n", + " resource=video_content_analyzer,\n", + ")\n", + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{video_analyzer_id}' created successfully!\")" ] }, { @@ -696,18 +1231,56 @@ "metadata": {}, "outputs": [], "source": [ - "sample_file_path = '../data/FlightSimulator.mp4'\n", - "video_analyzer_id = \"marketing-video-analytics-\" + str(uuid.uuid4())\n", - "\n", - "print(f\"Creating marketing video analyzer: {video_analyzer_id}\")\n", - "response = client.begin_create_analyzer(video_analyzer_id, analyzer_template_path=analyzer_template_path)\n", - "result = client.poll_result(response)\n", - "print(\"βœ… Marketing video analyzer created successfully!\")\n", - "\n", - "print(f\"Analyzing marketing video: {sample_file_path}\")\n", - "print(\"⏳ Note: Video analysis may take significantly longer than document analysis...\")\n", - "response = client.begin_analyze(video_analyzer_id, file_location=sample_file_path)\n", - "result_json = client.poll_result(response)" + "# Read the sample video file\n", + "video_path = '../data/FlightSimulator.mp4'\n", + "print(f\"πŸ“„ Reading video file: {video_path}\")\n", + "with open(video_path, \"rb\") as video_file:\n", + " video_content = video_file.read()\n", + "\n", + "# Begin video analysis operation\n", + "print(f\"πŸ” Starting video analysis with analyzer '{video_analyzer_id}'...\")\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=video_analyzer_id,\n", + " input=video_content,\n", + " content_type=\"application/octet-stream\",\n", + ")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for video analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Video analysis completed successfully!\")\n", + "\n", + "# Extract operation ID for get_result\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the analysis result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "print(f\"βœ… Analysis result retrieved successfully!\")\n", + "print(f\" Operation ID: {operation_status.id}\")\n", + "print(f\" Status: {operation_status.status}\")\n", + "\n", + "# The actual analysis result is in operation_status.result\n", + "operation_result = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + "\n", + "print(f\" Result contains {len(operation_result.contents)} contents\")\n", + "\n", + "# Save the analysis result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"video_content_analyzers_get_result\",\n", + ")\n", + "print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")" ] }, { @@ -724,7 +1297,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(json.dumps(result_json, indent=2))" + "print(json.dumps(operation_result.as_dict(), indent=2))" ] }, { @@ -742,7 +1315,9 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(video_analyzer_id)" + "print(f\"πŸ—‘οΈ Deleting analyzer '{video_analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=video_analyzer_id)\n", + "print(f\"βœ… Analyzer '{video_analyzer_id}' deleted successfully!\")" ] }, { @@ -756,22 +1331,73 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "analyzer_template_path = '../analyzer_templates/marketing_video_segmenation_auto.json'\n", - "with open(analyzer_template_path, 'r') as f:\n", - " template_content = json.load(f)\n", - " print(json.dumps(template_content, indent=2))" + "Create and run marketing video analyzer" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "Create and run marketing video analyzer" + "video_analyzer_id = \"marketing-video-analytics-\" + str(uuid.uuid4())\n", + "\n", + "# Create a custom analyzer using object model\n", + "print(f\"πŸ”§ Creating custom analyzer '{video_analyzer_id}'...\")\n", + "\n", + "video_content_analyzer = ContentAnalyzer(\n", + " description=\"Sample marketing video analytics\",\n", + " base_analyzer_id=\"prebuilt-videoAnalyzer\",\n", + " config=ContentAnalyzerConfig(\n", + " return_details=True,\n", + " segmentation_mode=SegmentationMode.NO_SEGMENTATION\n", + " ),\n", + " field_schema=FieldSchema(\n", + " fields={\n", + " \"Segments\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " properties={\n", + " \"Description\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Detailed summary of the video segment, focusing on product characteristics, lighting, and color palette.\"\n", + " ),\n", + " \"Sentiment\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.CLASSIFY,\n", + " enum=[\n", + " \"Positive\",\n", + " \"Neutral\",\n", + " \"Negative\"\n", + " ]\n", + " )\n", + " }\n", + " )\n", + " )\n", + " }\n", + " )\n", + ")\n", + "\n", + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=video_analyzer_id,\n", + " resource=video_content_analyzer,\n", + ")\n", + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{video_analyzer_id}' created successfully!\")" ] }, { @@ -780,18 +1406,56 @@ "metadata": {}, "outputs": [], "source": [ - "sample_file_path = '../data/FlightSimulator.mp4'\n", - "video_analyzer_id = \"marketing-video-analytics-\" + str(uuid.uuid4())\n", + "video_file_path = '../data/FlightSimulator.mp4'\n", + "\n", + "print(f\"πŸ“„ Reading video file: {video_path}\")\n", + "with open(video_path, \"rb\") as video_file:\n", + " video_content = video_file.read()\n", + "\n", + "# Begin video analysis operation\n", + "print(f\"πŸ” Starting video analysis with analyzer '{video_analyzer_id}'...\")\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=video_analyzer_id,\n", + " input=video_content,\n", + " content_type=\"application/octet-stream\",\n", + ")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for video analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Video analysis completed successfully!\")\n", + "\n", + "# Extract operation ID for get_result\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the analysis result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "print(f\"βœ… Analysis result retrieved successfully!\")\n", + "print(f\" Operation ID: {operation_status.id}\")\n", + "print(f\" Status: {operation_status.status}\")\n", "\n", - "print(f\"Creating marketing video analyzer: {video_analyzer_id}\")\n", - "response = client.begin_create_analyzer(video_analyzer_id, analyzer_template_path=analyzer_template_path)\n", - "result = client.poll_result(response)\n", - "print(\"βœ… Marketing video analyzer created successfully!\")\n", + "# The actual analysis result is in operation_status.result\n", + "operation_result = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", "\n", - "print(f\"Analyzing marketing video: {sample_file_path}\")\n", - "print(\"⏳ Note: Video analysis may take significantly longer than document analysis...\")\n", - "response = client.begin_analyze(video_analyzer_id, file_location=sample_file_path)\n", - "result_json = client.poll_result(response)" + "print(f\" Result contains {len(operation_result.contents)} contents\")\n", + "\n", + "# Save the analysis result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"video_content_analyzers_get_result\",\n", + ")\n", + "print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")" ] }, { @@ -809,7 +1473,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(json.dumps(result_json, indent=2))" + "print(json.dumps(operation_result.as_dict(), indent=2))" ] }, { @@ -827,7 +1491,9 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(video_analyzer_id)" + "print(f\"πŸ—‘οΈ Deleting analyzer '{video_analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=video_analyzer_id)\n", + "print(f\"βœ… Analyzer '{video_analyzer_id}' deleted successfully!\")" ] }, { @@ -842,22 +1508,74 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "analyzer_template_path = '../analyzer_templates/marketing_video_segmenation_custom.json'\n", - "with open(analyzer_template_path, 'r') as f:\n", - " template_content = json.load(f)\n", - " print(json.dumps(template_content, indent=2))" + "Create and run marketing video analyzer" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "Create and run marketing video analyzer" + "video_analyzer_id = \"marketing-video-analytics-\" + str(uuid.uuid4())\n", + "\n", + "# Create a custom analyzer using object model\n", + "print(f\"πŸ”§ Creating custom analyzer '{video_analyzer_id}'...\")\n", + "\n", + "video_content_analyzer = ContentAnalyzer(\n", + " description=\"Sample marketing video analytics\",\n", + " base_analyzer_id=\"prebuilt-videoAnalyzer\",\n", + " config=ContentAnalyzerConfig(\n", + " return_details=True,\n", + " segmentation_mode=SegmentationMode.NO_SEGMENTATION,\n", + " segmentation_definition=\"Segment the video at each clear narrative or visual transition that introduces a new marketing message, speaker, or brand moment. Segments should begin when there is a change in speaker, a shift in visual theme (e.g., logos, product shots, data center views, simulation footage, aircraft scenes), or the introduction of a new key message (e.g., quality of data, scale of infrastructure, customer benefit, real-world aviation use). Each segment should capture one distinct marketing idea or value point, ending when the focus transitions to the next theme.\"\n", + " ),\n", + " field_schema=FieldSchema(\n", + " fields={\n", + " \"Segments\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " properties={\n", + " \"Description\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Detailed summary of the video segment, focusing on product characteristics, lighting, and color palette.\"\n", + " ),\n", + " \"Sentiment\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.CLASSIFY,\n", + " enum=[\n", + " \"Positive\",\n", + " \"Neutral\",\n", + " \"Negative\"\n", + " ]\n", + " )\n", + " }\n", + " )\n", + " )\n", + " }\n", + " )\n", + ")\n", + "\n", + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=video_analyzer_id,\n", + " resource=video_content_analyzer,\n", + ")\n", + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{video_analyzer_id}' created successfully!\")" ] }, { @@ -866,18 +1584,56 @@ "metadata": {}, "outputs": [], "source": [ - "sample_file_path = '../data/FlightSimulator.mp4'\n", - "video_analyzer_id = \"marketing-video-analytics-\" + str(uuid.uuid4())\n", + "video_file_path = '../data/FlightSimulator.mp4'\n", + "\n", + "print(f\"πŸ“„ Reading video file: {video_path}\")\n", + "with open(video_path, \"rb\") as video_file:\n", + " video_content = video_file.read()\n", + "\n", + "# Begin video analysis operation\n", + "print(f\"πŸ” Starting video analysis with analyzer '{video_analyzer_id}'...\")\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=video_analyzer_id,\n", + " input=video_content,\n", + " content_type=\"application/octet-stream\",\n", + ")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for video analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Video analysis completed successfully!\")\n", "\n", - "print(f\"Creating marketing video analyzer: {video_analyzer_id}\")\n", - "response = client.begin_create_analyzer(video_analyzer_id, analyzer_template_path=analyzer_template_path)\n", - "result = client.poll_result(response)\n", - "print(\"βœ… Marketing video analyzer created successfully!\")\n", + "# Extract operation ID for get_result\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", "\n", - "print(f\"Analyzing marketing video: {sample_file_path}\")\n", - "print(\"⏳ Note: Video analysis may take significantly longer than document analysis...\")\n", - "response = client.begin_analyze(video_analyzer_id, file_location=sample_file_path)\n", - "result_json = client.poll_result(response)" + "# Get the analysis result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "print(f\"βœ… Analysis result retrieved successfully!\")\n", + "print(f\" Operation ID: {operation_status.id}\")\n", + "print(f\" Status: {operation_status.status}\")\n", + "\n", + "# The actual analysis result is in operation_status.result\n", + "operation_result = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + "\n", + "print(f\" Result contains {len(operation_result.contents)} contents\")\n", + "\n", + "# Save the analysis result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"video_content_analyzers_get_result\",\n", + ")\n", + "print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")" ] }, { @@ -896,7 +1652,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(json.dumps(result_json, indent=2))" + "print(json.dumps(operation_result.as_dict(), indent=2))" ] }, { @@ -914,7 +1670,9 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(video_analyzer_id)" + "print(f\"πŸ—‘οΈ Deleting analyzer '{video_analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=video_analyzer_id)\n", + "print(f\"βœ… Analyzer '{video_analyzer_id}' deleted successfully!\")" ] }, { @@ -932,7 +1690,8 @@ "source": [ "## 7. Chart and Image Analysis\n", "\n", - "Let's analyze a chart image to extract data points, trends, and insights from visual data representations." + "Let's analyze a chart image to extract data points, trends, and insights from visual data representations. \\\n", + "Create and run chart image analyzer:" ] }, { @@ -941,18 +1700,121 @@ "metadata": {}, "outputs": [], "source": [ - "# Image chart analytics template\n", - "analyzer_template_path = '../analyzer_templates/image_chart.json'\n", - "with open(analyzer_template_path, 'r') as f:\n", - " template_content = json.load(f)\n", - " print(json.dumps(template_content, indent=2))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create and run chart image analyzer:" + "chart_analyzer_id = \"chart-analysis-\" + str(uuid.uuid4())\n", + "\n", + "# Create a custom analyzer using object model\n", + "print(f\"πŸ”§ Creating custom analyzer '{chart_analyzer_id}'...\")\n", + "\n", + "chart_content_analyzer = ContentAnalyzer(\n", + " base_analyzer_id=\"prebuilt-imageAnalyzer\",\n", + " description=\"Extract detailed structured information from charts and diagrams.\",\n", + " config=ContentAnalyzerConfig(\n", + " return_details=False,\n", + " ),\n", + " field_schema=FieldSchema(\n", + " name=\"ChartAndDiagram\",\n", + " description=\"Structured information from charts and diagrams.\",\n", + " fields={\n", + " \"Title\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Verbatim title of the chart.\"\n", + " ),\n", + " \"ChartType\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.CLASSIFY,\n", + " description=\"The type of chart.\",\n", + " enum=[\n", + " \"area\",\n", + " \"bar\",\n", + " \"box\",\n", + " \"bubble\",\n", + " \"candlestick\",\n", + " \"funnel\",\n", + " \"heatmap\",\n", + " \"histogram\",\n", + " \"line\",\n", + " \"pie\",\n", + " \"radar\",\n", + " \"rings\",\n", + " \"rose\",\n", + " \"treemap\"\n", + " ],\n", + " enum_descriptions={\n", + " \"histogram\": \"Continuous values on the x-axis, which distinguishes it from bar.\",\n", + " \"rose\": \"In contrast to pie charts, the sectors are of equal angles and differ in how far each sector extends from the center of the circle.\"\n", + " },\n", + " ),\n", + " \"TopicKeywords\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Relevant topics associated with the chart, used for tagging.\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " examples=[\n", + " \"Business and finance\",\n", + " \"Arts and culture\",\n", + " \"Education and academics\"\n", + " ]\n", + " )\n", + " ),\n", + " \"DetailedDescription\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Detailed description of the chart or diagram, not leaving out any key information. Include numbers, trends, and other details.\"\n", + " ),\n", + " \"Summary\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Detailed summary of the chart, including highlights and takeaways.\"\n", + " ),\n", + " \"MarkdownDataTable\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Underlying data of the chart in tabular markdown format. Give markdown output with valid syntax and accurate numbers, and fill any uncertain values with empty cells. If not applicable, output an empty string.\"\n", + " ),\n", + " \"AxisTitles\": FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " method=GenerationMethod.GENERATE,\n", + " properties={\n", + " \"xAxisTitle\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Title of the x-axis.\"\n", + " ),\n", + " \"yAxisTitle\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Title of the y-axis.\"\n", + " )\n", + " }\n", + " ),\n", + " \"FootnotesAndAnnotations\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"All footnotes and textual annotations in the chart or diagram.\"\n", + " )\n", + " },\n", + " ),\n", + ")\n", + "\n", + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=chart_analyzer_id,\n", + " resource=chart_content_analyzer,\n", + ")\n", + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{chart_analyzer_id}' created successfully!\")" ] }, { @@ -962,16 +1824,60 @@ "outputs": [], "source": [ "sample_file_path = '../data/pieChart.jpg'\n", - "chart_analyzer_id = \"chart-analysis-\" + str(uuid.uuid4())\n", - "\n", - "print(f\"Creating chart analyzer: {chart_analyzer_id}\")\n", - "response = client.begin_create_analyzer(chart_analyzer_id, analyzer_template_path=analyzer_template_path)\n", - "result = client.poll_result(response)\n", - "print(\"βœ… Chart analyzer created successfully!\")\n", - "\n", - "print(f\"Analyzing chart: {sample_file_path}\")\n", - "response = client.begin_analyze(chart_analyzer_id, file_location=sample_file_path)\n", - "result_json = client.poll_result(response)" + "print(f\"πŸ“„ Reading document file: {sample_file_path}\")\n", + "with open(sample_file_path, \"rb\") as f:\n", + " chart_content = f.read()\n", + "\n", + "# Check if this is a Git LFS pointer file\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=chart_analyzer_id,\n", + " input=chart_content,\n", + " content_type=\"application/octet-stream\",\n", + ")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for document analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Document analysis completed successfully!\")\n", + "\n", + "# Extract operation ID for get_result\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the analysis result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "print(f\"βœ… Analysis result retrieved successfully!\")\n", + "print(f\" Operation ID: {operation_status.id}\")\n", + "print(f\" Status: {operation_status.status}\")\n", + "\n", + "# The actual analysis result is in operation_status.result\n", + "operation_result = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + "\n", + "print(f\" Result contains {len(operation_result.contents)} contents\")\n", + "\n", + "# Save the analysis result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"content_analyzers_get_result\",\n", + ")\n", + "print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")\n", + "\n", + "# Save the analysis result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"content_analyzers_get_result\",\n", + ")\n", + "print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")" ] }, { @@ -987,7 +1893,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(json.dumps(result_json, indent=2))" + "print(json.dumps(operation_result.as_dict(), indent=2))" ] }, { @@ -1005,7 +1911,9 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(chart_analyzer_id)" + "print(f\"πŸ—‘οΈ Deleting analyzer '{chart_analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=chart_analyzer_id)\n", + "print(f\"βœ… Analyzer '{chart_analyzer_id}' deleted successfully!\")" ] }, { @@ -1039,7 +1947,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "py312", "language": "python", "name": "python3" }, @@ -1053,7 +1961,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.13" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/notebooks/field_extraction_pro_mode.ipynb b/notebooks/field_extraction_pro_mode.ipynb index d6c65ba..bb4c98a 100644 --- a/notebooks/field_extraction_pro_mode.ipynb +++ b/notebooks/field_extraction_pro_mode.ipynb @@ -45,57 +45,6 @@ "%pip install -r ../requirements.txt" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Analyzer Template and Local Files Setup\n", - "- **analyzer_template**: In this sample, we define an analyzer template for invoice-contract verification.\n", - "- **input_docs**: You can have multiple input document files in one folder or specify a single document file location.\n", - "- **reference_docs (Optional)**: During analyzer creation, you can provide documents that aid in providing contextual information for the analyzer to reference during inference. OCR results will be extracted from these files if needed, a reference JSONL file will be generated, and these files will be uploaded to a designated Azure Blob Storage container.\n", - "\n", - "> For example, if you're analyzing invoices to ensure their consistency with a contractual agreement, you can supply the invoice and other relevant documents (e.g., a purchase order) as inputs, and provide the contract files as reference data. The service applies reasoning to validate the input documents against your schema, which might include identifying discrepancies to flag for further review." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Define paths for analyzer template, input documents, and reference documents\n", - "analyzer_template = \"../analyzer_templates/invoice_contract_verification_pro_mode.json\"\n", - "input_docs = \"../data/field_extraction_pro_mode/invoice_contract_verification/input_docs\"\n", - "\n", - "# NOTE: Reference documents are optional in Pro mode. Comment out the line below if not using reference documents.\n", - "reference_docs = \"../data/field_extraction_pro_mode/invoice_contract_verification/reference_docs\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> Let's examine the Pro mode analyzer template." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "with open(analyzer_template, \"r\") as file:\n", - " print(json.dumps(json.load(file), indent=2))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> In the analyzer, the field `\"mode\"` must be set to `\"pro\"`. The defined field `\"PaymentTermsInconsistencies\"` is a `\"generate\"` field designed to reason about inconsistencies. It can utilize the referenced documents uploaded in [reference docs](../data/field_extraction_pro_mode/invoice_contract_verification/reference_docs)." - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -118,38 +67,59 @@ "outputs": [], "source": [ "import logging\n", + "import json\n", "import os\n", - "import sys\n", "from pathlib import Path\n", - "from dotenv import find_dotenv, load_dotenv\n", - "from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n", + "import sys\n", + "from dotenv import load_dotenv\n", + "import base64\n", + "from azure.storage.blob import ContainerSasPermissions\n", + "from azure.core.credentials import AzureKeyCredential\n", + "from azure.identity import DefaultAzureCredential\n", + "from azure.ai.contentunderstanding.aio import ContentUnderstandingClient\n", + "from azure.ai.contentunderstanding.models import (\n", + " AnalyzeResult,\n", + " AnalyzeInput,\n", + " ContentAnalyzer,\n", + " ContentAnalyzerConfig,\n", + " AnalysisMode,\n", + " ProcessingLocation,\n", + " AudioVisualContent,\n", + " FieldSchema,\n", + " FieldDefinition,\n", + " FieldType,\n", + " GenerationMethod,\n", + ")\n", + "from datetime import datetime\n", + "from typing import Any\n", + "import uuid\n", "\n", - "# Import utility package from python samples root directory\n", - "parent_dir = Path(Path.cwd()).parent\n", - "sys.path.append(str(parent_dir))\n", - "from python.content_understanding_client import AzureContentUnderstandingClient\n", + "# Add the parent directory to the Python path to import the sample_helper module\n", + "sys.path.append(os.path.join(os.path.dirname(os.getcwd()), 'python'))\n", + "from extension.document_processor import DocumentProcessor\n", + "from extension.sample_helper import (\n", + " extract_operation_id_from_poller,\n", + " PollerType,\n", + " save_json_to_file,\n", + ")\n", "\n", - "load_dotenv(find_dotenv())\n", + "load_dotenv()\n", "logging.basicConfig(level=logging.INFO)\n", "\n", - "# For authentication, you can use either token-based auth or subscription key; only one is required\n", - "AZURE_AI_ENDPOINT = os.getenv(\"AZURE_AI_ENDPOINT\")\n", - "# IMPORTANT: Replace with your actual subscription key or set it in the \".env\" file if not using token auth\n", - "AZURE_AI_API_KEY = os.getenv(\"AZURE_AI_API_KEY\")\n", - "AZURE_AI_API_VERSION = os.getenv(\"AZURE_AI_API_VERSION\", \"2025-05-01-preview\")\n", - "\n", - "credential = DefaultAzureCredential()\n", - "token_provider = get_bearer_token_provider(credential, \"https://cognitiveservices.azure.com/.default\")\n", + "endpoint = os.environ.get(\"AZURE_CONTENT_UNDERSTANDING_ENDPOINT\")\n", + "# Return AzureKeyCredential if AZURE_CONTENT_UNDERSTANDING_KEY is set, otherwise DefaultAzureCredential\n", + "key = os.getenv(\"AZURE_CONTENT_UNDERSTANDING_KEY\")\n", + "credential = AzureKeyCredential(key) if key else DefaultAzureCredential()\n", + "# Create the ContentUnderstandingClient\n", + "client = ContentUnderstandingClient(endpoint=endpoint, credential=credential)\n", + "print(\"βœ… ContentUnderstandingClient created successfully\")\n", "\n", - "client = AzureContentUnderstandingClient(\n", - " endpoint=AZURE_AI_ENDPOINT,\n", - " api_version=AZURE_AI_API_VERSION,\n", - " # IMPORTANT: Comment out token_provider if using subscription key\n", - " token_provider=token_provider,\n", - " # IMPORTANT: Uncomment this line if using subscription key\n", - " # subscription_key=AZURE_AI_API_KEY,\n", - " x_ms_useragent=\"azure-ai-content-understanding-python/pro_mode\", # This header is used for sample usage telemetry; please comment out this line if you want to opt out.\n", - ")" + "try:\n", + " processor = DocumentProcessor(client)\n", + " print(\"βœ… DocumentProcessor created successfully\")\n", + "except Exception as e:\n", + " print(f\"❌ Failed to create DocumentProcessor: {e}\")\n", + " raise" ] }, { @@ -173,18 +143,23 @@ "outputs": [], "source": [ "# Load reference storage configuration from environment\n", - "reference_doc_path = os.getenv(\"REFERENCE_DOC_PATH\")\n", - "\n", + "reference_doc_path = os.getenv(\"REFERENCE_DOC_PATH\") or f\"reference_docs_{uuid.uuid4().hex[:8]}\"\n", "reference_doc_sas_url = os.getenv(\"REFERENCE_DOC_SAS_URL\")\n", + "\n", + "if not reference_doc_path.endswith(\"/\"):\n", + " reference_doc_path += \"/\"\n", + "\n", "if not reference_doc_sas_url:\n", - " REFERENCE_DOC_STORAGE_ACCOUNT_NAME = os.getenv(\"REFERENCE_DOC_STORAGE_ACCOUNT_NAME\")\n", - " REFERENCE_DOC_CONTAINER_NAME = os.getenv(\"REFERENCE_DOC_CONTAINER_NAME\")\n", - " if REFERENCE_DOC_STORAGE_ACCOUNT_NAME and REFERENCE_DOC_CONTAINER_NAME:\n", - " from azure.storage.blob import ContainerSasPermissions\n", + " reference_doc_storage_account_name = os.getenv(\"REFERENCE_DOC_STORAGE_ACCOUNT_NAME\")\n", + " reference_doc_container_name = os.getenv(\"REFERENCE_DOC_CONTAINER_NAME\")\n", + " print(f\"REFERENCE_DOC_STORAGE_ACCOUNT_NAME: {reference_doc_storage_account_name}\")\n", + " print(f\"REFERENCE_DOC_CONTAINER_NAME: {reference_doc_container_name}\")\n", + "\n", + " if reference_doc_storage_account_name and reference_doc_container_name:\n", " # We require \"Write\" permission to upload, modify, or append blobs\n", - " reference_doc_sas_url = AzureContentUnderstandingClient.generate_temp_container_sas_url(\n", - " account_name=REFERENCE_DOC_STORAGE_ACCOUNT_NAME,\n", - " container_name=REFERENCE_DOC_CONTAINER_NAME,\n", + " reference_doc_sas_url = processor.generate_container_sas_url(\n", + " account_name=reference_doc_storage_account_name,\n", + " container_name=reference_doc_container_name,\n", " permissions=ContainerSasPermissions(read=True, write=True, list=True),\n", " expiry_hours=1,\n", " )" @@ -207,7 +182,11 @@ "# Please name the OCR result files with the same name as the original document filenames including extension, and add the suffix \".result.json\"\n", "# For example, if the original document is \"invoice.pdf\", the OCR result file should be named \"invoice.pdf.result.json\"\n", "# NOTE: Please comment out the following line if you do not have any reference documents.\n", - "await client.generate_knowledge_base_on_blob(reference_docs, reference_doc_sas_url, reference_doc_path, skip_analyze=False)" + "reference_docs = \"../data/field_extraction_pro_mode/invoice_contract_verification/reference_docs\"\n", + "print(f\"REFERENCE_DOCS: {reference_docs}\")\n", + "print(f\"REFERENCE_DOC_SAS_URL: {reference_doc_sas_url}\")\n", + "print(f\"REFERENCE_DOC_PATH: {reference_doc_path}\")\n", + "await processor.generate_knowledge_base_on_blob(reference_docs, reference_doc_sas_url, reference_doc_path, skip_analyze=False)" ] }, { @@ -226,24 +205,153 @@ "metadata": {}, "outputs": [], "source": [ - "import uuid\n", - "CUSTOM_ANALYZER_ID = \"pro-mode-sample-\" + str(uuid.uuid4())\n", + "analyzer_id = f\"pro-mode-sample-{datetime.now().strftime('%Y%m%d')}-{datetime.now().strftime('%H%M%S')}-{uuid.uuid4().hex[:8]}\"\n", "\n", - "response = client.begin_create_analyzer(\n", - " CUSTOM_ANALYZER_ID,\n", - " analyzer_template_path=analyzer_template,\n", - " pro_mode_reference_docs_storage_container_sas_url=reference_doc_sas_url,\n", - " pro_mode_reference_docs_storage_container_path_prefix=reference_doc_path,\n", + "# Create a custom analyzer using object model\n", + "content_analyzer = ContentAnalyzer(\n", + " base_analyzer_id=\"prebuilt-documentAnalyzer\",\n", + " field_schema=FieldSchema(\n", + " name=\"InvoiceContractVerification\",\n", + " description=\"Analyze invoice to confirm total consistency with signed contract.\",\n", + " fields={\n", + " \"PaymentTermsInconsistencies\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"List all areas of inconsistency identified in the invoice with corresponding evidence.\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Area of inconsistency in the invoice with the company's contracts.\",\n", + " properties={\n", + " \"Evidence\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Evidence or reasoning for the inconsistency in the invoice.\"\n", + " ),\n", + " \"InvoiceField\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Invoice field or the aspect that is inconsistent with the contract.\"\n", + " )\n", + " }\n", + " )\n", + " ),\n", + " \"ItemInconsistencies\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"List all areas of inconsistency identified in the invoice in the goods or services sold (including detailed specifications for every line item).\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Area of inconsistency in the invoice with the company's contracts.\",\n", + " properties={\n", + " \"Evidence\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Evidence or reasoning for the inconsistency in the invoice.\"\n", + " ),\n", + " \"InvoiceField\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Invoice field or the aspect that is inconsistent with the contract.\"\n", + " )\n", + " }\n", + " )\n", + " ),\n", + " \"BillingLogisticsInconsistencies\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"List all areas of inconsistency identified in the invoice regarding billing logistics and administrative or legal issues.\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Area of inconsistency in the invoice with the company's contracts.\",\n", + " properties={\n", + " \"Evidence\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Evidence or reasoning for the inconsistency in the invoice.\"\n", + " ),\n", + " \"InvoiceField\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Invoice field or the aspect that is inconsistent with the contract.\"\n", + " )\n", + " }\n", + " )\n", + " ),\n", + " \"PaymentScheduleInconsistencies\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"List all areas of inconsistency identified in the invoice with corresponding evidence.\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Area of inconsistency in the invoice with the company's contracts.\",\n", + " properties={\n", + " \"Evidence\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Evidence or reasoning for the inconsistency in the invoice.\"\n", + " ),\n", + " \"InvoiceField\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Invoice field or the aspect that is inconsistent with the contract.\"\n", + " )\n", + " }\n", + " )\n", + " ),\n", + " \"TaxOrDiscountInconsistencies\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"List all areas of inconsistency identified in the invoice with corresponding evidence regarding taxes or discounts.\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Area of inconsistency in the invoice with the company's contracts.\",\n", + " properties={\n", + " \"Evidence\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Evidence or reasoning for the inconsistency in the invoice.\"\n", + " ),\n", + " \"InvoiceField\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Invoice field or the aspect that is inconsistent with the contract.\"\n", + " )\n", + " }\n", + " )\n", + " )\n", + " }\n", + " ),\n", + " mode=AnalysisMode.PRO,\n", + " processing_location=ProcessingLocation.GLOBAL,\n", + " knowledge_sources=[{\n", + " \"kind\": \"reference\",\n", + " \"containerUrl\": reference_doc_sas_url,\n", + " \"prefix\": reference_doc_path,\n", + " \"fileListPath\": processor.KNOWLEDGE_SOURCE_LIST_FILE_NAME,\n", + " }],\n", + ")\n", + "print(\"KNOWLEDGE_SOURCE_LIST_FILE_NAME\", processor.KNOWLEDGE_SOURCE_LIST_FILE_NAME)\n", + "print(f\"πŸ”§ Creating custom analyzer '{analyzer_id}'...\")\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=analyzer_id,\n", + " resource=content_analyzer,\n", ")\n", - "result = client.poll_result(response)\n", - "if result is not None and \"status\" in result and result[\"status\"] == \"Succeeded\":\n", - " logging.info(f\"Analyzer details for {result['result']['analyzerId']}\")\n", - " logging.info(json.dumps(result, indent=2))\n", - "else:\n", - " logging.warning(\n", - " \"An issue was encountered when trying to create the analyzer. \"\n", - " \"Please double-check your deployment and configurations for potential problems.\"\n", - " )" + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{analyzer_id}' created successfully!\")" ] }, { @@ -261,21 +369,51 @@ "metadata": {}, "outputs": [], "source": [ - "from IPython.display import FileLink, display\n", + "input_docs = \"../data/field_extraction_pro_mode/invoice_contract_verification/input_docs/contoso_lifts_invoice.pdf\"\n", "\n", - "response = client.begin_analyze(CUSTOM_ANALYZER_ID, file_location=input_docs)\n", - "result_json = client.poll_result(response, timeout_seconds=600) # Use a longer timeout for Pro mode\n", + "print(f\"πŸ“„ Reading document file: {input_docs}\")\n", + "with open(input_docs, \"rb\") as f:\n", + " pdf_content = f.read()\n", + "\n", + "print(f\"πŸ” Starting document analysis with analyzer '{analyzer_id}'...\")\n", + "analysis_poller = await client.content_analyzers.begin_analyze_binary(\n", + " analyzer_id=analyzer_id,\n", + " input=pdf_content,\n", + " content_type=\"application/pdf\"\n", + ")\n", + "\n", + "# Wait for analysis completion\n", + "print(f\"⏳ Waiting for document analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Document analysis completed successfully!\")\n", + "\n", + "# Extract operation ID for get_result\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the analysis result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "# The actual analysis result is in operation_status.result\n", + "operation_result = operation_status.result\n", "\n", "# Create output directory if it doesn't exist\n", "output_dir = \"output\"\n", "os.makedirs(output_dir, exist_ok=True)\n", "\n", - "output_path = os.path.join(output_dir, f\"{CUSTOM_ANALYZER_ID}_result.json\")\n", - "with open(output_path, \"w\", encoding=\"utf-8\") as file:\n", - " json.dump(result_json, file, indent=2)\n", + "saved_file_path = os.path.join(output_dir, f\"{analyzer_id}_result.json\")\n", + "with open(saved_file_path, \"w\", encoding=\"utf-8\") as file:\n", + " json.dump(operation_result.as_dict(), file, indent=2)\n", "\n", - "logging.info(f\"Full analyzer result saved to: {output_path}\")\n", - "display(FileLink(output_path))" + "logging.info(f\"Full analyzer result saved to: {saved_file_path}\")\n", + "print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")" ] }, { @@ -291,8 +429,9 @@ "metadata": {}, "outputs": [], "source": [ - "fields = result_json[\"result\"][\"contents\"][0][\"fields\"]\n", - "print(json.dumps(fields, indent=2))" + "print(json.dumps(operation_result.as_dict(), indent=2))\n", + "fields = operation_result.contents[0].fields\n", + "print(fields)" ] }, { @@ -316,7 +455,7 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(CUSTOM_ANALYZER_ID)" + "await client.content_analyzers.delete(analyzer_id)" ] }, { @@ -343,11 +482,11 @@ "# Define paths for analyzer template, input documents, and reference documents of the second sample\n", "analyzer_template_2 = \"../analyzer_templates/insurance_claims_review_pro_mode.json\"\n", "input_docs_2 = \"../data/field_extraction_pro_mode/insurance_claims_review/input_docs\"\n", - "reference_docs_2 = \"../data/field_extraction_pro_mode/insurance_claims_review/reference_docs\"\n", + "reference_docs_2 = \"../data/field_extraction_pro_mode/insurance_claims_review/reference_docs2\"\n", "\n", "# Load reference storage configuration from environment\n", - "reference_doc_path_2 = os.getenv(\"REFERENCE_DOC_PATH\").rstrip(\"/\") + \"_2/\" # NOTE: Use a different path for the second sample\n", - "CUSTOM_ANALYZER_ID_2 = \"pro-mode-sample-\" + str(uuid.uuid4())" + "reference_doc_path_2 = reference_doc_path.rstrip(\"/\") + \"_2/\" # NOTE: Use a different path for the second sample\n", + "analyzer_id_2 = \"pro-mode-sample-\" + str(uuid.uuid4())" ] }, { @@ -366,7 +505,7 @@ "source": [ "logging.info(\"Start generating knowledge base for the second sample...\")\n", "# Reuse the same blob container\n", - "await client.generate_knowledge_base_on_blob(reference_docs_2, reference_doc_sas_url, reference_doc_path_2, skip_analyze=True)" + "await processor.generate_knowledge_base_on_blob(reference_docs_2, reference_doc_sas_url, reference_doc_path_2, skip_analyze=True)" ] }, { @@ -383,21 +522,152 @@ "metadata": {}, "outputs": [], "source": [ - "response = client.begin_create_analyzer(\n", - " CUSTOM_ANALYZER_ID_2,\n", - " analyzer_template_path=analyzer_template_2,\n", - " pro_mode_reference_docs_storage_container_sas_url=reference_doc_sas_url,\n", - " pro_mode_reference_docs_storage_container_path_prefix=reference_doc_path_2,\n", + "analyzer_id = f\"pro-mode-sample-{datetime.now().strftime('%Y%m%d')}-{datetime.now().strftime('%H%M%S')}-{uuid.uuid4().hex[:8]}\"\n", + "# Create a custom analyzer using object model\n", + "print(f\"πŸ”§ Creating custom analyzer '{analyzer_id}' for bonus...\")\n", + "\n", + "bonus_content_analyzer = ContentAnalyzer(\n", + " base_analyzer_id=\"prebuilt-documentAnalyzer\",\n", + " description=\"Bonus content analyzer for cleanup demonstration\",\n", + " field_schema=FieldSchema(\n", + " name=\"InsuranceClaimsReview\",\n", + " description=\"Analyze documents for insurance claim approval strictly according to the provided insurance policy. Consider all aspects of the insurance claim documents, any potential discrepancies found among the documents, any claims that should be flagged for review, etc.\",\n", + " fields={\n", + " \"CarBrand\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Brand of the damaged vehicle.\",\n", + " ),\n", + " \"CarColor\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Color of the damaged vehicle. Only use color name from 17 web colors. Use CamalCase naming convention.\",\n", + " ),\n", + " \"CarModel\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Model of the damaged vehicle. Do not include brand name. Leave empty if not found.\",\n", + " ),\n", + " \"LicensePlate\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"License plate number of the damaged vehicle.\",\n", + " ),\n", + " \"VIN\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"VIN of the damaged vehicle. Leave empty if not found.\",\n", + " ),\n", + " \"ReportingOfficer\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Name of the reporting officer for the incident.\",\n", + " ),\n", + " \"LineItemCorroboration\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " description=\"Validation of all of the line items on the claim, including parts, services, labors, materials, shipping and other costs and fees. When in doubt about adherence to the policy, mark as suspicious.\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " description=\"Entry in the line item analysis table to analyze the pertinent information for the line item.\",\n", + " properties={\n", + " \"LineItemName\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Name of the line item in the claim.\",\n", + " ),\n", + " \"IdentifiedVehiclePart\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"The relevant associated vehicle part for this line item\",\n", + " enum=[\n", + " \"BODY_TRIM\",\n", + " \"DRIVER_SIDE_DRIVER_DOOR\",\n", + " \"DRIVER_SIDE_DRIVER_HANDLE\",\n", + " \"DRIVER_SIDE_FRONT_TIRE\",\n", + " \"DRIVER_SIDE_FRONT_WHEEL\",\n", + " \"DRIVER_SIDE_FUEL_CAP\",\n", + " \"DRIVER_SIDE_PASSENGER_DOOR\",\n", + " \"DRIVER_SIDE_PASSENGER_HANDLE\",\n", + " \"DRIVER_SIDE_PASSENGER_WINDOW\",\n", + " \"DRIVER_SIDE_REAR_HEADLAMP\",\n", + " \"DRIVER_SIDE_REAR_TIRE\",\n", + " \"DRIVER_SIDE_REAR_WHEEL\",\n", + " \"DRIVER_SIDE_SIDE_WINDOW\",\n", + " \"DRIVER_SIDE_WINDOW\",\n", + " \"DRIVER_SIDE_WING_MIRROR\",\n", + " \"FRONT_BONNET\",\n", + " \"FRONT_BUMPER_LOWER\",\n", + " \"FRONT_BUMPER_UPPER\",\n", + " \"FRONT_DRIVER_SIDE_FOG_LIGHT\",\n", + " \"FRONT_DRIVER_SIDE_HEADLAMP\",\n", + " \"FRONT_GRILL\",\n", + " \"FRONT_NUMBER_PLATE\",\n", + " \"FRONT_PASSENGER_SIDE_FOG_LIGHT\",\n", + " \"FRONT_PASSENGER_SIDE_HEADLAMP\",\n", + " \"FRONT_WINDSHIELD\",\n", + " \"PASSENGER_SIDE_DRIVER_DOOR\",\n", + " \"PASSENGER_SIDE_DRIVER_HANDLE\",\n", + " \"PASSENGER_SIDE_FRONT_TIRE\",\n", + " \"PASSENGER_SIDE_FRONT_WHEEL\",\n", + " \"PASSENGER_SIDE_PASSENGER_DOOR\",\n", + " \"PASSENGER_SIDE_PASSENGER_HANDLE\",\n", + " \"PASSENGER_SIDE_PASSENGER_WINDOW\",\n", + " \"PASSENGER_SIDE_REAR_HEADLAMP\",\n", + " \"PASSENGER_SIDE_REAR_TIRE\",\n", + " \"PASSENGER_SIDE_REAR_WHEEL\",\n", + " \"PASSENGER_SIDE_SIDE_WINDOW\",\n", + " \"PASSENGER_SIDE_WINDOW\",\n", + " \"PASSENGER_SIDE_WING_MIRROR\",\n", + " \"REAR_BUMPER\",\n", + " \"REAR_NUMBER_PLATE\",\n", + " \"REAR_TRUNK\",\n", + " \"REAR_WINDSHIELD\",\n", + " \"ROOF_PANEL\",\n", + " \"OTHER\"\n", + " ]\n", + " ),\n", + " \"Cost\": FieldDefinition(\n", + " type=FieldType.NUMBER,\n", + " description=\"The cost of this line item on the claim.\",\n", + " ),\n", + " \"Evidence\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " description=\"The evidence for this line item entry, a list of the document with analyzed evidence supporting the claim formatted as /. One of the insurance policy documents must be one of the documents.\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.STRING,\n", + " )\n", + " ),\n", + " \"ClaimStatus\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Determined by confidence in whether the claim should be approved based on the evidence. Item should be compliant to insurance policy and required for repairing the vehicle. Only use 'confirmed' for items explicitly approvable according to the policy. If unsure, use 'suspicious'.\",\n", + " enum=[\n", + " \"confirmed\",\n", + " \"suspicious\",\n", + " \"unconfirmed\"\n", + " ],\n", + " enum_descriptions={\n", + " \"confirmed\": \"Completely and explicitly corroborated by the policy.\",\n", + " \"suspicious\": \"Only partially verified, questionable, or otherwise uncertain evidence to approve automatically. Requires human review.\",\n", + " \"unconfirmed\": \"Explicitly not approved by the policy.\"\n", + " }\n", + " )\n", + " },\n", + " )\n", + " )\n", + " }\n", + " ),\n", + " mode=AnalysisMode.PRO,\n", + " processing_location=ProcessingLocation.GLOBAL,\n", ")\n", - "result = client.poll_result(response)\n", - "if result is not None and \"status\" in result and result[\"status\"] == \"Succeeded\":\n", - " logging.info(f\"Analyzer details for {result['result']['analyzerId']}\")\n", - " logging.info(json.dumps(result, indent=2))\n", - "else:\n", - " logging.warning(\n", - " \"An issue was encountered when trying to create the analyzer. \"\n", - " \"Please double-check your deployment and configurations for potential problems.\"\n", - " )" + "\n", + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=analyzer_id,\n", + " resource=bonus_content_analyzer,\n", + ")\n", + "\n", + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{analyzer_id}' created successfully!\")" ] }, { @@ -419,20 +689,65 @@ "metadata": {}, "outputs": [], "source": [ - "logging.info(\"Start analyzing input documents for the second sample...\")\n", - "response = client.begin_analyze(CUSTOM_ANALYZER_ID_2, file_location=input_docs_2)\n", - "result_json = client.poll_result(response, timeout_seconds=600) # Use a longer timeout for Pro mode\n", + "inputs_data: list[AnalyzeInput] = []\n", + "input_dir = Path(input_docs_2)\n", "\n", - "# Save the result to a JSON file\n", - "# Create output directory if it doesn't exist\n", - "output_dir = \"output\"\n", - "os.makedirs(output_dir, exist_ok=True)\n", - "output_path = os.path.join(output_dir, f\"{CUSTOM_ANALYZER_ID_2}_result.json\")\n", - "with open(output_path, \"w\", encoding=\"utf-8\") as file:\n", - " json.dump(result_json, file, indent=2)\n", + "for file_path in input_dir.glob(\"*\"):\n", + " if file_path.is_file() and processor.is_supported_doc_type_by_file_path(file_path, is_document=True):\n", + " # Get relative path and replace separators with underscores\n", + " relative_path = file_path.relative_to(input_dir)\n", + " name = str(relative_path).replace(os.sep, '_').replace('/', '_').replace('\\\\', '_')\n", + "\n", + " with open(file_path, 'rb') as f:\n", + " file_data = f.read()\n", + " base64_data = base64.b64encode(file_data).decode('utf-8')\n", + " \n", + " inputs_data.append({\n", + " 'name': name,\n", + " 'data': base64_data\n", + " })\n", + "\n", + "analysis_poller = await client.content_analyzers.begin_analyze(\n", + " analyzer_id=analyzer_id, \n", + " inputs=inputs_data, \n", + " content_type=\"application/json\")\n", "\n", - "logging.info(f\"Full analyzer result saved to: {output_path}\")\n", - "display(FileLink(output_path))" + " # Wait for analysis completion\n", + "print(f\"⏳ Waiting for document analysis to complete...\")\n", + "analysis_result = await analysis_poller.result()\n", + "print(f\"βœ… Document analysis completed successfully!\")\n", + "\n", + "# Extract operation ID for get_result\n", + "analysis_operation_id = extract_operation_id_from_poller(\n", + " analysis_poller, PollerType.ANALYZE_CALL\n", + ")\n", + "print(f\"πŸ“‹ Extracted analysis operation ID: {analysis_operation_id}\")\n", + "\n", + "# Get the analysis result using the operation ID\n", + "print(\n", + " f\"πŸ” Getting analysis result using operation ID '{analysis_operation_id}'...\"\n", + ")\n", + "operation_status = await client.content_analyzers.get_result(\n", + " operation_id=analysis_operation_id,\n", + ")\n", + "\n", + "print(f\"βœ… Analysis result retrieved successfully!\")\n", + "print(f\" Operation ID: {operation_status.id}\")\n", + "print(f\" Status: {operation_status.status}\")\n", + "\n", + "# The actual analysis result is in operation_status.result\n", + "operation_result = operation_status.result\n", + "if operation_result is None:\n", + " print(\"⚠️ No analysis result available\")\n", + "\n", + "print(f\" Result contains {len(operation_result.contents)} contents\")\n", + "\n", + "# Save the analysis result to a file\n", + "saved_file_path = save_json_to_file(\n", + " result=operation_result.as_dict(),\n", + " filename_prefix=\"bonus_content_analyzer_get_result\",\n", + ")\n", + "print(f\"πŸ’Ύ Analysis result saved to: {saved_file_path}\")" ] }, { @@ -448,7 +763,7 @@ "metadata": {}, "outputs": [], "source": [ - "result_json[\"result\"][\"contents\"][0][\"fields\"]" + "print(json.dumps(operation_result[\"contents\"][0].as_dict()[\"fields\"], indent=2))" ] }, { @@ -473,8 +788,8 @@ "metadata": {}, "outputs": [], "source": [ - "fields = result_json[\"result\"][\"contents\"][0][\"fields\"][\"LineItemCorroboration\"]\n", - "print(json.dumps(fields, indent=2))" + "fields = operation_result[\"contents\"][0][\"fields\"][\"LineItemCorroboration\"]\n", + "print(json.dumps(fields.as_dict(), indent=2))" ] }, { @@ -497,13 +812,13 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(CUSTOM_ANALYZER_ID_2)" + "await client.content_analyzers.delete(analyzer_id_2)" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "py312", "language": "python", "name": "python3" }, @@ -517,7 +832,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.12" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/notebooks/management.ipynb b/notebooks/management.ipynb index 1ab7f3c..fbce209 100644 --- a/notebooks/management.ipynb +++ b/notebooks/management.ipynb @@ -60,36 +60,45 @@ "import json\n", "import os\n", "import sys\n", - "from pathlib import Path\n", - "from dotenv import find_dotenv, load_dotenv\n", - "from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n", + "import uuid\n", + "from dotenv import load_dotenv\n", + "from azure.storage.blob import ContainerSasPermissions\n", + "from azure.core.credentials import AzureKeyCredential\n", + "from azure.identity import DefaultAzureCredential\n", + "from azure.ai.contentunderstanding.aio import ContentUnderstandingClient\n", + "from azure.ai.contentunderstanding.models import (\n", + " ContentAnalyzer,\n", + " ContentAnalyzerConfig,\n", + " FieldSchema,\n", + " FieldDefinition,\n", + " FieldType,\n", + " GenerationMethod,\n", + " AnalysisMode,\n", + " ProcessingLocation,\n", + ")\n", + "\n", + "# Add the parent directory to the Python path to import the sample_helper module\n", + "sys.path.append(os.path.join(os.path.dirname(os.getcwd()), 'python'))\n", + "from extension.document_processor import DocumentProcessor\n", + "from extension.sample_helper import extract_operation_id_from_poller, PollerType, save_json_to_file\n", "\n", - "load_dotenv(find_dotenv())\n", + "load_dotenv()\n", "logging.basicConfig(level=logging.INFO)\n", "\n", - "# For authentication, you can use either token-based auth or subscription key. Use only one of these methods.\n", - "AZURE_AI_ENDPOINT = os.getenv(\"AZURE_AI_ENDPOINT\")\n", - "# IMPORTANT: Replace with your actual subscription key or set it in your \".env\" file if not using token auth\n", - "AZURE_AI_API_KEY = os.getenv(\"AZURE_AI_API_KEY\")\n", - "AZURE_AI_API_VERSION = os.getenv(\"AZURE_AI_API_VERSION\", \"2025-05-01-preview\")\n", - "\n", - "# Add the parent directory to the system path to use shared modules\n", - "parent_dir = Path(Path.cwd()).parent\n", - "sys.path.append(str(parent_dir))\n", - "from python.content_understanding_client import AzureContentUnderstandingClient\n", - "\n", - "credential = DefaultAzureCredential()\n", - "token_provider = get_bearer_token_provider(credential, \"https://cognitiveservices.azure.com/.default\")\n", - "\n", - "client = AzureContentUnderstandingClient(\n", - " endpoint=AZURE_AI_ENDPOINT,\n", - " api_version=AZURE_AI_API_VERSION,\n", - " # IMPORTANT: Comment out token_provider if using subscription key\n", - " token_provider=token_provider,\n", - " # IMPORTANT: Uncomment this line if using subscription key\n", - " # subscription_key=AZURE_AI_API_KEY,\n", - " x_ms_useragent=\"azure-ai-content-understanding-python/analyzer_management\", # This header is used for sample usage telemetry. Please comment out this line if you want to opt out.\n", - ")" + "endpoint = os.environ.get(\"AZURE_CONTENT_UNDERSTANDING_ENDPOINT\")\n", + "# Return AzureKeyCredential if AZURE_CONTENT_UNDERSTANDING_KEY is set, otherwise DefaultAzureCredential\n", + "key = os.getenv(\"AZURE_CONTENT_UNDERSTANDING_KEY\")\n", + "credential = AzureKeyCredential(key) if key else DefaultAzureCredential()\n", + "# Create the ContentUnderstandingClient\n", + "client = ContentUnderstandingClient(endpoint=endpoint, credential=credential)\n", + "print(\"βœ… ContentUnderstandingClient created successfully\")\n", + "\n", + "try:\n", + " processor = DocumentProcessor(client)\n", + " print(\"βœ… DocumentProcessor created successfully\")\n", + "except Exception as e:\n", + " print(f\"❌ Failed to create DocumentProcessor: {e}\")\n", + " raise" ] }, { @@ -106,15 +115,109 @@ "metadata": {}, "outputs": [], "source": [ - "import uuid\n", + "analyzer_id = \"management-sample-\" + str(uuid.uuid4())\n", + "\n", + "# Create a custom analyzer using object model\n", + "print(f\"πŸ”§ Creating custom analyzer '{analyzer_id}'...\")\n", "\n", - "ANALYZER_TEMPLATE = \"../analyzer_templates/call_recording_analytics.json\"\n", - "CUSTOM_ANALYZER_ID = \"analyzer-management-sample-\" + str(uuid.uuid4())\n", + "call_analyzer = ContentAnalyzer(\n", + " base_analyzer_id=\"prebuilt-callCenter\",\n", + " description=\"Sample call recording analytics\",\n", + " config=ContentAnalyzerConfig(\n", + " return_details=True,\n", + " locales=[\"en-US\"]\n", + " ),\n", + " field_schema=FieldSchema(\n", + " fields={\n", + " \"Summary\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"A one-paragraph summary\"\n", + " ),\n", + " \"Topics\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"Top 5 topics mentioned\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.STRING\n", + " )\n", + " ),\n", + " \"Companies\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"List of companies mentioned\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.STRING\n", + " )\n", + " ),\n", + " \"People\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.GENERATE,\n", + " description=\"List of people mentioned\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.OBJECT,\n", + " properties={\n", + " \"Name\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Person's name\"\n", + " ),\n", + " \"Role\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " description=\"Person's title/role\"\n", + " )\n", + " }\n", + " )\n", + " ),\n", + " \"Sentiment\": FieldDefinition(\n", + " type=FieldType.STRING,\n", + " method=GenerationMethod.CLASSIFY,\n", + " description=\"Overall sentiment\",\n", + " enum=[\n", + " \"Positive\",\n", + " \"Neutral\",\n", + " \"Negative\"\n", + " ]\n", + " ),\n", + " \"Categories\": FieldDefinition(\n", + " type=FieldType.ARRAY,\n", + " method=GenerationMethod.CLASSIFY,\n", + " description=\"List of relevant categories\",\n", + " items_property=FieldDefinition(\n", + " type=FieldType.STRING,\n", + " enum=[\n", + " \"Agriculture\",\n", + " \"Business\",\n", + " \"Finance\",\n", + " \"Health\",\n", + " \"Insurance\",\n", + " \"Mining\",\n", + " \"Pharmaceutical\",\n", + " \"Retail\",\n", + " \"Technology\",\n", + " \"Transportation\"\n", + " ]\n", + " )\n", + " )\n", + " }\n", + " )\n", + ")\n", "\n", - "response = client.begin_create_analyzer(CUSTOM_ANALYZER_ID, analyzer_template_path=ANALYZER_TEMPLATE)\n", - "result = client.poll_result(response)\n", + "# Start the analyzer creation operation\n", + "poller = await client.content_analyzers.begin_create_or_replace(\n", + " analyzer_id=analyzer_id,\n", + " resource=call_analyzer,\n", + ")\n", "\n", - "print(json.dumps(result, indent=2))" + "# Extract operation ID from the poller\n", + "operation_id = extract_operation_id_from_poller(\n", + " poller, PollerType.ANALYZER_CREATION\n", + ")\n", + "print(f\"πŸ“‹ Extracted creation operation ID: {operation_id}\")\n", + "\n", + "# Wait for the analyzer to be created\n", + "print(f\"⏳ Waiting for analyzer creation to complete...\")\n", + "await poller.result()\n", + "print(f\"βœ… Analyzer '{analyzer_id}' created successfully!\")" ] }, { @@ -137,10 +240,28 @@ "metadata": {}, "outputs": [], "source": [ - "response = client.get_all_analyzers()\n", - "print(f\"Number of analyzers in your resource: {len(response['value'])}\")\n", - "print(f\"Details of the first 3 analyzers: {json.dumps(response['value'][:3], indent=2)}\")\n", - "print(f\"Details of the last analyzer: {json.dumps(response['value'][-1], indent=2)}\")" + "response = client.content_analyzers.list()\n", + "analyzers = [analyzer async for analyzer in response]\n", + "\n", + "print(f\"βœ… Found {len(analyzers)} analyzers\")\n", + "\n", + "# Display detailed information about each analyzer\n", + "for i, analyzer in enumerate(analyzers, 1):\n", + " print(f\"πŸ” Analyzer {i}:\")\n", + " print(f\" ID: {analyzer.analyzer_id}\")\n", + " print(f\" Description: {analyzer.description}\")\n", + " print(f\" Status: {analyzer.status}\")\n", + " print(f\" Created at: {analyzer.created_at}\")\n", + "\n", + " # Check if it's a prebuilt analyzer\n", + " if analyzer.analyzer_id.startswith(\"prebuilt-\"):\n", + " print(f\" Type: Prebuilt analyzer\")\n", + " else:\n", + " print(f\" Type: Custom analyzer\")\n", + "\n", + " # Show tags if available\n", + " if hasattr(analyzer, \"tags\") and analyzer.tags:\n", + " print(f\" Tags: {analyzer.tags}\")" ] }, { @@ -158,8 +279,11 @@ "metadata": {}, "outputs": [], "source": [ - "result_json = client.get_analyzer_detail_by_id(CUSTOM_ANALYZER_ID)\n", - "print(json.dumps(result_json, indent=2))" + "retrieved_analyzer: ContentAnalyzer = await client.content_analyzers.get(analyzer_id=analyzer_id)\n", + "print(f\"βœ… Analyzer '{analyzer_id}' retrieved successfully!\")\n", + "print(f\" Description: {retrieved_analyzer.description}\")\n", + "print(f\" Status: {retrieved_analyzer.status}\")\n", + "print(f\" Created at: {retrieved_analyzer.created_at}\")" ] }, { @@ -176,13 +300,17 @@ "metadata": {}, "outputs": [], "source": [ - "client.delete_analyzer(CUSTOM_ANALYZER_ID)" + "# Clean up: delete the analyzer (demo purposes only)\n", + "# Note: You can leave the analyzer for later use if desired\n", + "print(f\"πŸ—‘οΈ Deleting analyzer '{analyzer_id}' (demo cleanup)...\")\n", + "await client.content_analyzers.delete(analyzer_id=analyzer_id)\n", + "print(f\"βœ… Analyzer '{analyzer_id}' deleted successfully!\")" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "py312", "language": "python", "name": "python3" }, @@ -196,7 +324,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.12" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/python/extension/document_processor.py b/python/extension/document_processor.py new file mode 100644 index 0000000..35fb1cf --- /dev/null +++ b/python/extension/document_processor.py @@ -0,0 +1,339 @@ +from datetime import datetime, timedelta, timezone +import os +import json +import asyncio +from typing import List, Dict, Any, Optional +from pathlib import Path +from azure.identity import DefaultAzureCredential +from azure.storage.blob.aio import ContainerClient +from azure.ai.contentunderstanding.aio import ContentUnderstandingClient +from azure.storage.blob import ( + BlobServiceClient, + generate_container_sas, + ContainerSasPermissions +) +from dataclasses import dataclass + +@dataclass +class ReferenceDocItem: + file_name: str = "" + file_path: str = "" + result_file_name: str = "" + result_file_path: str = "" + +class DocumentProcessor: + OCR_RESULT_FILE_SUFFIX: str = ".result.json" + LABEL_FILE_SUFFIX: str = ".labels.json" + KNOWLEDGE_SOURCE_LIST_FILE_NAME: str = "sources.jsonl" + SAS_EXPIRY_HOURS: int = 1 + + SUPPORTED_FILE_TYPES_DOCUMENT_TXT: List[str] = [ + ".pdf", ".tiff", ".jpg", ".jpeg", ".png", ".bmp", ".heif", ".docx", + ".xlsx", ".pptx", ".txt", ".html", ".md", ".eml", ".msg", ".xml", + ] + + SUPPORTED_FILE_TYPES_DOCUMENT: List[str] = [ + ".pdf", ".tiff", ".jpg", ".jpeg", ".png", ".bmp", ".heif", + ] + + def __init__(self, client: ContentUnderstandingClient): + self._client = client + + def generate_container_sas_url( + self, + account_name: str, + container_name: str, + permissions: Optional[ContainerSasPermissions] = None, + expiry_hours: Optional[int] = None, + ) -> str: + """Generate a temporary SAS URL for an Azure Blob container using Azure AD authentication.""" + print(f"account_name: {account_name}") + if not all([account_name, container_name]): + raise ValueError("Account name and container name must be provided.") + + permissions = permissions or ContainerSasPermissions(read=True, write=True, list=True) + hours = expiry_hours or self.SAS_EXPIRY_HOURS + + now = datetime.now(timezone.utc) + expiry = now + timedelta(hours=hours) + account_url = f"https://{account_name}.blob.core.windows.net" + client = BlobServiceClient(account_url=account_url, credential=DefaultAzureCredential()) + + delegation_key = client.get_user_delegation_key(now, expiry) + sas_token = generate_container_sas( + account_name=account_name, + container_name=container_name, + user_delegation_key=delegation_key, + permission=permissions, + expiry=expiry, + start=now, + ) + + return f"{account_url}/{container_name}?{sas_token}" + + async def generate_knowledge_base_on_blob( + self, + reference_docs_folder: str, + storage_container_sas_url: str, + storage_container_path_prefix: str, + skip_analyze: bool = False + ): + if not storage_container_path_prefix.endswith("/"): + storage_container_path_prefix += "/" + + try: + resources = [] + container_client = ContainerClient.from_container_url(storage_container_sas_url) + + if not skip_analyze: + analyze_list: List[ReferenceDocItem] = self._get_analyze_list(reference_docs_folder) + + for analyze_item in analyze_list: + try: + print(analyze_item.file_path) + + with open(analyze_item.file_path, "rb") as f: + pdf_bytes: bytes = f.read() + + print(f"πŸ” Analyzing {analyze_item.file_path} with prebuilt-documentAnalyzer...") + poller = await self._client.content_analyzers.begin_analyze_binary( + analyzer_id="prebuilt-documentAnalyzer", + input=pdf_bytes, + content_type="application/pdf", + cls=lambda pipeline_response, deserialized_obj, response_headers: ( + deserialized_obj, + pipeline_response.http_response, + ), + ) + _, raw_http_response = await poller.result() + + print(f"Analysis completed for {raw_http_response}.") + json_string = json.dumps(raw_http_response.json()) + print("json string type:", type(json_string)) + print(f"Analysis result: {json_string}") + + result_file_blob_path = storage_container_path_prefix + analyze_item.result_file_name + file_blob_path = storage_container_path_prefix + analyze_item.file_name + + await self._upload_json_to_blob(container_client, json_string, result_file_blob_path) + await self._upload_file_to_blob(container_client, analyze_item.file_path, file_blob_path) + + resources.append({ + "file": analyze_item.file_name, + "resultFile": analyze_item.result_file_name + }) + except json.JSONDecodeError as json_ex: + raise ValueError( + f"Failed to parse JSON result for file '{analyze_item.file_path}'. " + f"Ensure the file is a valid document and the analyzer is set up correctly." + ) from json_ex + except Exception as ex: + raise ValueError( + f"Failed to analyze file '{analyze_item.file_path}'. " + f"Ensure the file is a valid document and the analyzer is set up correctly." + ) from ex + else: + upload_list: List[ReferenceDocItem] = [] + + # Process subdirectories + for dir_path in Path(reference_docs_folder).rglob("*"): + if dir_path.is_dir(): + self._process_directory(str(dir_path), upload_list) + + # Process root directory + self._process_directory(reference_docs_folder, upload_list) + + for upload_item in upload_list: + result_file_blob_path = storage_container_path_prefix + upload_item.result_file_name + file_blob_path = storage_container_path_prefix + upload_item.file_name + + await self._upload_file_to_blob(container_client, upload_item.result_file_path, result_file_blob_path) + await self._upload_file_to_blob(container_client, upload_item.file_path, file_blob_path) + + resources.append({ + "file": upload_item.file_name, + "resultFile": upload_item.result_file_name + }) + + # Convert resources to JSON strings + jsons = [json.dumps(record) for record in resources] + + await self._upload_jsonl_to_blob(container_client, jsons, storage_container_path_prefix + self.KNOWLEDGE_SOURCE_LIST_FILE_NAME) + finally: + if container_client: + await container_client.close() + + def _process_directory(self, dir_path: str, upload_only_list: List[ReferenceDocItem]): + # Get all files in the directory + try: + file_names = set(os.listdir(dir_path)) + file_paths = [os.path.join(dir_path, f) for f in file_names if os.path.isfile(os.path.join(dir_path, f))] + except OSError: + return + + for file_path in file_paths: + file_name = os.path.basename(file_path) + file_ext = os.path.splitext(file_name)[1] + + if self.is_supported_doc_type_by_file_ext(file_ext, is_document=True): + result_file_name = file_name + self.OCR_RESULT_FILE_SUFFIX + result_file_path = os.path.join(dir_path, result_file_name) + + if not os.path.exists(result_file_path): + raise FileNotFoundError( + f"Result file '{result_file_name}' not found in directory '{dir_path}'. " + f"Please run analyze first or remove this file from the folder." + ) + + upload_only_list.append(ReferenceDocItem( + file_name=file_name, + file_path=file_path, + result_file_name=result_file_name, + result_file_path=result_file_path + )) + elif file_name.lower().endswith(self.OCR_RESULT_FILE_SUFFIX.lower()): + ocr_suffix = self.OCR_RESULT_FILE_SUFFIX + original_file_name = file_name[:-len(ocr_suffix)] + original_file_path = os.path.join(dir_path, original_file_name) + + if os.path.exists(original_file_path): + origin_file_ext = os.path.splitext(original_file_name)[1] + + if self.is_supported_doc_type_by_file_ext(origin_file_ext, is_document=True): + continue + else: + raise ValueError( + f"The '{original_file_name}' is not a supported document type, " + f"please remove the result file '{file_name}' and '{original_file_name}'." + ) + else: + raise ValueError( + f"Result file '{file_name}' is not corresponding to an original file, " + f"please remove it." + ) + else: + raise ValueError( + f"File '{file_name}' is not a supported document type, " + f"please remove it or convert it to a supported type." + ) + + def _get_analyze_list(self, reference_docs_folder: str) -> List[ReferenceDocItem]: + """ + Get a list of reference document items from the specified folder and its subdirectories. + + Args: + reference_docs_folder: Path to the folder containing reference documents + + Returns: + List of ReferenceDocItem objects for supported document types + + Raises: + ValueError: If unsupported document types are found + """ + analyze_list: List[ReferenceDocItem] = [] + root_path = Path(reference_docs_folder) + + # Check if the root path exists + if not root_path.exists(): + return analyze_list + + # Get all files recursively (including root folder) + for file_path in root_path.rglob("*"): + if not file_path.is_file(): + continue + + try: + file_name_only = file_path.name + file_ext = file_path.suffix + + if self.is_supported_doc_type_by_file_ext(file_ext, is_document=True): + result_file_name = file_name_only + self.OCR_RESULT_FILE_SUFFIX + analyze_list.append(ReferenceDocItem( + file_name=file_name_only, + file_path=str(file_path), + result_file_name=result_file_name + )) + else: + raise ValueError( + f"File '{file_name_only}' is not a supported document type, " + f"please remove it or convert it to a supported type." + ) + except OSError: + # Skip files that can't be accessed + continue + + return analyze_list + + async def generate_training_data_on_blob( + self, + training_docs_folder: str, + storage_container_sas_url: str, + storage_container_path_prefix: str, + ) -> None: + if not storage_container_path_prefix.endswith("/"): + storage_container_path_prefix += "/" + + async with ContainerClient.from_container_url(storage_container_sas_url) as container_client: + for file_name in os.listdir(training_docs_folder): + file_path = os.path.join(training_docs_folder, file_name) + _, file_ext = os.path.splitext(file_name) + if os.path.isfile(file_path) and ( + file_ext == "" or file_ext.lower() in self.SUPPORTED_FILE_TYPES_DOCUMENT): + # Training feature only supports Standard mode with document data + # Document files uploaded to AI Foundry will be convert to uuid without extension + label_file_name = file_name + self.LABEL_FILE_SUFFIX + label_path = os.path.join(training_docs_folder, label_file_name) + ocr_result_file_name = file_name + self.OCR_RESULT_FILE_SUFFIX + ocr_result_path = os.path.join(training_docs_folder, ocr_result_file_name) + + if os.path.exists(label_path) and os.path.exists(ocr_result_path): + file_blob_path = storage_container_path_prefix + file_name + label_blob_path = storage_container_path_prefix + label_file_name + ocr_result_blob_path = storage_container_path_prefix + ocr_result_file_name + + # Upload files + await self._upload_file_to_blob(container_client, file_path, file_blob_path) + await self._upload_file_to_blob(container_client, label_path, label_blob_path) + await self._upload_file_to_blob(container_client, ocr_result_path, ocr_result_blob_path) + print(f"Uploaded training data for {file_name}") + else: + raise FileNotFoundError( + f"Label file '{label_file_name}' or OCR result file '{ocr_result_file_name}' " + f"does not exist in '{training_docs_folder}'. " + f"Please ensure both files exist for '{file_name}'." + ) + + async def _upload_file_to_blob( + self, container_client: ContainerClient, file_path: str, target_blob_path: str + ) -> None: + with open(file_path, "rb") as data: + await container_client.upload_blob(name=target_blob_path, data=data, overwrite=True) + print(f"Uploaded file to {target_blob_path}") + + async def _upload_json_to_blob( + self, container_client: ContainerClient, json_string: str, target_blob_path: str + ) -> None: + json_bytes = json_string.encode('utf-8') + await container_client.upload_blob(name=target_blob_path, data=json_bytes, overwrite=True) + print(f"Uploaded json to {target_blob_path}") + + async def _upload_jsonl_to_blob( + self, container_client: ContainerClient, data_list: List[str], target_blob_path: str + ) -> None: + jsonl_string = "\n".join(data_list) + jsonl_bytes = jsonl_string.encode("utf-8") + await container_client.upload_blob(name=target_blob_path, data=jsonl_bytes, overwrite=True) + print(f"Uploaded jsonl to blob '{target_blob_path}'") + + def is_supported_doc_type_by_file_ext(self, file_ext: str, is_document: bool=False) -> bool: + supported_types = ( + self.SUPPORTED_FILE_TYPES_DOCUMENT + if is_document else self.SUPPORTED_FILE_TYPES_DOCUMENT_TXT + ) + return file_ext.lower() in supported_types + + def is_supported_doc_type_by_file_path(self, file_path: Path, is_document: bool=False) -> bool: + if not file_path.is_file(): + return False + file_ext = file_path.suffix.lower() + return self.is_supported_doc_type_by_file_ext(file_ext, is_document) \ No newline at end of file diff --git a/python/extension/sample_helper.py b/python/extension/sample_helper.py new file mode 100644 index 0000000..3c3e21b --- /dev/null +++ b/python/extension/sample_helper.py @@ -0,0 +1,181 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------- +""" +Helper functions for Azure AI Content Understanding samples. +""" + +import json +import os +from datetime import datetime, timezone +from typing import Any, Optional, Dict +from enum import Enum +from azure.ai.contentunderstanding.models import ( + ContentField, +) + +def get_field_value(fields: Dict[str, ContentField], field_name: str) -> Any: + """ + Extract the actual value from a ContentField using the unified .value property. + + Args: + fields: A dictionary of field names to ContentField objects. + field_name: The name of the field to extract. + + Returns: + The extracted value or None if not found. + """ + if not fields or field_name not in fields: + return None + + field_data = fields[field_name] + + # Simply use the .value property which works for all ContentField types + return field_data.value + + +class PollerType(Enum): + """Enum to distinguish different types of pollers for operation ID extraction.""" + + ANALYZER_CREATION = "analyzer_creation" + ANALYZE_CALL = "analyze_call" + CLASSIFIER_CREATION = "classifier_creation" + CLASSIFY_CALL = "classify_call" + + +def save_json_to_file( + result, output_dir: str = "test_output", filename_prefix: str = "analysis_result" +) -> str: + """Persist the full AnalyzeResult as JSON and return the file path.""" + os.makedirs(output_dir, exist_ok=True) + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + path = os.path.join(output_dir, f"{filename_prefix}_{timestamp}.json") + with open(path, "w", encoding="utf-8") as fp: + json.dump(result, fp, indent=2, ensure_ascii=False) + print(f"πŸ’Ύ Analysis result saved to: {path}") + return path + + +def extract_operation_id_from_poller(poller: Any, poller_type: PollerType) -> str: + """Extract operation ID from an LROPoller or AsyncLROPoller. + + The poller stores the initial response in `_initial_response`, which contains + the Operation-Location header. The extraction pattern depends on the poller type: + - AnalyzerCreation: https://endpoint/contentunderstanding/operations/{operation_id}?api-version=... + - AnalyzeCall: https://endpoint/contentunderstanding/analyzerResults/{operation_id}?api-version=... + - ClassifierCreation: https://endpoint/contentunderstanding/operations/{operation_id}?api-version=... + - ClassifyCall: https://endpoint/contentunderstanding/classifierResults/{operation_id}?api-version=... + + Args: + poller: The LROPoller or AsyncLROPoller instance + poller_type: The type of poller (ANALYZER_CREATION, ANALYZE_CALL, CLASSIFIER_CREATION, or CLASSIFY_CALL) - REQUIRED + + Returns: + str: The operation ID extracted from the poller + + Raises: + ValueError: If no operation ID can be extracted from the poller or if poller_type is not provided + """ + if poller_type is None: + raise ValueError("poller_type is required and must be specified") + # Extract from Operation-Location header (standard approach) + initial_response = poller.polling_method()._initial_response + operation_location = initial_response.http_response.headers.get( + "Operation-Location" + ) + + if operation_location: + if ( + poller_type == PollerType.ANALYZER_CREATION + or poller_type == PollerType.CLASSIFIER_CREATION + ): + # Pattern: https://endpoint/.../operations/{operation_id}?api-version=... + if "/operations/" in operation_location: + operation_id = operation_location.split("/operations/")[1].split("?")[0] + return operation_id + elif poller_type == PollerType.ANALYZE_CALL: + # Pattern: https://endpoint/.../analyzerResults/{operation_id}?api-version=... + if "/analyzerResults/" in operation_location: + operation_id = operation_location.split("/analyzerResults/")[1].split( + "?" + )[0] + return operation_id + elif poller_type == PollerType.CLASSIFY_CALL: + # Pattern: https://endpoint/.../classifierResults/{operation_id}?api-version=... + if "/classifierResults/" in operation_location: + operation_id = operation_location.split("/classifierResults/")[1].split( + "?" + )[0] + return operation_id + + raise ValueError( + f"Could not extract operation ID from poller for type {poller_type}" + ) + + +def save_keyframe_image_to_file( + image_content: bytes, + keyframe_id: str, + test_name: str, + test_py_file_dir: str, + identifier: Optional[str] = None, + output_dir: str = "test_output", +) -> str: + """Save keyframe image to output file using pytest naming convention. + + Args: + image_content: The binary image content to save + keyframe_id: The keyframe ID (e.g., "keyFrame.1") + test_name: Name of the test case (e.g., function name) + test_py_file_dir: Directory where pytest files are located + identifier: Optional unique identifier to avoid conflicts (e.g., analyzer_id) + output_dir: Directory name to save the output file (default: "test_output") + + Returns: + str: Path to the saved image file + + Raises: + OSError: If there are issues creating directory or writing file + """ + # Generate timestamp and frame ID + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + frame_id = keyframe_id.replace("keyFrame.", "") + + # Create output directory if it doesn't exist + output_dir_path = os.path.join(test_py_file_dir, output_dir) + os.makedirs(output_dir_path, exist_ok=True) + + # Generate output filename with optional identifier to avoid conflicts + if identifier: + output_filename = f"{test_name}_{identifier}_{timestamp}_{frame_id}.jpg" + else: + output_filename = f"{test_name}_{timestamp}_{frame_id}.jpg" + + saved_file_path = os.path.join(output_dir_path, output_filename) + + # Write the image content to file + with open(saved_file_path, "wb") as image_file: + image_file.write(image_content) + + print(f"πŸ–ΌοΈ Image file saved to: {saved_file_path}") + return saved_file_path + + +def read_image_to_base64(image_path: str) -> str: + """Read image file and return base64-encoded string.""" + import base64 + + with open(image_path, "rb") as image_file: + image_bytes = image_file.read() + return base64.b64encode(image_bytes).decode("utf-8") + + +def read_image_to_base64_bytes(image_path: str) -> bytes: + """Read image file and return base64-encoded bytes.""" + import base64 + + with open(image_path, "rb") as image_file: + image_bytes = image_file.read() + return base64.b64encode(image_bytes) \ No newline at end of file