fix: resolve team integration citation, BYOD flow, and pipeline issues (#1927)

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: Roopan-Microsoft <[email protected]>
Co-authored-by: Ajit Padhi <[email protected]>
Co-authored-by: Roopan P M <[email protected]>
Co-authored-by: Ross Smith <[email protected]>
Co-authored-by: gpickett <[email protected]>
Co-authored-by: Francia Riesco <[email protected]>
Co-authored-by: Francia Riesco <[email protected]>
Co-authored-by: Prajwal D C <[email protected]>
Co-authored-by: Harmanpreet-Microsoft <[email protected]>
Co-authored-by: UtkarshMishra-Microsoft <[email protected]>
Co-authored-by: Priyanka-Microsoft <[email protected]>
Co-authored-by: Prasanjeet-Microsoft <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Kiran-Siluveru-Microsoft <[email protected]>
Co-authored-by: Prashant-Microsoft <[email protected]>
Co-authored-by: Rohini-Microsoft <[email protected]>
Co-authored-by: Avijit-Microsoft <[email protected]>
Co-authored-by: RaviKiran-Microsoft <[email protected]>
Co-authored-by: Somesh Joshi <[email protected]>
Co-authored-by: Himanshi Agrawal <[email protected]>
Co-authored-by: pradeepjha-microsoft <[email protected]>
Co-authored-by: Harmanpreet Kaur <[email protected]>
Co-authored-by: Bangarraju-Microsoft <[email protected]>
Co-authored-by: Harsh-Microsoft <[email protected]>
Co-authored-by: Kanchan-Microsoft <[email protected]>
Co-authored-by: Cristopher Coronado <[email protected]>
Co-authored-by: Cristopher Coronado Moreira <[email protected]>
Co-authored-by: Vamshi-Microsoft <[email protected]>
Co-authored-by: Thanusree-Microsoft <[email protected]>
Co-authored-by: Niraj Chaudhari (Persistent Systems Inc) <[email protected]>
Co-authored-by: Rohini-Microsoft <[email protected]>
Co-authored-by: Kingshuk-Microsoft <[email protected]>

Author: 30 people

.github/workflows/ci.yml

@@ -25,12 +25,17 @@ jobs:
       AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
       AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
       AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+      PRINCIPAL_ID: ${{ secrets.PRINCIPAL_ID }}
+      PRINCIPAL_NAME: ${{ secrets.PRINCIPAL_NAME }}
+      PRINCIPAL_TYPE: 'ServicePrincipal'
 
     outputs:
       imageTag: ${{ steps.set-image-tag.outputs.imageTag }}
       web_url: ${{ steps.extract-urls.outputs.web_url }}
       admin_url: ${{ steps.extract-urls.outputs.admin_url }}
       DEPLOYMENT_SUCCESS: ${{ steps.final-status.outputs.DEPLOYMENT_SUCCESS }}
+      resource_group: ${{ steps.check_create_rg.outputs.RESOURCE_GROUP_NAME }}
+      solution_suffix: ${{ steps.generate_solution_prefix.outputs.SOLUTION_SUFFIX }}
 
     steps:
       - name: Checkout code
@@ -50,7 +55,7 @@ jobs:
           export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
           export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
           export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
-          export GPT_MIN_CAPACITY="30"
+          export GPT_MIN_CAPACITY="150"
           export TEXT_EMBEDDING_MIN_CAPACITY="30"
           export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}"
 
@@ -105,7 +110,7 @@ jobs:
           rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
           if [ "$rg_exists" = "false" ]; then
             echo "Resource group does not exist. Creating..."
-            az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }} --tags SecurityControl=Ignore || { echo "Error creating resource group"; exit 1; }
+            az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }} --tags SecurityControl=Ignore CreatedBy="Pipeline" || { echo "Error creating resource group"; exit 1; }
           else
             echo "Resource group already exists."
           fi
@@ -203,6 +208,9 @@ jobs:
                               DISABLE_AUTHENTICATION=true
                               NO_AUTH=true
                               SKIP_AUTH=true
+                              PRINCIPAL_ID
+                              PRINCIPAL_NAME
+                              PRINCIPAL_TYPE
 
       - name: Extract URLs from deployment
         id: extract-urls
@@ -299,8 +307,6 @@ jobs:
 
                 echo "=== PostgreSQL Configuration Summary ==="
                 echo "Host Endpoint: $PG_HOST_DESTINATION"
-                echo "Username: admintest (hardcoded)"
-                echo "Password: Initial_0524 (hardcoded)"
                 echo "Database: postgres (hardcoded)"
                 echo "Port: 5432 (hardcoded)"
 
@@ -311,19 +317,28 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-                pip install psycopg2-binary python-dotenv
-
+                pip install psycopg2-binary python-dotenv azure-identity
 
       - name: Populate PostgreSQL Database
         run: |
           python - <<EOF
           import os
           import psycopg2
+          from azure.identity import ClientSecretCredential
+
+          tenant_id = os.environ["AZURE_TENANT_ID"]
+          client_id = os.environ["AZURE_CLIENT_ID"]
+          client_secret = os.environ["AZURE_CLIENT_SECRET"]
+          pg_host = os.environ.get("PG_HOST_DESTINATION", "localhost")
+
+          # Acquire Azure AD access token for PostgreSQL
+          credential = ClientSecretCredential(tenant_id, client_id, client_secret)
+          token = credential.get_token("https://ossrdbms-aad.database.windows.net/.default").token
 
           db_params = {
-              "user": "admintest",
-              "password": "Initial_0524",
-              "host": os.environ.get("PG_HOST_DESTINATION", "localhost"),
+              "user": client_id,   # Use service principal clientId
+              "password": token,   # Use AAD token instead of password
+              "host": pg_host,
               "port": "5432",
               "dbname": "postgres",
               "sslmode": "require"
@@ -343,8 +358,10 @@ jobs:
           except Exception as e:
               print(f"❌ Error during import: {e}")
           EOF
-
-
+        env:
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
 
       - name: Final Status Check
         id: final-status
@@ -386,9 +403,10 @@ jobs:
       AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
       AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
       AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
-      AZURE_ENV_NAME: ${{ github.run_id }}
+      AZURE_ENV_NAME: ${{ needs.deploy.outputs.solution_suffix }}  # Get from deploy job
       AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
       imageTag: ${{ needs.deploy.outputs.imageTag }}
+      AZURE_RESOURCE_GROUP: ${{ needs.deploy.outputs.resource_group }}
 
     steps:
       - name: Checkout code
@@ -408,6 +426,7 @@ jobs:
             AZURE_SUBSCRIPTION_ID
             AZURE_ENV_NAME
             AZURE_LOCATION
+            AZURE_RESOURCE_GROUP
 
       - name: Send Notification on Failure
         if: failure()

.github/workflows/create-release.yml

@@ -1,6 +1,6 @@
 on:
   workflow_run:
-    workflows: ["CI"]
+    workflows: ["Validate Deployment"]
     types:
       - completed
 

Makefile

@@ -57,11 +57,6 @@ azd-login: ## 🔑 Login to Azure with azd and a SPN
 	@echo -e "\e[34m$@\e[0m" || true
 	@azd auth login --client-id ${AZURE_CLIENT_ID} --client-secret ${AZURE_CLIENT_SECRET} --tenant-id ${AZURE_TENANT_ID}
 
-azd-login: ## 🔑 Login to Azure with azd and a SPN
-	@echo -e "\e[34m$@\e[0m" || true
-	@azd auth login --client-id ${AZURE_CLIENT_ID} --client-secret ${AZURE_CLIENT_SECRET} --tenant-id ${AZURE_TENANT_ID}
-
-# Fixed Makefile section for deploy target
 # Fixed Makefile section for deploy target
 deploy: azd-login ## Deploy everything to Azure
 	@echo -e "\e[34m$@\e[0m" || true
@@ -126,15 +121,10 @@ deploy: azd-login ## Deploy everything to Azure
 		echo "$$PG_HOST_VAL" > pg_host.txt
 
 
-
-
-
 	@echo "=== PostgreSQL Configuration ==="
-	@echo "Username: admintest (hardcoded)"
 	@echo "Database: postgres (hardcoded)"
 	@echo "Port: 5432 (hardcoded)"
 	@echo "Host: $$(cat pg_host.txt 2>/dev/null || echo 'Not available')"
-	@echo "Password: Initial_0524 (hardcoded)"
 
 # Helper target to check current authentication status
 check-auth:
@@ -178,4 +168,5 @@ disable-auth-fixed:
 
 destroy: azd-login ## 🧨 Destroy everything in Azure
 	@echo -e "\e[34m$@\e[0m" || true
+	@azd env select $(AZURE_ENV_NAME) || true
 	@azd down --force --purge --no-prompt

README.md

@@ -188,7 +188,8 @@ To review Cosmos DB configuration overview and steps, follow the link [here](doc
  <br/>To ensure sufficient quota is available in your subscription, please follow [quota check instructions guide](./docs/QuotaCheck.md) before you deploy the solution.
 
 <br/>
-The "Deploy to Azure" button offers a one-click deployment where you don’t have to clone the code. If you would like a developer experience instead, follow the [local deployment instructions](./docs/LOCAL_DEPLOYMENT.md).
+
+The "Deploy to Azure" button offers a one-click deployment where you don’t have to clone the code. If you would like a developer experience instead, follow the [local deployment instructions](./docs/LOCAL_DEPLOYMENT.md)
 
 Once you deploy to Azure, you will have the option to select PostgreSQL or Cosmos DB, see screenshot below.
 
@@ -205,6 +206,23 @@ locations support this version. If you're deploying to a location that doesn't s
 switch to a lower version. To find out which versions are supported in different regions, visit the
 [GPT-4.1 Model Availability](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models?tabs=global-standard%2Cstandard-chat-completions#global-standard-model-availability) page.
 
+
+### Supported Azure Regions
+
+The solution has been tested and is compatible with the following Azure regions:
+- **Australia East**
+- **East US 2**
+- **Japan East**
+- **UK South**
+
+These regions are specifically configured in the deployment template to guarantee compatibility with paired regions and data redundancy. This restriction ensures reliable failover scenarios based on Azure's region availability and the requirements of services like Azure Database for PostgreSQL Flexible Server.
+
+When deploying the solution using the "Deploy to Azure" button, you'll see two fields in the Azure portal:
+- **Region**: This refers to the Azure region where the deployment metadata is stored
+- **Location**: This corresponds to the "location" parameter in the bicep template and determines where all your solution resources will be deployed
+
+**Important**: For this solution, you must select one of the supported regions listed above in the "Location" field. The "Region" field can be set to any available region since it only affects deployment metadata storage.
+
 ### Testing the deployment
 1. Navigate to the admin site, where you can upload documents. It will be located at:
 

azure.yaml

@@ -3,6 +3,10 @@
 name: chat-with-your-data-solution-accelerator
 metadata:
    template: [email protected]
+
+requiredVersions:
+  azd: '>= 1.18.0'
+
 hooks:
   postprovision:
     # run: ./infra/prompt-flow/create-prompt-flow.sh

code/backend/batch/combine_pages_chunknos.py

@@ -0,0 +1,58 @@
+import logging
+import azure.functions as func
+import json
+
+bp_combine_pages_and_chunknos = func.Blueprint()
+
+
+@bp_combine_pages_and_chunknos.route(route="combine_pages_and_chunknos", methods=["POST"], auth_level=func.AuthLevel.ANONYMOUS)
+def combine_pages_and_chunknos(req: func.HttpRequest) -> func.HttpResponse:
+    """
+    This function is designed to be called by an Azure Cognitive Search WebApiSkill.
+    It expects a JSON payload with two arrays ("pages" and "chunk_nos") and
+    combines them into a single array of objects.
+    """
+    logging.info("Combine pages and chunk numbers function processed a request.")
+
+    try:
+        req_body = req.get_json()
+        values = req_body.get("values", [])
+
+        response_values = []
+
+        for value in values:
+            record_id = value.get("recordId")
+            data = value.get("data", {})
+
+            pages = data.get("pages", [])
+            chunk_nos = data.get("chunk_nos", [])
+
+            # Zip the two arrays together
+            zipped_data = [
+                {"page_text": page, "chunk_no": chunk}
+                for page, chunk in zip(pages, chunk_nos)
+            ]
+
+            response_values.append(
+                {
+                    "recordId": record_id,
+                    "data": {"pages_with_chunks": zipped_data},
+                    "errors": None,
+                    "warnings": None,
+                }
+            )
+
+        # Return the response in the format expected by the WebApiSkill
+        return func.HttpResponse(
+            body=json.dumps({"values": response_values}),
+            mimetype="application/json",
+            status_code=200,
+        )
+
+    except Exception as e:
+        logging.error(f"Error in combine_pages_and_chunknos function: {e}")
+        return func.HttpResponse(
+            body=json.dumps({"values": [{"recordId": "error", "data": {}, "errors": [{"message": str(e)}], "warnings": []}]}),
+            mimetype="application/json",
+            status_code=500,
+        )

code/backend/batch/function_app.py

@@ -5,6 +5,7 @@
 from batch_push_results import bp_batch_push_results
 from batch_start_processing import bp_batch_start_processing
 from get_conversation_response import bp_get_conversation_response
+from combine_pages_chunknos import bp_combine_pages_and_chunknos
 from azure.monitor.opentelemetry import configure_azure_monitor
 
 logging.captureWarnings(True)
@@ -20,3 +21,4 @@
 app.register_functions(bp_batch_push_results)
 app.register_functions(bp_batch_start_processing)
 app.register_functions(bp_get_conversation_response)
+app.register_functions(bp_combine_pages_and_chunknos)

code/backend/batch/utilities/integrated_vectorization/azure_search_indexer.py

@@ -1,5 +1,5 @@
 import logging
-from azure.search.documents.indexes.models import SearchIndexer, FieldMapping
+from azure.search.documents.indexes.models import SearchIndexer, FieldMapping, FieldMappingFunction
 from azure.search.documents.indexes import SearchIndexerClient
 from ..helpers.env_helper import EnvHelper
 from ..helpers.azure_credential_utils import get_azure_credential
@@ -35,6 +35,13 @@ def create_or_update_indexer(self, indexer_name: str, skillset_name: str):
                 }
             },
             field_mappings=[
+                FieldMapping(
+                    source_field_name="metadata_storage_path",
+                    target_field_name="id",
+                    mapping_function=FieldMappingFunction(
+                        name="base64Encode", parameters={"useHttpServerUtilityUrlTokenEncode": False}
+                    )
+                ),
                 FieldMapping(
                     source_field_name="metadata_storage_path",
                     target_field_name="source",