From 50c64497498e671f16a548472680361a3159887e Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Mon, 6 Jan 2025 11:24:26 -0600 Subject: [PATCH 01/24] Changed database pods to be internal to VNet and updated AKS version --- .../Resources/ARM-Templates/KubernetesCluster/parameters.json | 2 +- .../Student/Resources/HelmCharts/MySQL57/values.yaml | 2 +- .../Student/Resources/HelmCharts/Oracle184/values.yaml | 2 +- .../Student/Resources/HelmCharts/Oracle21c/values.yaml | 2 +- .../Student/Resources/HelmCharts/PostgreSQL116/values.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/033-OSSDatabaseMigration/Student/Resources/ARM-Templates/KubernetesCluster/parameters.json b/033-OSSDatabaseMigration/Student/Resources/ARM-Templates/KubernetesCluster/parameters.json index 72d35151dd..8886a04055 100644 --- a/033-OSSDatabaseMigration/Student/Resources/ARM-Templates/KubernetesCluster/parameters.json +++ b/033-OSSDatabaseMigration/Student/Resources/ARM-Templates/KubernetesCluster/parameters.json @@ -40,7 +40,7 @@ "value": "ossdbmigration-dns" }, "kubernetesVersion": { - "value": "1.25.5" + "value": "1.31.2" }, "networkPlugin": { "value": "azure" diff --git a/033-OSSDatabaseMigration/Student/Resources/HelmCharts/MySQL57/values.yaml b/033-OSSDatabaseMigration/Student/Resources/HelmCharts/MySQL57/values.yaml index 4ce5bad0bd..8352bcc95b 100644 --- a/033-OSSDatabaseMigration/Student/Resources/HelmCharts/MySQL57/values.yaml +++ b/033-OSSDatabaseMigration/Student/Resources/HelmCharts/MySQL57/values.yaml @@ -18,7 +18,7 @@ image: tag: "5.7.32" service: - type: LoadBalancer + type: ClusterIP port: 3306 protocol: TCP diff --git a/033-OSSDatabaseMigration/Student/Resources/HelmCharts/Oracle184/values.yaml b/033-OSSDatabaseMigration/Student/Resources/HelmCharts/Oracle184/values.yaml index 6b81c2ca93..53408b6721 100644 --- a/033-OSSDatabaseMigration/Student/Resources/HelmCharts/Oracle184/values.yaml +++ b/033-OSSDatabaseMigration/Student/Resources/HelmCharts/Oracle184/values.yaml @@ -18,7 +18,7 @@ image: tag: "18.4.0-xe" service: - type: LoadBalancer + type: ClusterIP port: 1521 protocol: TCP diff --git a/033-OSSDatabaseMigration/Student/Resources/HelmCharts/Oracle21c/values.yaml b/033-OSSDatabaseMigration/Student/Resources/HelmCharts/Oracle21c/values.yaml index 53323ea3e3..b90078f7cc 100644 --- a/033-OSSDatabaseMigration/Student/Resources/HelmCharts/Oracle21c/values.yaml +++ b/033-OSSDatabaseMigration/Student/Resources/HelmCharts/Oracle21c/values.yaml @@ -18,7 +18,7 @@ image: tag: "21.3.0-xe" service: - type: LoadBalancer + type: ClusterIP port: 1521 protocol: TCP diff --git a/033-OSSDatabaseMigration/Student/Resources/HelmCharts/PostgreSQL116/values.yaml b/033-OSSDatabaseMigration/Student/Resources/HelmCharts/PostgreSQL116/values.yaml index eb358ca3e1..a42e92f082 100644 --- a/033-OSSDatabaseMigration/Student/Resources/HelmCharts/PostgreSQL116/values.yaml +++ b/033-OSSDatabaseMigration/Student/Resources/HelmCharts/PostgreSQL116/values.yaml @@ -18,7 +18,7 @@ image: tag: "11.6" service: - type: LoadBalancer + type: ClusterIP port: 5432 protocol: TCP From 7fe06c5b27dcf8ebcbfb11e9c6092d54c35d653a Mon Sep 17 00:00:00 2001 From: Devanshi Thakar <33441411+devanshithakar12@users.noreply.github.com> Date: Mon, 28 Apr 2025 18:23:35 -0700 Subject: [PATCH 02/24] Got rid of any mention of gpt-4 --- 066-OpenAIFundamentals/Student/Challenge-00.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/066-OpenAIFundamentals/Student/Challenge-00.md b/066-OpenAIFundamentals/Student/Challenge-00.md index ab109bd8dd..703895e7cc 100644 --- a/066-OpenAIFundamentals/Student/Challenge-00.md +++ b/066-OpenAIFundamentals/Student/Challenge-00.md @@ -144,7 +144,7 @@ Now we will deploy the needed large language models from Azure OpenAI. - Navigate to the [AI Foundry](https://ai.azure.com) and click on **Hub Overview**. In the Connected resources, you should see Azure OpenAI. - On the left navigation bar, click on Deployments. - Deploy the following models in your Azure OpenAI resource. - - `gpt-4` or `gpt-4o` + - `gpt-4o` - `gpt-35-turbo` - `text-embedding-ada-002` From 6c87e929dbcb8ec386282c9ae167c3afdf6f61fe Mon Sep 17 00:00:00 2001 From: Devanshi Thakar <33441411+devanshithakar12@users.noreply.github.com> Date: Mon, 28 Apr 2025 18:25:51 -0700 Subject: [PATCH 03/24] Delete 066-OpenAIFundamentals/Student/Resources/notebooks/CH-02-ModelComparison.ipynb --- .../notebooks/CH-02-ModelComparison.ipynb | 1061 ----------------- 1 file changed, 1061 deletions(-) delete mode 100644 066-OpenAIFundamentals/Student/Resources/notebooks/CH-02-ModelComparison.ipynb diff --git a/066-OpenAIFundamentals/Student/Resources/notebooks/CH-02-ModelComparison.ipynb b/066-OpenAIFundamentals/Student/Resources/notebooks/CH-02-ModelComparison.ipynb deleted file mode 100644 index 99e5c73133..0000000000 --- a/066-OpenAIFundamentals/Student/Resources/notebooks/CH-02-ModelComparison.ipynb +++ /dev/null @@ -1,1061 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Challenge 02 - OpenAI Models & Capabilities" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Introduction\n", - "\n", - "In this challenge, you will learn about the different capabilities of OpenAI models and learn how to choose the best model for your use case.\n", - "\n", - "You are going to compare GPT-3.5 to GPT-4 model in this Challenge. If you do not have GPT-4 access, you can compare the legacy models if they are deployed, or go through this challenge conceptually to understand how to best pick a model from the ones you have deployed as well as the ones in the model catalog.\n", - "\n", - "In a world where the availability and development of models are always changing, the model we compare may change over time. But we encourage you to understand the general concepts and material in this Challenge because the comparison techniques utilized can be applicable to scenarios where you are comparing Large Language Models.\n", - "\n", - "Questions you will be able to answer by the end of this challenge:\n", - "\n", - "* How do responses differ for each model?\n", - "* What are ways to benchmark the performance of models? " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## 1. Overview on finding the right model for you\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 1.1 Model Families\n", - "\n", - "Azure OpenAI provides access to many different models, grouped by family and capability. A model family typically associates models by their intended task. \n", - "\n", - "Model families currently available as of _Dec 1, 2023_ in Azure OpenAI includes GPT-4, GPT-3.5, Embeddings, DALL-E, and Whisper. Please reference this link for more information: https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models\n", - "\n", - "For GPT-3 and other models retiring in July 2024, see [Azure OpenAI Service legacy models](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/legacy-models)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 1.2 Model Capacities\n", - "#### GPT-4\n", - "GPT-4 can solve difficult problems with greater accuracy than any of OpenAI's previous models. Like GPT-3.5 Turbo, GPT-4 is optimized for chat and works well for traditional completions tasks. Use the Chat Completions API to use GPT-4.\n", - "\n", - "#### GPT-3.5\n", - "GPT-3.5 models can understand and generate natural language or code. The most capable and cost effective model in the GPT-3.5 family is GPT-3.5 Turbo, which has been optimized for chat and works well for traditional completions tasks as well. GPT-3.5 Turbo is available for use with the Chat Completions API. GPT-3.5 Turbo Instruct has similar capabilities to text-davinci-003 using the Completions API instead of the Chat Completions API. We recommend using GPT-3.5 Turbo and GPT-3.5 Turbo Instruct over legacy GPT-3.5 and GPT-3 models.\n", - "\n", - "`gpt-35-turbo`\n", - "\n", - "`gpt-35-turbo-16k`\n", - "\n", - "`gpt-35-turbo-instruct`\n", - "\n", - "You can see the token context length supported by each model in the [model summary table](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#model-summary-table-and-region-availability).\n", - "\n", - "#### Embeddings \n", - "The previous embeddings models have been consolidated into the following new replacement model:\n", - "\n", - "`text-embedding-ada-002`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "\n", - "[Azure OpenAI models](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/concepts/models) \n", - "\n", - "\n", - "| Models | Description |\n", - "| --- | --- |\n", - "| GPT-4 | A set of models that improve on GPT-3.5 and can understand and generate natural language and code. | \n", - "| GPT-3.5 | A set of models that improve on GPT-3 and can understand and generate natural language and code. | \n", - "| Embeddings | A set of models that can convert text into numerical vector form to facilitate text similarity. | \n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 1.3 Pricing Details\n", - "\n", - "For the most up-to-date information, check out the Azure OpenAI [pricing page](https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/).\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 1.4 Quotas and Limits\n", - "\n", - "*The below limits are subject to change. We anticipate that you will need higher limits as you move toward production and your solution scales. When you know your solution requirements, please reach out to us by applying for a quota increase here: https://aka.ms/oai/quotaincrease\n", - "\n", - "|Limit Name\t|Limit Value|\n", - "|---|---|\n", - "|OpenAI resources per region per Azure subscription|\t30|\n", - "| Default DALL-E 2 quota limits| 2 concurrent requests |\n", - "| Default DALL-E 3 quota limits | 2 capacity units (6 requests per minute)|\n", - "|Maximum prompt tokens per request| Varies per model, see [Azure OpenAI Service models](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models) |\n", - "|Max fine-tuned model deployments|5|\n", - "|Total number of training jobs per resource|\t100|\n", - "|Max simultaneous running training jobs per resource|\t1|\n", - "|Max training jobs queued\t|20|\n", - "|Max Files per resource\t|30|\n", - "|Total size of all files per resource\t|1 GB|\n", - "|Max training job time (job will fail if exceeded)\t|720 hours|\n", - "|Max training job size (tokens in training file) x (# of epochs)\t|2 Billion|\n", - "|Max size of all files per upload (Azure OpenAI on your data)\t|16 MB|" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 1.5 Model Selection\n", - "\n", - "Here is some general guidance on well-suited scenarios that tend to differentiate models. Note that these are not hard and fast rules, and oftentimes experimentation and benchmarking are important to making the best decision for your solution.\n", - "\n", - "|Model|Use Cases|\n", - "|---|---|\n", - "|GPT-3.5| Faster/cheaper app performance;
More quota allocated by default|\n", - "|GPT-4| More advanced reasoning or logical processing needed;
32k token window is an absolut must;
Multilingual proficiency needed;
No hard requirement on low latency|" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 1.6 Model Selection Best Practices\n", - "Our recommendation is for users to start with GPT-3.5 Turbo if identify with the use case listed above, and move to GPT-4 if needed.\n", - "\n", - "Once you have a prototype working, you can then optimize your model choice with the best latency/performance balance for your application." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## 2. Let's Start Implementation" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "If you don't already have the OpenAI, Python-dotenv, plotly, or scikit-learn packages installed on your compute, the following cells will install them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1685909662455 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "import openai\n", - "import os\n", - "import json\n", - "from dotenv import load_dotenv, find_dotenv\n", - "import pandas as pd\n", - "load_dotenv(find_dotenv())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Set up your environment to access your OpenAI keys. Refer to your OpenAI resource in the Azure Portal to retrieve information regarding your OpenAI endpoint and keys.\n", - "\n", - "For security purposes, store your sensitive information in a .env file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1686331271142 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "API_KEY = os.getenv(\"OPENAI_API_KEY\")\n", - "assert API_KEY, \"ERROR: Azure OpenAI Key is missing\"\n", - "openai.api_key = API_KEY\n", - "\n", - "RESOURCE_ENDPOINT = os.getenv(\"OPENAI_API_BASE\",\"\").strip()\n", - "assert RESOURCE_ENDPOINT, \"ERROR: Azure OpenAI Endpoint is missing\"\n", - "assert \"openai.azure.com\" in RESOURCE_ENDPOINT.lower(), \"ERROR: Azure OpenAI Endpoint should be in the form: \\n\\n\\t.openai.azure.com\"\n", - "\n", - "openai.api_base = RESOURCE_ENDPOINT\n", - "openai.api_type = os.getenv(\"OPENAI_API_TYPE\")\n", - "openai.api_version = os.getenv(\"OPENAI_API_VERSION\")\n", - "\n", - "chat_model=os.getenv(\"CHAT_MODEL_NAME\")\n", - "chat_model2=os.getenv(\"CHAT_MODEL_NAME2\")\n", - "text_model=os.getenv(\"EMBEDDING_MODEL_NAME\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 2.0 Helper Functions\n", - "Throughout this course, we will use OpenAI's `gpt-3.5-turbo` and `gpt-4` models and the [chat completions endpoint](https://platform.openai.com/docs/guides/chat). \n", - "\n", - "This helper function will make it easier to use prompts and look at the generated outputs." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "**timer wrapper** helps us monitor and compare the latency of each model.\n", - "\n", - "**get_chat_completion** helps create the OpenAI response using the chat model of your choice.\n", - "\n", - "**get_completion_from_messages** helps create the OpenAI response using the chat model of your choice, enabling chat history." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1686334202300 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "import functools\n", - "import time\n", - "\n", - "def timer(func):\n", - " @functools.wraps(func)\n", - " def wrapper(*args, **kwargs):\n", - " start_time = time.perf_counter()\n", - " value = func(*args, **kwargs)\n", - " end_time = time.perf_counter()\n", - " run_time = end_time - start_time\n", - " print(\"Finished {} in {} secs\".format(repr(func.__name__), round(run_time, 3)))\n", - " return value[0], value[1], round(run_time, 3)\n", - "\n", - " return wrapper" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1686334262004 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "@timer\n", - "def get_chat_completion(prompt, model):\n", - " messages = [{\"role\": \"user\", \"content\": prompt}]\n", - " response = openai.ChatCompletion.create(\n", - " engine=model,\n", - " messages=messages,\n", - " temperature=0, # this is the degree of randomness of the model's output\n", - " max_tokens = 200,\n", - " top_p = 1.0,\n", - " )\n", - " return response.choices[0].message[\"content\"],response['usage']['total_tokens']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1686334263077 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "@timer\n", - "def get_completion_from_messages(messages, model, temperature=0):\n", - " response = openai.ChatCompletion.create(\n", - " engine=model,\n", - " messages=messages,\n", - " temperature=temperature, # this is the degree of randomness of the model's output\n", - " )\n", - " #print(str(response.choices[0].message))\n", - " return response.choices[0].message[\"content\"],response['usage']['total_tokens']" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 2.1 Summarize Text" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1686334268062 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "model_pricing = pd.DataFrame(columns=['model', 'price', 'time'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1686334278657 - } - }, - "outputs": [], - "source": [ - "text = f\"\"\"\n", - "The Olympic Games Tokyo 2020 reached a global broadcast audience of 3.05 billion people, according to independent research conducted on behalf of the International Olympic Committee (IOC). Official coverage on Olympic broadcast partners\\' digital platforms alone generated 28 billion video views in total – representing a 139 per cent increase compared with the Olympic Games Rio 2016 and underlining the changing media landscape and Tokyo 2020\\'s designation as the first streaming Games and the most watched Olympic Games ever on digital platforms.Sony and Panasonic partnered with NHK to develop broadcasting standards for 8K resolution television, with a goal to release 8K television sets in time for the 2020 Summer Olympics. In early 2019, Italian broadcaster RAI announced its intention to deploy 8K broadcasting for the Games. NHK broadcast the opening and closing ceremonies, and coverage of selected events in 8K. Telecom company NTT Docomo signed a deal with Finland\\'s Nokia to provide 5G-ready baseband networks in Japan in time for the Games.The Tokyo Olympics were broadcast in the United States by NBCUniversal networks, as part of a US$4.38 billion agreement that began at the 2014 Winter Olympics in Sochi. The United States Olympic & Paralympic Committee asserted that a \"right of abatement\" clause in the contract was triggered by the delay of the Games to 2021, requiring the IOC to \"negotiate in good faith an equitable reduction in the applicable broadcast rights payments\" by NBC, which remains one of IOC\\'s biggest revenue streams. According to NBCUniversal CEO Jeff Shell, the Tokyo games could be the most profitable Olympics in NBC\\'s history. The Tokyo games were NBC\\'s first Olympics broadcast under current president Susan Rosner Rovner.In Europe, this was the first Summer Olympics under the IOC\\'s exclusive pan-European rights deal with Eurosport, which began at the 2018 Winter Olympics and is contracted to run through 2024. The rights for the 2020 Summer Olympics covered almost all of Europe; a pre-existing deal with a marketer excludes Russia. Eurosport planned to sub-license coverage to free-to-air networks in each territory, and other channels owned by Discovery, Inc. subsidiaries. In the United Kingdom, these were set to be the last Games with rights owned primarily by the BBC, although as a condition of a sub-licensing agreement due to carry into the 2022 and 2024 Games, Eurosport holds exclusive pay television rights. In France, these were the last Games whose rights are primarily owned by France Télévisions. Eurosport debuted as pay television rightsholder, after Canal+ elected to sell its pay television rights as a cost-saving measure.In Canada, the 2020 Games were shown on CBC/Radio-Canada platforms, Sportsnet, TSN and TLN. In Australia, they were aired by Seven Network. In the Indian subcontinent, they were aired by Sony Pictures Networks India (SPN).\n", - "\"\"\"\n", - "prompt = f\"\"\"\n", - "Summarize the text delimited by triple backticks into a single sentence.\n", - "```{text}```\n", - "\"\"\"\n", - "\n", - "gpt35_response, gpt35_price, gpt35_time = get_chat_completion(prompt, model=chat_model)\n", - "gpt4_response, gpt4_price, gpt4_time = get_chat_completion(prompt, model=chat_model2)\n", - "\n", - "print(f\"GPT-3.5 Response: {gpt35_response}\\n\")\n", - "print(f\"GPT-4 Response: {gpt4_response}\\n\")\n", - "\n", - "new_rows = pd.DataFrame([{'model': 'gpt3.5', 'price': gpt35_price, 'time': gpt35_time},\n", - " {'model': 'gpt4', 'price': gpt4_price, 'time': gpt4_time}])\n", - "pricing = pd.concat([model_pricing, new_rows], ignore_index=True)\n", - "print(pricing)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Student Task #1:\n", - "With tactics learned in CH1, edit the prompt to get more concise answer from the assistant. Do you find any difference in the result?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# Edit the prompt to get more concise answer from assistant" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 2.2 Summarization for a targeted audience" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1686332538379 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "prompt = f\"\"\"\n", - "Summarize the text delimited by triple backticks into a single sentence for 7-year-old to understand.\n", - "```{text}```\n", - "\"\"\"\n", - "\n", - "gpt35_response, gpt35_price, gpt35_time = get_chat_completion(prompt, model=chat_model)\n", - "gpt4_response, gpt4_price, gpt4_time = get_chat_completion(prompt, model=chat_model2)\n", - "print(f\"GPT-3.5 Response: {gpt35_response}\\n\")\n", - "print(f\"GPT-4 Response: {gpt4_response}\\n\")\n", - "\n", - "new_rows = pd.DataFrame([{'model': 'gpt3.5', 'price': gpt35_price, 'time': gpt35_time},\n", - " {'model': 'gpt4', 'price': gpt4_price, 'time': gpt4_time}])\n", - "pricing = pd.concat([model_pricing, new_rows], ignore_index=True)\n", - "print(pricing)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Student Challenge #2:\n", - "Edit the prompt to summarize the text for eye-catching newspaper title. Compare different results." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# Edit the prompt to summarize the text for eye-catching newspaper title" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 2.3 Summarize Cause & Effect" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1686332587257 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "prompt = f\"\"\"\n", - "Summarize the major event's cause and effect for the text delimited by triple backticks into a single sentence less than 50 words.\n", - "```{text}```\n", - "\"\"\"\n", - "\n", - "gpt35_response, gpt35_price, gpt35_time = get_chat_completion(prompt, model=chat_model)\n", - "gpt4_response, gpt4_price, gpt4_time = get_chat_completion(prompt, model=chat_model2)\n", - "print(f\"GPT-3.5 Response: {gpt35_response}\\n\")\n", - "print(f\"GPT-4 Response: {gpt4_response}\\n\")\n", - "\n", - "new_rows = pd.DataFrame([{'model': 'gpt3.5', 'price': gpt35_price, 'time': gpt35_time},\n", - " {'model': 'gpt4', 'price': gpt4_price, 'time': gpt4_time}])\n", - "pricing = pd.concat([model_pricing, new_rows], ignore_index=True)\n", - "print(pricing)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Student Task #3: Model Comparison\n", - "Use the model comparison chart to briefly summarize your findings after comparing different model output & time taken. eg. GPT-4: Performance (+++), time (+). You may also leverage other python packages to visualize your findings.\n", - "\n", - "|Model| Performance |Time|\n", - "|---|---|---|\n", - "|GPT-3.5|||\n", - "|GPT-4|||" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - " #### Student Task #4: Text Classification\n", - " Edit the prompt to make the models generate key topic categories for the text. Compare different model performance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# Edit the prompt to make the models generate key topic categories for the text" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Student Task #5:\n", - "Edit the prompt to make the models generate more precise results. Compare different model performance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# Edit the prompt to make the models generate more precise results. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Student Task #6: Model Comparison\n", - "\n", - "Write code to create two bar charts comparing the **price** and **time for completion** between the models. We recommend using the `matplotlib.pyplot` library for making visualizations.\n", - "\n", - "Instructions for completion:\n", - "\n", - "* Utilize the `model_comparison` dataframe to calculate the averages of price and time for each model\n", - "* Produce the bar chart in a currency amount. Note that the `price` column in the `model_comparison` dataframe is in the unit of tokens. Refer to the Azure [OpenAI pricing page](https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/) to convert the units." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "\"\"\" STUDENT TASK \"\"\"\n", - "\n", - "import matplotlib.pyplot as plt\n", - "\n", - "### 1. Bar chart to compare pricing\n", - "\n", - "\n", - "### 2. Bar chart to compare time for completion" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 2.4 Generate Nick Names" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Student Task #7:\n", - "Use different models to create nick names for players from examples words. Compare different model performance. (You can set the temperature value high to increase randomness and more innovative responses.)\n", - "\n", - "Player description: The champion of Men's 100 metre freestyle swimming. Seed words: fast, strong, talented.Nick names: Swimming Genius, Dark Horse, 100-Metre-Freestyle Killer\n", - "\n", - "Player description: The champion of Women Figure Skating. Seed words: elegant, talented, soft." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1685916265011 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# Write your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Model Comparison\n", - "|Model| Performance |Time|Tokens|Pricing |\n", - "|---|---|---|---|---|\n", - "|GPT-3.5|||||\n", - "|GPT-4||||||" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 2.5 Embeddings\n", - "This section focuses on how to retrieve embeddings using different embedding models, and find similarity between documents. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Student Task #8:\n", - "Compare the summaries of two swimming games at the 2020 Summer Olympics using the data provided below.\n", - "\n", - "See whether there are differences using different embedding models to compare." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1686117865502 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from openai.embeddings_utils import get_embedding, cosine_similarity" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1686117698204 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "import pandas as pd\n", - "game_summary = [\n", - " \"The mixed 100 metre medley relay event at the 2020 Summer Olympics was held in 2021 at the Tokyo Aquatics Centre. These Games marked the first time to feature a mixed-gender swimming event in the program. Each 4-person team features two male and two female swimmers in no particular order. The medals for the competition were presented by Kirsty Coventry IOC Executive Board Member, Zimbabwe; Olympian, 2 Gold Medals, 4 Silver Medals, 1 Bronze Medal, and the medalists bouquets were presented by Errol Clarke, FINA Bureau Member; Barbados.\",\n", - " \"The men's 200 metre breaststroke event at the 2020 Summer Olympics was held from 27 to 29 July 2021 at the Tokyo Aquatics Centre. It was the event's twenty-sixth consecutive appearance, having been held at every edition since 1908.\"\n", - "]\n", - "\n", - "game_highlight = [\n", - " 'The 2020 Summer Olympics featured the first ever mixed-gender swimming event, the 100 metre medley relay. Medals were presented by Kirsty Coventry and bouquets by Errol Clarke.',\n", - " \"The men's 200 metre breaststroke event was held at the 2020 Summer Olympics in Tokyo, making it the event's 26th consecutive appearance since 1908.\"\n", - "]\n", - "\n", - "olympics_game_df = pd.DataFrame({\"summary\":game_summary, \"qualification\":game_highlight})\n", - "\n", - "olympics_game_df.head() " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1686117707487 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "@timer\n", - "def get_embedding(text, model=text_model):\n", - " response = openai.Embedding.create(\n", - " input=text,\n", - " engine=model\n", - " )\n", - " return response[\"data\"][0][\"embedding\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1686117710151 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "article1_embedding = get_embedding(text=olympics_game_df.summary.iloc[0])\n", - "article2_embedding = get_embedding(text=olympics_game_df.summary.iloc[1])\n", - "print(cosine_similarity(article1_embedding, article2_embedding))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Conclusion\n", - "\n", - "In this Challenge, you learned about techniques to compare different types of models from Azure OpenAI. Although we recommended using GPT-3.5 and GPT-4, these methods can be applied to other models as well to determine the best solution for your use case. In Challenge 3, you will learn how to work with larger amounts of data." - ] - } - ], - "metadata": { - "kernel_info": { - "name": "python310-sdkv2" - }, - "kernelspec": { - "display_name": "Python 3.10 - SDK v2", - "language": "python", - "name": "python310-sdkv2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.10" - }, - "microsoft": { - "host": { - "AzureML": { - "notebookHasBeenCompleted": true - } - }, - "ms_spell_check": { - "ms_spell_check_language": "en" - } - }, - "nteract": { - "version": "nteract-front-end@1.0.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file From 40fec1aec1138e18bd46d10f494890a1be5d4971 Mon Sep 17 00:00:00 2001 From: Devanshi Thakar <33441411+devanshithakar12@users.noreply.github.com> Date: Mon, 28 Apr 2025 18:26:04 -0700 Subject: [PATCH 04/24] Delete 066-OpenAIFundamentals/Student/Resources/notebooks/CH-05-ResponsibleAI.ipynb --- .../notebooks/CH-05-ResponsibleAI.ipynb | 1296 ----------------- 1 file changed, 1296 deletions(-) delete mode 100644 066-OpenAIFundamentals/Student/Resources/notebooks/CH-05-ResponsibleAI.ipynb diff --git a/066-OpenAIFundamentals/Student/Resources/notebooks/CH-05-ResponsibleAI.ipynb b/066-OpenAIFundamentals/Student/Resources/notebooks/CH-05-ResponsibleAI.ipynb deleted file mode 100644 index de78389bed..0000000000 --- a/066-OpenAIFundamentals/Student/Resources/notebooks/CH-05-ResponsibleAI.ipynb +++ /dev/null @@ -1,1296 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "id": "e3964eae", - "metadata": {}, - "source": [ - "# Challenge 5: Responsible AI" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "132f6f55", - "metadata": {}, - "source": [ - "As LLMs grow in popularity and use around the world, the need to manage and monitor their outputs becomes increasingly important. In this challenge, you will learn how to evaluate the outputs of LLMs and how to identify and mitigate potential biases in the model.\n", - "\n", - "Questions you should be able to answer by the end of this challenge:\n", - "- How can you leverage content filtering? \n", - "- What are ways to evaluate truthfulness and reduce hallucinations?\n", - "- How can you identify and mitigate bias in your model?\n", - "\n", - "Sections in this Challenge:\n", - "\n", - "1. Identifying harms and detecting Personal Identifiable Information (PII)\n", - "1. Evaluating truthfulness using Ground Truth Datasets\n", - "1. Evaluating truthfulness using GPT without Ground Truth Datasets\n", - "\n", - "Resources:\n", - "- [Overview of Responsible AI practices for Azure OpenAI models](https://learn.microsoft.com/en-us/legal/cognitive-services/openai/overview)\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "1fdf2ed6", - "metadata": {}, - "source": [ - "## 1. Content filtering, Content Safety, and Personal Identifiable Information (PII) detection" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "1e860826", - "metadata": {}, - "source": [ - "The four stages of the Responsible AI recommendations when using OpenAI are to identify, measure, mitigate, and operate harms. In this section, we will focus on identifying harms.\n", - "\n", - "This step has the goal of identifying potential harms so you can effectively mitigate them. It's important to remember that identifying harms is highly dependent on the context. For example, a model that is used to generate text for a children's book will have different harms than a model that is used to generate text for a news article. Language will also have different meaning in different contexts, so an identification framework should be flexible enough to adapt to various situations.\n", - "\n", - "We present three tools to identifying harms:\n", - "- Azure Cognitive Services Content Filtering\n", - "- Azure AI Content safety\n", - "- PII detection via OpenAI Plug-ins" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "b5a13fe6", - "metadata": {}, - "source": [ - "### 1.1 Azure Cognitive Services Content Filtering\n", - "\n", - "From [Azure documentation](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/concepts/content-filter): \n", - "\n", - " Azure OpenAI Service includes a content management system that works alongside core models to filter content. This system works by running both the input prompt and generated content through an ensemble of classification models aimed at detecting misuse. \n", - "\n", - "You should evaluate all potential harms carefully and add scenario-specific mitigation as needed. For example, you may want to filter out content that is offensive, profane, sexually explicit, or hateful.\n", - "\n", - "**Knowledge Check #1**:\n", - "\n", - "To assess your understanding of the concept of content filtering, answer the following questions based on the documentation:\n", - "\n", - "* True or False: If you make a streaming completions request for multiple responses, the content filter will evaluate each response individually and return only the ones that pass.\n", - "* True or False: the `finish_reason` parameter will be returned on every response from the content filter.\n", - "* True or False: If the content filtering system is down, you will not be able to receive results about your request." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "67c23d99", - "metadata": {}, - "source": [ - "### 1.2 Azure AI Content Safety (Preview)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "c9e0d7a6", - "metadata": {}, - "source": [ - "The [Azure AI Content Safety](https://learn.microsoft.com/en-us/azure/cognitive-services/content-safety/overview) was created to help organizations responsible manage and moderate user- and AI-generated content. It is a managed service that provides a scalable, low-latency, and cost-effective content moderation solution for your image and text content. It is designed to help you detect potentially unsafe content, including hate speech, violence, sexually explicit material, and self-harm.\n", - "\n", - "You can read more about the service in this [Microsoft article](https://techcommunity.microsoft.com/t5/ai-cognitive-services-blog/introducing-azure-ai-content-safety-helping-organizations-to/ba-p/3825744).\n", - "\n", - "**Knowledge Check #2**:\n", - "\n", - "Check your understanding of the AI Content Safety Service by answering the following questions:\n", - "\n", - "* True or False: The Text Moderation API is designed to support over 100 languages as input.\n", - "* True or False: The AI Content Safety Service has a feature to monitor activity statistics of your application.\n", - "* True or False: The Azure AI Content Safety Studio and the API have different risk scores (severity levels) across the categories of harm.\n", - "* True or False: You can only customize severity thresholds through the API.\n", - "* True or False: The API always returns a severity level for all four content categories." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "99f9e49d", - "metadata": {}, - "source": [ - "To run the example, first install some packages and load your environment variables from a `.env` file.\n", - "\n", - "**NOTE:** The openai-python library support for Azure OpenAI is in preview. We have specified the API Preview version below.\n", - "\n", - "`os.getenv()` for the endpoint and key assumes that you are using environment variables." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "220b62a1", - "metadata": { - "gather": { - "logged": 1694716972271 - } - }, - "outputs": [], - "source": [ - "import os\n", - "import openai\n", - "from dotenv import load_dotenv, find_dotenv\n", - "load_dotenv(find_dotenv())\n", - "\n", - "API_KEY = os.getenv(\"OPENAI_API_KEY\")\n", - "assert API_KEY, \"ERROR: Azure OpenAI Key is missing\"\n", - "openai.api_key = API_KEY\n", - "RESOURCE_ENDPOINT = os.getenv(\"OPENAI_API_BASE\",\"\").strip()\n", - "CHAT_MODEL = os.getenv(\"CHAT_MODEL_NAME\")\n", - "assert RESOURCE_ENDPOINT, \"ERROR: Azure OpenAI Endpoint is missing\"\n", - "assert \"openai.azure.com\" in RESOURCE_ENDPOINT.lower(), \"ERROR: Azure OpenAI Endpoint should be in the form: \\n\\n\\t.openai.azure.com\"\n", - "openai.api_base = RESOURCE_ENDPOINT\n", - "openai.api_type = os.environ['OPENAI_API_TYPE']\n", - "CHAT_INSTRUCT_MODEL = os.getenv(\"CHAT_INSTRUCT_MODEL\")\n", - "openai.api_version = \"2023-06-01-preview\" # API version required to test out Annotations preview" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "4fac5b67", - "metadata": {}, - "source": [ - "Below is an example OpenAI call using the Preview version which enables [Annotations](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/concepts/content-filter#annotations-preview). Replace the input prompt with different text to see how the annotations change." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ee9a8bad", - "metadata": { - "gather": { - "logged": 1694716864019 - } - }, - "outputs": [], - "source": [ - "pii_prompt = \"{Example prompt where a severity level of low is detected}\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "55521442", - "metadata": {}, - "outputs": [], - "source": [ - "response = openai.Completion.create(\n", - " engine=CHAT_MODEL,\n", - " prompt=pii_prompt \n", - " # Content that is detected at severity level medium or high is filtered, \n", - " # while content detected at severity level low isn't filtered by the content filters.\n", - ")\n", - "print(response)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "8f314ff6", - "metadata": {}, - "source": [ - "### 1.3 Checking for PII data" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "44894b31", - "metadata": {}, - "source": [ - "Plugins are chat extensions designed specifically for language models like ChatGPT, enabling them to access up-to-date information, run computations, or interact with third-party services in response to a user's request. They unlock a wide range of potential use cases and enhance the capabilities of language models.\n", - "\n", - "The below function, `screen_text_for_pii`, can be helpful if you want to avoid uploading sensitive or private documents to a database unintentionally.\n", - "\n", - "This feature is not foolproof and may not catch all instances of personally identifiable information. Use this feature with caution and verify its effectiveness for your specific use case. You can read more about the background of this function from OpenAI [here](https://github.com/openai/chatgpt-retrieval-plugin/tree/main#plugins).\n", - "\n", - "For other ways to ensure your data is secure when using OpenAI, check out ways to [configure the OpenAI service with managed identities](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/managed-identity).\n", - "\n", - "Read through the function `screen_text_for_pii` in the cell below to understand how it works. You can replace the input text with information relevant to your use case." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ae37954e", - "metadata": { - "gather": { - "logged": 1694716864051 - } - }, - "outputs": [], - "source": [ - "def get_completion_from_messages(messages, model=CHAT_MODEL, temperature=0):\n", - " response = openai.ChatCompletion.create(\n", - " engine=model,\n", - " messages=messages,\n", - " temperature=temperature, # this is the degree of randomness of the model's output\n", - " )\n", - " return response.choices[0].message[\"content\"]\n", - "\n", - "def screen_text_for_pii(text: str) -> bool:\n", - " # This prompt is just an example, change it to fit your use case\n", - " messages = [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": f\"\"\"\n", - " You can only respond with the word \"True\" or \"False\", where your answer indicates whether the text in the user's message contains PII.\n", - " Do not explain your answer, and do not use punctuation.\n", - " Your task is to identify whether the text extracted from your company files\n", - " contains sensitive PII information that should not be shared with the broader company. Here are some things to look out for:\n", - " - An email address that identifies a specific person in either the local-part or the domain\n", - " - The postal address of a private residence (must include at least a street name)\n", - " - The postal address of a public place (must include either a street name or business name)\n", - " - Notes about hiring decisions with mentioned names of candidates. The user will send a document for you to analyze.\n", - " \"\"\",\n", - " },\n", - " {\"role\": \"user\", \"content\": text},\n", - " ]\n", - "\n", - " completion = get_completion_from_messages(messages)\n", - " \n", - " if completion.startswith(\"True\"):\n", - " return True\n", - "\n", - " return False" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "073cfbcb-dad4-47ba-8384-30a5d7a215b2", - "metadata": { - "gather": { - "logged": 1694716864099 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# Optional: test out the screening for PII using input data\n", - "text = \"INPUT YOUR TEXT HERE\"\n", - "screen_text_for_pii(text)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "3cadcf88", - "metadata": {}, - "source": [ - "## 2. Evaluating truthfulness using Ground Truth data\n", - "\n", - "In this section, we will focus on evaluating truthfulness in model outputs. Model hallucinations is a common enough problem in using LLMs that it is important to evaluate whether the model is generating responses based on data rather than making up information. The goal is to improve truthfulness in results to make your model more consistent and reliable for production.\n", - "\n", - "This section will focus on how to evaluate your model when you have access to [Ground Truth](https://en.wikipedia.org/wiki/Ground_truth) data. This will allow us to compare the model's output to the correct answer. In the next section, we will focus on how to evaluate your model when you do not have access to Ground Truth data.\n", - "\n", - "When we use Ground Truth data, we can deduce a numerical representation of how similar the predicted answer is to the correct one using various metrics. You will also have the opportunity to identify and implement additional metrics to evaluate the use case in this section.\n", - "\n", - "We will evaluate models using datasets from Hugging Face as well as Hugging Face's [Evaluate library](https://huggingface.co/docs/evaluate/index).\n", - "\n", - "We will also be utilizing LangChain, which has a package (QAEvalChain) for this specific purpose. [Read more](https://python.langchain.com/en/latest/use_cases/evaluation/question_answering.html) about how Evaluation is implemented by LangChain. You may have heard of LangChain and Semantic Kernel. LangChain is a third-party, open-source framework that you can use to develop applications that are powered by language models. LangChain makes the complexities of working and building with AI models easier by providing the pipeline orchestration framework and helper utilities to run powerful, multiple-model pipelines. It can also be integrated with Prompt Flow to scale prompt engineering workflows.\n", - "\n", - "By the end of this section, you can review which approach (Hugging Face's Evaluate or LangChain's QAEvalChain) is preferable for future use cases." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "0e3ce977", - "metadata": {}, - "source": [ - "### 2.1 Setup\n", - "\n", - "For demonstration purposes, we will evaluate a simple question answering system." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4c10054f", - "metadata": { - "gather": { - "logged": 1694716864133 - } - }, - "outputs": [], - "source": [ - "from langchain.prompts import PromptTemplate\n", - "from langchain.chains import LLMChain\n", - "from langchain.chat_models import AzureChatOpenAI" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "d4cfc9b3", - "metadata": {}, - "source": [ - "Now we'll create a Prompt Template that will allow us to use the same prompt with different inputs. We will utilize [LangChain](https://docs.langchain.com/docs/), an open-source framework for working with language models.\n", - "\n", - "Read more about LangChain Chains and how they work [here](https://docs.langchain.com/docs/components/chains/)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9abdf160", - "metadata": { - "gather": { - "logged": 1694716864167 - } - }, - "outputs": [], - "source": [ - "prompt = PromptTemplate(template=\"Question: {question}\\nAnswer:\", input_variables=[\"question\"])\n", - "llm = AzureChatOpenAI(deployment_name=CHAT_MODEL, temperature=0.9)\n", - "chain = LLMChain(llm=llm, prompt=prompt)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "cbea2132", - "metadata": {}, - "source": [ - "### 2.2 Loading data\n", - "\n", - "Now we load a dataset from Hugging Face, and then convert it to a list of dictionaries for easier usage." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d2373cf1", - "metadata": { - "gather": { - "logged": 1694716864214 - } - }, - "outputs": [], - "source": [ - "from datasets import load_dataset\n", - "dataset = load_dataset(\"truthful_qa\", \"generation\")" - ] - }, - { - "cell_type": "markdown", - "id": "b91f88b3-0b95-4814-bcd0-36527b0b49db", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Let's work with the first five examples in the [Truthful QA dataset from Hugging Face](https://huggingface.co/datasets/truthful_qa). We are working with the \"Generation\" subsection of the dataset because we are applying this to a text-generating system, but notice how there is another subsection for evaluating the model's performance on multiple choice scenarios." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e591ee7f", - "metadata": { - "gather": { - "logged": 1694716864248 - } - }, - "outputs": [], - "source": [ - "num_examples = 3\n", - "examples = list(dataset['validation'])[:num_examples]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0bf71517", - "metadata": { - "gather": { - "logged": 1694716864285 - } - }, - "outputs": [], - "source": [ - "examples[0]" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "b8c3c8ef", - "metadata": {}, - "source": [ - "### 2.3 Predictions\n", - "\n", - "We can now make and inspect the predictions for these questions." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "22b2849c", - "metadata": { - "gather": { - "logged": 1694716864314 - } - }, - "outputs": [], - "source": [ - "predictions = chain.apply(examples)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "35e1d71c", - "metadata": { - "gather": { - "logged": 1694716864346 - } - }, - "outputs": [], - "source": [ - "predictions" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "de420cf5", - "metadata": {}, - "source": [ - "### 2.4 Evaluation\n", - "We can see that if we tried to just do exact match on the answer answers they would not match what the language model answered. However, semantically the language model is correct in both cases. In order to account for this, we can use a language model itself to evaluate the answers.\n", - "\n", - "Because these answers are more complex than multiple choice, we can now evaluate their accuracy using a language model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d6e87e11", - "metadata": { - "gather": { - "logged": 1694716864379 - } - }, - "outputs": [], - "source": [ - "from langchain.evaluation.qa import QAEvalChain" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cfc2e624", - "metadata": { - "gather": { - "logged": 1694716864406 - } - }, - "outputs": [], - "source": [ - "# Create an Evaluation Chain using LangChain's QAEValChain\n", - "eval_chain = QAEvalChain.from_llm(llm)\n", - "graded_outputs = eval_chain.evaluate(examples, predictions, question_key=\"question\", answer_key=\"best_answer\", prediction_key=\"text\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "10238f86", - "metadata": { - "gather": { - "logged": 1694716864440 - } - }, - "outputs": [], - "source": [ - "graded_outputs" - ] - }, - { - "cell_type": "markdown", - "id": "f7d216ad-4ae9-49a9-940c-84fccac2a343", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Now we're going to count the number of outputs that were graded as \"Correct\" or \"Incorrect\" based on the evaluation from the QAEvalChain." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "83e70271", - "metadata": { - "gather": { - "logged": 1694715843342 - } - }, - "outputs": [], - "source": [ - "num_correct = sum([1 for x in graded_outputs if str(x['results']).upper().startswith('CORRECT')])\n", - "num_incorrect = sum([1 for x in graded_outputs if str(x['results']).upper().startswith('INCORRECT')])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "386764e3", - "metadata": { - "gather": { - "logged": 1694715843378 - } - }, - "outputs": [], - "source": [ - "print(num_correct, num_incorrect)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "e6dc737a", - "metadata": {}, - "source": [ - "### 2.5 Comparing to other evaluation metrics\n", - "\n", - "We can compare the evaluation results we get to other common evaluation metrics. To do this, let’s load some evaluation metrics from HuggingFace’s Evaluate package." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dd5f575e", - "metadata": { - "gather": { - "logged": 1694715843415 - } - }, - "outputs": [], - "source": [ - "print(examples[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "207be70d", - "metadata": { - "gather": { - "logged": 1694715843446 - } - }, - "outputs": [], - "source": [ - "# Some data munging to get the examples in the right format\n", - "for i, eg in enumerate(examples):\n", - " eg['id'] = str(i)\n", - " eg['answers'] = {\"text\": eg['correct_answers'], \"answer_start\": [0]}\n", - " predictions[i]['id'] = str(i)\n", - " predictions[i]['prediction_text'] = predictions[i]['text']\n", - "\n", - "for p in predictions:\n", - " del p['text']\n", - "\n", - "# references need id, answers as list with text and answer_start\n", - "new_examples = examples.copy()\n", - "# print(new_examples)\n", - "for eg in new_examples:\n", - " del eg ['question']\n", - " del eg['best_answer']\n", - " del eg['type']\n", - " del eg['correct_answers']\n", - " del eg['category']\n", - " del eg['incorrect_answers']\n", - " del eg['source']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9d7ef7ec-d7f9-4626-b4a9-12d002737796", - "metadata": { - "gather": { - "logged": 1694715843488 - } - }, - "outputs": [], - "source": [ - "from evaluate import load\n", - "squad_metric = load(\"squad\")\n", - "results = squad_metric.compute(\n", - " references=new_examples,\n", - " predictions=predictions,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2a2ab1dc", - "metadata": { - "gather": { - "logged": 1694715843520 - } - }, - "outputs": [], - "source": [ - "results" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "4df686a3", - "metadata": {}, - "source": [ - "#### (Optional) Student Task\n", - "\n", - "Now add two additional metrics to evaluate the model using the Hugging Face Evaluate library. One of those could be the BERT_score metric.\n", - "\n", - "Resources for reference:\n", - "\n", - "* [Hugging Face's Evaluate Library on GitHub](https://github.com/huggingface/evaluate) \n", - "* [Evaluate Library Documentation](https://huggingface.co/docs/transformers/tasks/translation#evaluate) \n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2424f5f7", - "metadata": { - "gather": { - "logged": 1694715843567 - } - }, - "outputs": [], - "source": [ - "### STUDENT TASK ###" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "f88f1a76", - "metadata": {}, - "source": [ - "## 3. Evaluating Models for Truthfulness using GPT without Ground Truth Datasets" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "767bceab", - "metadata": {}, - "source": [ - "You won't always have Ground Truth data available to assess your model. Luckily, GPT does a really good job at generating Ground Truth data from your original dataset.\n", - "\n", - "Research has shown that LLMs such as GPT-3 and ChatGPT are good at assessing text inconsistency. Based on these findings, the models can be used to evaluate sentences for truthfulness by prompting GPT. Let's assess the accuracy of GPT through a technique of GPT evaluating itself.\n", - "\n", - "In this section, we will evaluate the model you worked on in the previous challenge applied to the CNN Dailymail dataset." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2b7e21f7", - "metadata": {}, - "outputs": [], - "source": [ - "from langchain.chains import LLMChain, QAGenerationChain\n", - "from langchain.requests import Requests\n", - "from langchain.llms import AzureOpenAI\n", - "from langchain.document_loaders import TextLoader\n", - "import pandas as pd\n", - "import json" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "5a19c79e", - "metadata": {}, - "source": [ - "### 3.1. Create a Ground Truth Dataset on Custom Data\n", - "Let's start by using GPT to create a dataset of question-answer pairs as our \"ground-truth\" data from the CNN Dailymail dataset from the previous challenge." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9681f823", - "metadata": {}, - "outputs": [], - "source": [ - "# Load the provided CNN file, the path of which may change based on folder structure\n", - "CNN_FILE_PATH = \"../data/structured/cnn_dailymail_data.csv\"\n", - "\n", - "# Optional: limit to 11 samples for simple scope to avoid RateLimitErrors\n", - "# You are welcome to change `num_samples` or delete it to run this example on\n", - "# the entire dataset but doing so may take 1+ hour\n", - "num_samples = 11\n", - "df = pd.read_csv(CNN_FILE_PATH)[:num_samples]\n", - "df.drop([4,9], axis=0, inplace=True)\n", - "df = df.drop(columns=[\"highlights\"])\n", - "pd.set_option('display.max_colwidth', None) # Show all columns" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bc9ad597", - "metadata": {}, - "outputs": [], - "source": [ - "# Take a look at the data\n", - "df.head(3)" - ] - }, - { - "cell_type": "markdown", - "id": "b447a9b6-3f2d-4bf9-84a5-4f269c78b45a", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Time for some data scrubbing for consistency." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6599f2f6", - "metadata": {}, - "outputs": [], - "source": [ - "# Convert the column \"article\" to a list of dictionaries\n", - "df_copy = df.copy().rename(columns={\"article\": \"text\"})\n", - "df_copy = df_copy.drop(columns=[\"id\"])\n", - "df_dict = df_copy.to_dict(\"records\")\n", - "\n", - "print(df_dict)" - ] - }, - { - "cell_type": "markdown", - "id": "1cf8a1f8-369f-4fb4-909c-600e1c5102d3", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "We've gone ahead and generated a question-answer pair for each article. This will help us assess GPT's performance on how well it answers the test questions. The answers in each pairing are considered our ground truth data and the ideal answer.\n", - "\n", - "We created these pairs using Langchain's [QAGenerationChain](https://api.python.langchain.com/en/latest/chains/langchain.chains.qa_generation.base.QAGenerationChain.html#). Check out the [source code](https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/chains/qa_generation) to see how the question-answer pairs are being generated through QAGenerationChain. The implementation may surprise you!\n", - "\n", - "In the process, we removed articles that triggered the OpenAI content filter. \n", - "\n", - "Below, we're going to load the provided question-answer dataset for later assessment." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "af36819b", - "metadata": {}, - "outputs": [], - "source": [ - "llm = AzureOpenAI(deployment_name=CHAT_MODEL, temperature=0, max_tokens=1000)\n", - "chain = QAGenerationChain.from_llm(llm)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8dc5ba51", - "metadata": {}, - "outputs": [], - "source": [ - "# Load cnn_qa_set.json\n", - "cnn_qa_set_filepath = '../data/structured/cnn_qa_set.json'\n", - "with open(cnn_qa_set_filepath, 'r') as file:\n", - " qa_set = json.load(file)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8b597401-c969-454f-a7ee-6a91195239f7", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - }, - "vscode": { - "languageId": "plaintext" - } - }, - "outputs": [], - "source": [ - "qa_set[:3]" - ] - }, - { - "cell_type": "markdown", - "id": "fd04f598", - "metadata": {}, - "source": [ - "Now we have the question and Ground Truth answers. Let's test the GPT + Cognitive Search solution you implemented in the last challenge! We are going to compare the differences between `truth_answers` (provided answers) and `prompt_answers` (model performance)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c06d9ab0", - "metadata": {}, - "outputs": [], - "source": [ - "questions = [(set[\"question\"] for set in qa_set)]\n", - "truth_answers = [(set[\"answers\"] for set in qa_set)]\n", - "prompt_answers = list()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "8c9127ee", - "metadata": {}, - "source": [ - "### 3.2 Instantiate the Cognitive Search Index\n", - "\n", - "We're using the Index you created in the last challenge to retrieve documents that are relevant to any input user query." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1c97e90f", - "metadata": {}, - "outputs": [], - "source": [ - "import os, json, requests, sys, re\n", - "import requests\n", - "from pprint import pprint\n", - "import pandas as pd\n", - "from azure.core.credentials import AzureKeyCredential\n", - "from azure.search.documents.indexes import SearchIndexClient \n", - "from azure.search.documents import SearchClient\n", - "from azure.search.documents.indexes.models import (\n", - " SearchIndex,\n", - " SearchField,\n", - " SearchFieldDataType,\n", - " SimpleField,\n", - " SearchableField,\n", - " SemanticConfiguration,\n", - " PrioritizedFields,\n", - " SemanticField,\n", - " SemanticSettings\n", - ")\n", - "\n", - "import numpy as np\n", - "from openai.embeddings_utils import get_embedding, cosine_similarity" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a38c0644", - "metadata": {}, - "outputs": [], - "source": [ - "# Create an SDK client\n", - "service_endpoint = os.getenv(\"AZURE_COGNITIVE_SEARCH_ENDPOINT\") \n", - "key = os.getenv(\"AZURE_COGNITIVE_SEARCH_KEY\")\n", - "credential = AzureKeyCredential(key)\n", - "index_name = os.getenv(\"AZURE_COGNITIVE_SEARCH_INDEX_NAME\")\n", - "\n", - "index_client = SearchIndexClient(\n", - " endpoint=service_endpoint, credential=credential)\n", - "search_client = SearchClient(endpoint=service_endpoint, index_name=index_name, credential=credential)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1f0501d0", - "metadata": {}, - "outputs": [], - "source": [ - "# Create a pandas dataframe with columns from qa_set\n", - "pd.set_option('display.max_colwidth', None)\n", - "df = pd.DataFrame(qa_set)\n", - "df = df.rename(columns={\"answer\": \"truth_answer\"})\n", - "df.head(3)" - ] - }, - { - "cell_type": "markdown", - "id": "e794a118-188b-4e70-b0f0-3376146d077c", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Let's retrieve the relevant articles for each question in our qa_set dataframe." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "93f48c97", - "metadata": {}, - "outputs": [], - "source": [ - "# Get the articles for the search terms\n", - "# Optional: change `num_docs` to change how many relevant ranked documents the Search index should return\n", - "num_docs=1\n", - "for i, row in df.iterrows():\n", - " search_term = row['question']\n", - " results = search_client.search(search_text=search_term, include_total_count=num_docs)\n", - " df.loc[i, \"context\"] = next(results)['article']\n", - "df.head(3)" - ] - }, - { - "cell_type": "markdown", - "id": "4c172dfb-a2fa-48b2-b723-96d6d521d7e8", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Using a prompt template, we can feed questions into GPT using the information from the retrieved documents.\n", - "\n", - "Notice which model we're now using to generate answers. Why might this be? What happens if you used the chat model we've used earlier?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "31713b2b", - "metadata": {}, - "outputs": [], - "source": [ - "from langchain.prompts import PromptTemplate\n", - "\n", - "# Ask the model using the embeddings from Challenges 3 and 4 to answer the questions\n", - "template = \"\"\"You are a search assistant trying to answer the following question. Use only the context given. Your answer should only be one sentence.\n", - "\n", - " > Question: {question}\n", - " \n", - " > Context: {context}\"\"\"\n", - "\n", - "# Create a prompt template\n", - "prompt = PromptTemplate(template=template, input_variables=[\"question\", \"context\"])\n", - "llm = AzureOpenAI(deployment_name=CHAT_INSTRUCT_MODEL, temperature=0)\n", - "search_chain = LLMChain(llm=llm, prompt=prompt, verbose=False)\n", - "\n", - "prompt_answers = []\n", - "for question, context in list(zip(df.question, df.context)):\n", - " response = search_chain.run(question=question, context=context)\n", - " prompt_answers.append(response.replace('\\n',''))\n", - "df['prompt_answer'] = prompt_answers " - ] - }, - { - "cell_type": "markdown", - "id": "252b880b", - "metadata": {}, - "source": [ - "Examine the first three answers from the model based on the articles. How could you utilize Prompt Engineering techniques to refine the answers?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "569b27de", - "metadata": {}, - "outputs": [], - "source": [ - "df['prompt_answer'].head(3)" - ] - }, - { - "cell_type": "markdown", - "id": "9e340641-3ffd-4ea0-a35d-5a684b17d40d", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "After generating responses to our test questions, we can use GPT (can be another model if you would like, such as GPT 4) to evaluate the correctness to our Ground Truth answers using a rubric.\n", - "\n", - "Notice how the prompt is using techniques you learned from Challenges 1 and 2." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f0583718", - "metadata": {}, - "outputs": [], - "source": [ - "eval_template = \"\"\"You are trying to answer the following question from the context provided:\n", - "\n", - "> Question: {question}\n", - "\n", - "The correct answer is:\n", - "\n", - "> Query: {truth_answer}\n", - "\n", - "Is the following predicted query semantically the same (eg likely to produce the same answer)?\n", - "\n", - "> Predicted Query: {prompt_answer}\n", - "\n", - "Please give the Predicted Query a grade of either an A, B, C, D, or F, along with an explanation of why. End the evaluation with 'Final Grade: '\n", - "\n", - "> Explanation: Let's think step by step.\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eba357c3", - "metadata": {}, - "outputs": [], - "source": [ - "eval_prompt = PromptTemplate(template=eval_template, input_variables=[\"question\", \"truth_answer\", \"prompt_answer\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e8062ada", - "metadata": {}, - "outputs": [], - "source": [ - "# Create a new LLM Chain to submit the prompt we created\n", - "eval_chain = LLMChain(llm=llm, prompt=eval_prompt, verbose=False)\n", - "\n", - "# Submit the prompt using our dataset\n", - "eval_results = []\n", - "for question, truth_answer, prompt_answer in list(zip(df.question, df.truth_answer, df.prompt_answer)):\n", - " eval_output = eval_chain.run(\n", - " question=question,\n", - " truth_answer=truth_answer,\n", - " prompt_answer=prompt_answer,\n", - " )\n", - " eval_results.append(eval_output)\n", - "eval_results" - ] - }, - { - "cell_type": "markdown", - "id": "d0dabe6a-659a-44ca-bc82-ac8793f713ef", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Now let's parse the rubric results in order to quantify and summarize them in aggregate." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8449fb78", - "metadata": {}, - "outputs": [], - "source": [ - "import re\n", - "from typing import List\n", - "from collections import defaultdict\n", - "\n", - "# Parse the evaluation chain responses into a rubric\n", - "def parse_eval_results(results: List[str]) -> List[float]:\n", - " rubric = {\n", - " \"A\": 1.0,\n", - " \"B\": 0.75,\n", - " \"C\": 0.5,\n", - " \"D\": 0.25,\n", - " \"F\": 0\n", - " }\n", - " return [rubric[re.search(r'Final Grade: (\\w+)', res).group(1)] for res in results]\n", - "\n", - "scores = defaultdict(list)\n", - "parsed_results = parse_eval_results(eval_results)\n", - "\n", - "# Collect the scores for a final evaluation table\n", - "scores['request_synthesizer'].extend(parsed_results)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e2090296", - "metadata": {}, - "outputs": [], - "source": [ - "# Reusing the rubric from above, parse the evaluation chain responses\n", - "parsed_eval_results = parse_eval_results(eval_results)\n", - "# Collect the scores for a final evaluation table\n", - "scores['result_synthesizer'].extend(parsed_eval_results)\n", - "\n", - "# Print out Score statistics for the evaluation session\n", - "header = \"{:<20}\\t{:<10}\\t{:<10}\\t{:<10}\".format(\"Metric\", \"Min\", \"Mean\", \"Max\")\n", - "print(header)\n", - "for metric, metric_scores in scores.items():\n", - " mean_scores = sum(metric_scores) / len(metric_scores) if len(metric_scores) > 0 else float('nan')\n", - " row = \"{:<20}\\t{:<10.2f}\\t{:<10.2f}\\t{:<10.2f}\".format(metric, min(metric_scores), mean_scores, max(metric_scores))\n", - " print(row)" - ] - }, - { - "cell_type": "markdown", - "id": "a055d128-672d-4bd7-83cf-544ad9b6a803", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "There you have it! We can now review the results of evaluating the model in conjunction with Azure Cognitive Search from the last challenge. You can perform a similar analysis on your use case and custom data." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "0b743a96", - "metadata": {}, - "source": [ - "## Conclusion" - ] - }, - { - "cell_type": "markdown", - "id": "461ee50f-c2c4-4a80-bc0e-68fad17ee380", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "In this challenge, we covered the principles of Responsible AI, particularly when working with OpenAI, and how to evaluate the performance of a model implementation using Ground Truth data.\n", - "\n", - "We introduced you to several tools and services, some from Azure and others that are Open-Source. You can refer to them for your own projects to decide which works best for your scenarios." - ] - }, - { - "cell_type": "markdown", - "id": "718b928c-6041-4aba-9d65-23b493fbf84c", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "**Knowledge Check #1 Answers**:\n", - "* True\n", - "* False - it will be returned if it was not deemed inappropriate\n", - "* False - your request will still complete without content filtering. You can see if it wasn't applied by looking for an error message in the `content_filter_result` object." - ] - }, - { - "cell_type": "markdown", - "id": "8a9fe7e6-4a36-4627-9df5-fe936f9dd93b", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "**Knowledge Check #2 Answers**:\n", - "* False: the service was trained on more than 100 languages but is designed to support only a handful.\n", - "* True: Content Safety has a monitoring page to help you track you moderation API performance and trends to inform your content moderation strategy.\n", - "* True: The Studio uses four levels of risk, whereas the API scores the risk on a scale of 0 to 6.\n", - "* False: You can also customize severity thresholds in the Studio.\n", - "* False: You can specify which categories you want to assess your text on in the API using the `categories` parameter." - ] - } - ], - "metadata": { - "kernel_info": { - "name": "python3" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - }, - "microsoft": { - "ms_spell_check": { - "ms_spell_check_language": "en" - } - }, - "nteract": { - "version": "nteract-front-end@1.0.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From e10bcdfc6c1129db3095ceb3af0a5864aa142057 Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Fri, 16 May 2025 13:18:48 -0500 Subject: [PATCH 05/24] Update Solution-00.md Changed how to handle issues with local.settings.json --- 068-AzureOpenAIApps/Coach/Solution-00.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/068-AzureOpenAIApps/Coach/Solution-00.md b/068-AzureOpenAIApps/Coach/Solution-00.md index 5ea9f341f6..edc19c0bcc 100644 --- a/068-AzureOpenAIApps/Coach/Solution-00.md +++ b/068-AzureOpenAIApps/Coach/Solution-00.md @@ -67,7 +67,9 @@ This information is subject to change over time, for the most up to date list of ### Deployment -Make sure the student uses the terminal window in GitHub Codespaces or the local workstation and **NOT** Azure Cloud Shell to do the deployment. If the student does the deployment from the Azure Cloud Shell by cloning the repo to the Cloud Shell and then running deployment script there, the `local.settings.json` file generated by the deployment script will be in the wrong place! The student will either have to redeploy FROM their Codespace/Local Workstation or copy the `local.settings.json` for the `ContosoAIBackend` to their Codespace or local workstation manually FROM the Azure Cloud Shell if it ended up there. +Make sure the student uses the terminal window in GitHub Codespaces or the local workstation and **NOT** Azure Cloud Shell to do the deployment. If the student does the deployment from the Azure Cloud Shell by cloning the repo to the Cloud Shell and then running deployment script there, the `local.settings.json` file generated by the deployment script will be in the wrong place! The student can run `genlocalsettings.sh` with the name of the resource group to create the local.settings.json in the correct environment. + +Also, if a student edits the local.settings.json and accidentally removes a line or changes a value to an incorrect setting, they can re-run the `genlocalsettings.sh` script again to fix the problem. ### Codespaces vs Local Workstation From d4c1d169f88751e3788fca69dc7b228a2c1652e8 Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Thu, 22 May 2025 15:07:32 -0500 Subject: [PATCH 06/24] Update Challenge-00-lab.md Updated local machine instructions --- .../Student/Challenge-00-lab.md | 30 ++++--------------- 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/068-AzureOpenAIApps/Student/Challenge-00-lab.md b/068-AzureOpenAIApps/Student/Challenge-00-lab.md index 82a355ce94..d7b631c838 100644 --- a/068-AzureOpenAIApps/Student/Challenge-00-lab.md +++ b/068-AzureOpenAIApps/Student/Challenge-00-lab.md @@ -80,31 +80,13 @@ If you want to setup your environment on your local workstation, expand the sect
Click to expand/collapse Local Workstation Requirements -To work on your local workstation, please ensure you have the following tools and resources before hacking: -- [Windows Subsystem for Linux](../../000-HowToHack/WTH-Common-Prerequisites.md#windows-subsystem-for-linux) -- [Managing Cloud Resources](../../000-HowToHack/WTH-Common-Prerequisites.md#managing-cloud-resources) - - [Azure Portal](../../000-HowToHack/WTH-Common-Prerequisites.md#azure-portal) - - [Azure CLI](../../000-HowToHack/WTH-Common-Prerequisites.md#azure-cli) - - [Note for Windows Users](../../000-HowToHack/WTH-Common-Prerequisites.md#note-for-windows-users) - - [Azure PowerShell CmdLets](../../000-HowToHack/WTH-Common-Prerequisites.md#azure-powershell-cmdlets) - - [Azure Cloud Shell](../../000-HowToHack/WTH-Common-Prerequisites.md#azure-cloud-shell) -- [Visual Studio Code](../../000-HowToHack/WTH-Common-Prerequisites.md#visual-studio-code) - -**NOTE:** For Windows users, we recommend that the following tools be installed in your WSL environment, and NOT on Windows itself. (This includes the Azure CLI itself, which is listed above.) - -- [Node v20.11.0](https://nodejs.org/en/download) - Only v20.11.0 -- Make sure [NPM 10.2.4](https://nodejs.org/en/download) - Comes with Node Installation -- Install [Angular CLI](https://angular.io/cli#installing-angular-cli) globally -- Install the [Azure Functions Core Tools](https://www.npmjs.com/package/azure-functions-core-tools#installing) V4 Globally Using NPM -- Install [Python 3.11](https://www.python.org/downloads/) -- Install [Python Package Install PIP](https://pypi.org/project/pip/) -- Create a Python virtual environment for Python 3.11
- ```bash - python3.11 -m venv .venv - source .venv/bin/activate - python3.11 -m pip install --upgrade pip - ``` +On Mac OS (Note: only tested on Apple Silicon): +- Download and install Docker Desktop +- Clone the Codespace's GitHub repo locally on your Mac +- Open the root folder for the repo in Visual Studio Code +- You might get prompted to re-open the container in a Dev Container. You can do that by clicking the Yes button, but if you miss it or hit no, you can also use the Command Palette in VS Code and select Dev Containers: Reopen in Container +WSL on Windows ARM64 does not currently support running the Azure Function Core Tools in x86_64 emulation using QEMU so you will need to use a Codespace instead #### Student Resources From 02b3295a78a935982ea8ffda026114463bf42c9c Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Thu, 22 May 2025 15:16:15 -0500 Subject: [PATCH 07/24] Update Solution-00.md Moved local workstation direct instructions from student guide to here --- 068-AzureOpenAIApps/Coach/Solution-00.md | 30 +++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/068-AzureOpenAIApps/Coach/Solution-00.md b/068-AzureOpenAIApps/Coach/Solution-00.md index edc19c0bcc..cff6c63a67 100644 --- a/068-AzureOpenAIApps/Coach/Solution-00.md +++ b/068-AzureOpenAIApps/Coach/Solution-00.md @@ -73,12 +73,30 @@ Also, if a student edits the local.settings.json and accidentally removes a line ### Codespaces vs Local Workstation -We **strongly** recommend students use GitHub Codespaces as their development environment over a local workstation. - ->[!NOTE] ->As of July 2024, the Challenge 0 instructions for how to set up a local workstation have NOT been tested! - -Students should avoid doing the local workstation setup because there is the potential to adversely affect their local workstation (especially if they accidentally change the default Python version on Linux/Mac/WSL). There can be a lot of variations in terms of the student's OS version, already installed software packages like Python, Node, etc. that may cause them to lose time trying to get their environment working. +We recommend students use GitHub Codespaces as their development environment over a local workstation and is the only option if a student does not have admin privileges on their machine. + +Running a local Devcontainer will require Docker Desktop to be installed. + +Students should avoid doing the local workstation setup natively in their OS (not in a container) because there is the potential to adversely affect their local workstation (especially if they accidentally change the default Python version on Linux/Mac/WSL). There can be a lot of variations in terms of the student's OS version, already installed software packages like Python, Node, etc. that may cause them to lose time trying to get their environment working. If they insist, they would need these software packages to be installed: + +- [Windows Subsystem for Linux](../../000-HowToHack/WTH-Common-Prerequisites.md#windows-subsystem-for-linux) (for Windows users) +- [Azure CLI](../../000-HowToHack/WTH-Common-Prerequisites.md#azure-cli) +- [Visual Studio Code](../../000-HowToHack/WTH-Common-Prerequisites.md#visual-studio-code) + +**NOTE:** For Windows users, we recommend that the following tools be installed in their WSL environment, and NOT on Windows itself. (This includes the Azure CLI itself, which is listed above.) + +- [Node v20.11.0](https://nodejs.org/en/download) - Only v20.11.0 +- Make sure [NPM 10.2.4](https://nodejs.org/en/download) - Comes with Node Installation +- Install [Angular CLI](https://angular.io/cli#installing-angular-cli) globally +- Install the [Azure Functions Core Tools](https://www.npmjs.com/package/azure-functions-core-tools#installing) V4 Globally Using NPM +- Install [Python 3.11](https://www.python.org/downloads/) +- Install [Python Package Install PIP](https://pypi.org/project/pip/) +- Create a Python virtual environment for Python 3.11
+ ```bash + python3.11 -m venv .venv + source .venv/bin/activate + python3.11 -m pip install --upgrade pip + ``` #### CORS Error When Running the Front End App From 15556cdca523e23a14f2bcaf598c1b8787110d28 Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Thu, 22 May 2025 15:17:42 -0500 Subject: [PATCH 08/24] Update Challenge-00-nolab.md Updated local workstation instructions --- .../Student/Challenge-00-nolab.md | 31 +++++-------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/068-AzureOpenAIApps/Student/Challenge-00-nolab.md b/068-AzureOpenAIApps/Student/Challenge-00-nolab.md index d1dcc4dd4d..2e291d8132 100644 --- a/068-AzureOpenAIApps/Student/Challenge-00-nolab.md +++ b/068-AzureOpenAIApps/Student/Challenge-00-nolab.md @@ -78,30 +78,13 @@ If you want to setup your environment on your local workstation, expand the sect
Click to expand/collapse Local Workstation Requirements -To work on your local workstation, please ensure you have the following tools and resources before hacking: -- [Windows Subsystem for Linux](../../000-HowToHack/WTH-Common-Prerequisites.md#windows-subsystem-for-linux) -- [Managing Cloud Resources](../../000-HowToHack/WTH-Common-Prerequisites.md#managing-cloud-resources) - - [Azure Portal](../../000-HowToHack/WTH-Common-Prerequisites.md#azure-portal) - - [Azure CLI](../../000-HowToHack/WTH-Common-Prerequisites.md#azure-cli) - - [Note for Windows Users](../../000-HowToHack/WTH-Common-Prerequisites.md#note-for-windows-users) - - [Azure PowerShell CmdLets](../../000-HowToHack/WTH-Common-Prerequisites.md#azure-powershell-cmdlets) - - [Azure Cloud Shell](../../000-HowToHack/WTH-Common-Prerequisites.md#azure-cloud-shell) -- [Visual Studio Code](../../000-HowToHack/WTH-Common-Prerequisites.md#visual-studio-code) - -**NOTE:** For Windows users, we recommend that the following tools be installed in your WSL environment, and NOT on Windows itself. (This includes the Azure CLI itself, which is listed above.) - -- [Node v20.11.0](https://nodejs.org/en/download) - Only v20.11.0 -- Make sure [NPM 10.2.4](https://nodejs.org/en/download) - Comes with Node Installation -- Install [Angular CLI](https://angular.io/cli#installing-angular-cli) globally -- Install the [Azure Functions Core Tools](https://www.npmjs.com/package/azure-functions-core-tools#installing) V4 Globally Using NPM -- Install [Python 3.11](https://www.python.org/downloads/) -- Install [Python Package Install PIP](https://pypi.org/project/pip/) -- Create a Python virtual environment for Python 3.11
- ```bash - python3.11 -m venv .venv - source .venv/bin/activate - python3.11 -m pip install --upgrade pip - ``` +On Mac OS (Note: only tested on Apple Silicon): +- Download and install Docker Desktop +- Clone the Codespace's GitHub repo locally on your Mac +- Open the root folder for the repo in Visual Studio Code +- You might get prompted to re-open the container in a Dev Container. You can do that by clicking the Yes button, but if you miss it or hit no, you can also use the Command Palette in VS Code and select Dev Containers: Reopen in Container + +WSL on Windows ARM64 does not currently support running the Azure Function Core Tools in x86_64 emulation using QEMU so you will need to use a Codespace instead #### Student Resources From 93878bd61f4e5799f4083c48e7885e3c32f19951 Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Thu, 22 May 2025 15:18:34 -0500 Subject: [PATCH 09/24] Update Challenge-00-nolab.md --- 068-AzureOpenAIApps/Student/Challenge-00-nolab.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/068-AzureOpenAIApps/Student/Challenge-00-nolab.md b/068-AzureOpenAIApps/Student/Challenge-00-nolab.md index 2e291d8132..7b85a2638e 100644 --- a/068-AzureOpenAIApps/Student/Challenge-00-nolab.md +++ b/068-AzureOpenAIApps/Student/Challenge-00-nolab.md @@ -78,7 +78,7 @@ If you want to setup your environment on your local workstation, expand the sect
Click to expand/collapse Local Workstation Requirements -On Mac OS (Note: only tested on Apple Silicon): +On Mac OS (**NOTE:**: only tested on Apple Silicon): - Download and install Docker Desktop - Clone the Codespace's GitHub repo locally on your Mac - Open the root folder for the repo in Visual Studio Code From 65bdb028055dbc4fe57dad93390499809a9fe33a Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Thu, 22 May 2025 15:18:52 -0500 Subject: [PATCH 10/24] Update Challenge-00-lab.md --- 068-AzureOpenAIApps/Student/Challenge-00-lab.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/068-AzureOpenAIApps/Student/Challenge-00-lab.md b/068-AzureOpenAIApps/Student/Challenge-00-lab.md index d7b631c838..207902f103 100644 --- a/068-AzureOpenAIApps/Student/Challenge-00-lab.md +++ b/068-AzureOpenAIApps/Student/Challenge-00-lab.md @@ -80,7 +80,7 @@ If you want to setup your environment on your local workstation, expand the sect
Click to expand/collapse Local Workstation Requirements -On Mac OS (Note: only tested on Apple Silicon): +On Mac OS (**NOTE:**: only tested on Apple Silicon): - Download and install Docker Desktop - Clone the Codespace's GitHub repo locally on your Mac - Open the root folder for the repo in Visual Studio Code From 67d9e15d825000e5786c2efd9992eaa44e731ed8 Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Thu, 22 May 2025 15:19:17 -0500 Subject: [PATCH 11/24] Update Challenge-00-lab.md --- 068-AzureOpenAIApps/Student/Challenge-00-lab.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/068-AzureOpenAIApps/Student/Challenge-00-lab.md b/068-AzureOpenAIApps/Student/Challenge-00-lab.md index 207902f103..9414dbb38f 100644 --- a/068-AzureOpenAIApps/Student/Challenge-00-lab.md +++ b/068-AzureOpenAIApps/Student/Challenge-00-lab.md @@ -80,7 +80,7 @@ If you want to setup your environment on your local workstation, expand the sect
Click to expand/collapse Local Workstation Requirements -On Mac OS (**NOTE:**: only tested on Apple Silicon): +On Mac OS (**NOTE:** only tested on Apple Silicon): - Download and install Docker Desktop - Clone the Codespace's GitHub repo locally on your Mac - Open the root folder for the repo in Visual Studio Code From 8150fdb88dba51c5321e4f7ed0d8326e808eb1bc Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Thu, 22 May 2025 15:19:43 -0500 Subject: [PATCH 12/24] Update Challenge-00-nolab.md --- 068-AzureOpenAIApps/Student/Challenge-00-nolab.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/068-AzureOpenAIApps/Student/Challenge-00-nolab.md b/068-AzureOpenAIApps/Student/Challenge-00-nolab.md index 7b85a2638e..81c3889a97 100644 --- a/068-AzureOpenAIApps/Student/Challenge-00-nolab.md +++ b/068-AzureOpenAIApps/Student/Challenge-00-nolab.md @@ -78,7 +78,7 @@ If you want to setup your environment on your local workstation, expand the sect
Click to expand/collapse Local Workstation Requirements -On Mac OS (**NOTE:**: only tested on Apple Silicon): +On Mac OS (**NOTE:** only tested on Apple Silicon): - Download and install Docker Desktop - Clone the Codespace's GitHub repo locally on your Mac - Open the root folder for the repo in Visual Studio Code From dac3236fe653e0fedd4152161e7375919613f725 Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Tue, 27 May 2025 15:21:26 -0500 Subject: [PATCH 13/24] Update Challenge-02.md --- 068-AzureOpenAIApps/Student/Challenge-02.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/068-AzureOpenAIApps/Student/Challenge-02.md b/068-AzureOpenAIApps/Student/Challenge-02.md index a64028a49c..17f826f8d0 100644 --- a/068-AzureOpenAIApps/Student/Challenge-02.md +++ b/068-AzureOpenAIApps/Student/Challenge-02.md @@ -53,9 +53,9 @@ In this challenge, you will be asked to configure the system message and tools u #### Configuring Your Virtual Assistants - In your `/ContosoAIAppsBackend` folder there is an `/assistant_configurations` folder that contains two files: one json and one text file + In your `/ContosoAIAppsBackend` folder there is an `/assistant_configurations` folder that contains two files: one JSON and one text file. - The text file (`.txt`) shares the same name as the AI assistant and this is where you enter the system message instructing the AI assistant how it should behave. + The text file (`.txt`) shares the same name as the AI assistant and this is where you enter the system message instructing the AI assistant how it should behave. You should be modifying each text file using the description of what each assistant does (hint: they are each described above). The JSON file (`.json`) share the same name as the AI assistant and this is where we define all the tools that the AI assistant is going to use when interacting with the users. @@ -175,8 +175,8 @@ Once you have proved the backend is responding properly using the REST Client, y To complete the challenge successfully, the solution should demonstrate the following: - Ensure that the application is able to handle the natural language inputs from the customer -- Configure the assistants that receives that question/query from the customer and responses. -- The assistant should be able to keep track of the conversation history during the session and remember important details about the customer making the inquiry or reservation. This should be true for at least the last 3 to 5 sentences input from the customer. +- Configure the assistants that receives that question/query from the customer and responses +- The assistant should be able to keep track of the conversation history during the session and remember important details about the customer making the inquiry or reservation. This should be true for at least the last 3 to 5 sentences input from the customer - The virtual assistant should be able to handle natural language inputs and be trained to understand different variations of the questions related to the tour status. - It should also be able to all the scenarios identified as capabilities for each assistant - For read/write scenarios, the changes requested by the customer/user should be captured and saved correctly to the databases. From 32d6141612dc564ed4b35bc4062be5b0a0982444 Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Wed, 28 May 2025 11:37:14 -0500 Subject: [PATCH 14/24] Update Challenge-03.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit modified the query the assistants portion to ask the student to modify the system prompt to avoid "One moment, please. I’ll retrieve the details for you now." --- 068-AzureOpenAIApps/Student/Challenge-03.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/068-AzureOpenAIApps/Student/Challenge-03.md b/068-AzureOpenAIApps/Student/Challenge-03.md index fd2e8a2e70..4f11ba8c88 100644 --- a/068-AzureOpenAIApps/Student/Challenge-03.md +++ b/068-AzureOpenAIApps/Student/Challenge-03.md @@ -168,6 +168,8 @@ Once you have verified that these documents have been parsed and the data has be - Murphy: answers questions about exams, grades and exam submissions from students. - Priscilla: answers questions about things to do on Contoso Islands as well as make recommendations to guests based on their activity preferences. +When working with the assistants, you may get a response that says something like "One moment, please. I’ll retrieve the details for you now". If that happens, you can try your prompt again. However, a better way would be to change the system prompt so that doesn't happen. Can you think of a way to modify the system prompt to prevent that? Alternatively, ask your coach for a system prompt that avoids you having to wait if you are running short on time. + ## Success Criteria During this challenge you will: From 5d9eccf1da066cd7322f87ffc5e607342c818d46 Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Wed, 28 May 2025 11:40:17 -0500 Subject: [PATCH 15/24] Update Solution-03.md added a sample system prompt for a student to use to avoid getting a "one moment please" message --- 068-AzureOpenAIApps/Coach/Solution-03.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/068-AzureOpenAIApps/Coach/Solution-03.md b/068-AzureOpenAIApps/Coach/Solution-03.md index f1bc3071d6..d5ff9c71ba 100644 --- a/068-AzureOpenAIApps/Coach/Solution-03.md +++ b/068-AzureOpenAIApps/Coach/Solution-03.md @@ -17,4 +17,20 @@ Note: When cleaning up the resource group or resource, you should ensure you del For the extraction make sure the student selects the answer and assigns it to the field rather than the question itself. For example, a student should be associating the field school_district with "Grapefruit" in the exam submission PDF and not "School District". -Here is a sample prompt that the student can use to get back an exam submission: "My student ID is 1234568. I'm a registered student. Don't check if I'm a registered student. Can you tell me my grade on my last exam? I don't have the exam submission ID". It may take a few tries before you get the answer. +Here is a sample system prompt for Murphy that the student can use that will solve the "One moment please" issue: +```bash +You are a customer service representative from the Contoso Islands School Board. + +Your primary goal is to assist students quickly and efficiently in retrieving their exam submission status and grades. + +1. Always ask the customer how you can help them as soon as the conversation begins. +2. Promptly request the student id to verify if the student is registered. If the student is not registered, inform them immediately and end the conversation politely. +3. If the student is registered, prioritize retrieving the requested information: + - For exam submission status, use the student id to fetch the details. + - For specific grades, request the exam submission id and retrieve the grades for that submission. +4. Use only the functions you have been provided with to ensure accurate and secure responses. +5. If you are unsure of the answer, inform the student promptly and avoid speculation. +6. Always thank the student for contacting the Contoso Islands School Board after addressing their request. + +Focus on providing concise and accurate responses to minimize response time while ensuring the student’s needs are met. Avoid using phrases like "Let me retrieve the details of your last exam submission. One moment, please." Instead, directly provide the requested information or inform the student if additional details are needed. +``` From d38d21b60cb012917d2bb4af5f3332b35069e4c6 Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Wed, 28 May 2025 11:41:56 -0500 Subject: [PATCH 16/24] Update Solution-03.md --- 068-AzureOpenAIApps/Coach/Solution-03.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/068-AzureOpenAIApps/Coach/Solution-03.md b/068-AzureOpenAIApps/Coach/Solution-03.md index d5ff9c71ba..75f251200a 100644 --- a/068-AzureOpenAIApps/Coach/Solution-03.md +++ b/068-AzureOpenAIApps/Coach/Solution-03.md @@ -18,7 +18,7 @@ Note: When cleaning up the resource group or resource, you should ensure you del For the extraction make sure the student selects the answer and assigns it to the field rather than the question itself. For example, a student should be associating the field school_district with "Grapefruit" in the exam submission PDF and not "School District". Here is a sample system prompt for Murphy that the student can use that will solve the "One moment please" issue: -```bash +```text You are a customer service representative from the Contoso Islands School Board. Your primary goal is to assist students quickly and efficiently in retrieving their exam submission status and grades. @@ -32,5 +32,7 @@ Your primary goal is to assist students quickly and efficiently in retrieving th 5. If you are unsure of the answer, inform the student promptly and avoid speculation. 6. Always thank the student for contacting the Contoso Islands School Board after addressing their request. -Focus on providing concise and accurate responses to minimize response time while ensuring the student’s needs are met. Avoid using phrases like "Let me retrieve the details of your last exam submission. One moment, please." Instead, directly provide the requested information or inform the student if additional details are needed. +Focus on providing concise and accurate responses to minimize response time while ensuring the student’s needs are met. +Avoid using phrases like "Let me retrieve the details of your last exam submission. +One moment, please." Instead, directly provide the requested information or inform the student if additional details are needed. ``` From a38e83fdb7b9180189f66f6b70af764d71e54beb Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Fri, 30 May 2025 12:03:02 -0500 Subject: [PATCH 17/24] Update Challenge-00-lab.md --- 068-AzureOpenAIApps/Student/Challenge-00-lab.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/068-AzureOpenAIApps/Student/Challenge-00-lab.md b/068-AzureOpenAIApps/Student/Challenge-00-lab.md index 9414dbb38f..96108c3100 100644 --- a/068-AzureOpenAIApps/Student/Challenge-00-lab.md +++ b/068-AzureOpenAIApps/Student/Challenge-00-lab.md @@ -79,15 +79,7 @@ If you want to setup your environment on your local workstation, expand the sect
Click to expand/collapse Local Workstation Requirements - -On Mac OS (**NOTE:** only tested on Apple Silicon): -- Download and install Docker Desktop -- Clone the Codespace's GitHub repo locally on your Mac -- Open the root folder for the repo in Visual Studio Code -- You might get prompted to re-open the container in a Dev Container. You can do that by clicking the Yes button, but if you miss it or hit no, you can also use the Command Palette in VS Code and select Dev Containers: Reopen in Container -WSL on Windows ARM64 does not currently support running the Azure Function Core Tools in x86_64 emulation using QEMU so you will need to use a Codespace instead - #### Student Resources The sample application code, Azure deployment scripts, and sample data sources for this hack are available in a Student Resources package. @@ -96,6 +88,15 @@ The sample application code, Azure deployment scripts, and sample data sources f The rest of the challenges will refer to the relative paths inside the `Resources.zip` file where you can find the various resources to complete the challenges. +You will next be setting up your local workstation so that it can use dev containers. A dev container is a Docker-based environment designed to provide a consistent and reproducible development setup. The VS Code Dev Containers extension lets you easily open projects inside a containerized environment. + +On Windows and Mac OS (**NOTE:** only tested on Apple Silicon): +- Download and install Docker Desktop +- (Mac OS only) In Docker Desktop settings, choose Apple Virtualization Framework for the Virtual Machine Manager. Also, click the checkbox to use Rosetta for x86_64/amd64 emulation on Apple Silicon +- (Windows only) Install the Windows Subsystem for Linux along with a Linux distribution such as Ubuntu. You will need to copy the Resources.zip to your Linux home directory and unzip it there. **NOTE:** WSL on Windows ARM64 does not currently support running the Azure Function Core Tools in x86_64 emulation using QEMU so you will need to use a Codespace instead +- Open the root folder for the repo in Visual Studio Code +- You might get prompted to re-open the container in a Dev Container. You can do that by clicking the Yes button, but if you miss it or hit no, you can also use the Command Palette in VS Code and select Dev Containers: Reopen in Container +
### Setup Citrus Bus Application From 69fd034e4283cb531f77f674aaac3f773d1cf7af Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Fri, 30 May 2025 12:03:31 -0500 Subject: [PATCH 18/24] Update Challenge-00-nolab.md --- .../Student/Challenge-00-nolab.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/068-AzureOpenAIApps/Student/Challenge-00-nolab.md b/068-AzureOpenAIApps/Student/Challenge-00-nolab.md index 81c3889a97..4e4f3bece1 100644 --- a/068-AzureOpenAIApps/Student/Challenge-00-nolab.md +++ b/068-AzureOpenAIApps/Student/Challenge-00-nolab.md @@ -77,16 +77,7 @@ If you want to setup your environment on your local workstation, expand the sect
Click to expand/collapse Local Workstation Requirements - -On Mac OS (**NOTE:** only tested on Apple Silicon): -- Download and install Docker Desktop -- Clone the Codespace's GitHub repo locally on your Mac -- Open the root folder for the repo in Visual Studio Code -- You might get prompted to re-open the container in a Dev Container. You can do that by clicking the Yes button, but if you miss it or hit no, you can also use the Command Palette in VS Code and select Dev Containers: Reopen in Container - -WSL on Windows ARM64 does not currently support running the Azure Function Core Tools in x86_64 emulation using QEMU so you will need to use a Codespace instead - #### Student Resources The sample application code, Azure deployment scripts, and sample data sources for this hack are available in a Student Resources package. @@ -95,6 +86,15 @@ The sample application code, Azure deployment scripts, and sample data sources f The rest of the challenges will refer to the relative paths inside the `Resources.zip` file where you can find the various resources to complete the challenges. +You will next be setting up your local workstation so that it can use dev containers. A dev container is a Docker-based environment designed to provide a consistent and reproducible development setup. The VS Code Dev Containers extension lets you easily open projects inside a containerized environment. + +On Windows and Mac OS (**NOTE:** only tested on Apple Silicon): +- Download and install Docker Desktop +- (Mac OS only) In Docker Desktop settings, choose Apple Virtualization Framework for the Virtual Machine Manager. Also, click the checkbox to use Rosetta for x86_64/amd64 emulation on Apple Silicon +- (Windows only) Install the Windows Subsystem for Linux along with a Linux distribution such as Ubuntu. You will need to copy the Resources.zip to your Linux home directory and unzip it there. **NOTE:** WSL on Windows ARM64 does not currently support running the Azure Function Core Tools in x86_64 emulation using QEMU so you will need to use a Codespace instead +- Open the root folder for the repo in Visual Studio Code +- You might get prompted to re-open the container in a Dev Container. You can do that by clicking the Yes button, but if you miss it or hit no, you can also use the Command Palette in VS Code and select Dev Containers: Reopen in Container +
### Setup Citrus Bus Application From 963576212814706c47113f65995f71acfc882ce3 Mon Sep 17 00:00:00 2001 From: "Peter C. Laudati" Date: Fri, 30 May 2025 16:42:51 -0400 Subject: [PATCH 19/24] Update Challenge-00-lab.md fixed Resources.zip --- 068-AzureOpenAIApps/Student/Challenge-00-lab.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/068-AzureOpenAIApps/Student/Challenge-00-lab.md b/068-AzureOpenAIApps/Student/Challenge-00-lab.md index 96108c3100..4becf99bf2 100644 --- a/068-AzureOpenAIApps/Student/Challenge-00-lab.md +++ b/068-AzureOpenAIApps/Student/Challenge-00-lab.md @@ -84,7 +84,7 @@ If you want to setup your environment on your local workstation, expand the sect The sample application code, Azure deployment scripts, and sample data sources for this hack are available in a Student Resources package. -- [Download and unpack the Resources.zip](https://aka.ms/openaiapps/resources) package to your local workstation. +- [Download and unpack the Resources.zip](https://aka.ms/wth/openaiapps/resources) package to your local workstation. The rest of the challenges will refer to the relative paths inside the `Resources.zip` file where you can find the various resources to complete the challenges. From afc65f477a753098c804e290545415d41872c8fb Mon Sep 17 00:00:00 2001 From: "Peter C. Laudati" Date: Fri, 30 May 2025 16:56:46 -0400 Subject: [PATCH 20/24] Update Challenge-00-lab.md wordsmithed Windows ARM issue --- 068-AzureOpenAIApps/Student/Challenge-00-lab.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/068-AzureOpenAIApps/Student/Challenge-00-lab.md b/068-AzureOpenAIApps/Student/Challenge-00-lab.md index 4becf99bf2..10c0bae2f6 100644 --- a/068-AzureOpenAIApps/Student/Challenge-00-lab.md +++ b/068-AzureOpenAIApps/Student/Challenge-00-lab.md @@ -79,7 +79,7 @@ If you want to setup your environment on your local workstation, expand the sect
Click to expand/collapse Local Workstation Requirements - + #### Student Resources The sample application code, Azure deployment scripts, and sample data sources for this hack are available in a Student Resources package. @@ -88,14 +88,18 @@ The sample application code, Azure deployment scripts, and sample data sources f The rest of the challenges will refer to the relative paths inside the `Resources.zip` file where you can find the various resources to complete the challenges. -You will next be setting up your local workstation so that it can use dev containers. A dev container is a Docker-based environment designed to provide a consistent and reproducible development setup. The VS Code Dev Containers extension lets you easily open projects inside a containerized environment. +#### Set Up Local Dev Container + +You will next be setting up your local workstation so that it can use dev containers. A Dev Container is a Docker-based environment designed to provide a consistent and reproducible development setup. The VS Code Dev Containers extension lets you easily open projects inside a containerized environment. + +**NOTE:** On Windows, Dev Containers run in the Windows Subsystem for Linux (WSL). As of May 2025, WSL on Windows ARM64 does not currently support running the Azure Function Core Tools needed for this hackathon in x86_64 emulation using QEMU. IF you are using a Windows on ARM device, you will need to use a GitHub Codespace instead. On Windows and Mac OS (**NOTE:** only tested on Apple Silicon): - Download and install Docker Desktop - (Mac OS only) In Docker Desktop settings, choose Apple Virtualization Framework for the Virtual Machine Manager. Also, click the checkbox to use Rosetta for x86_64/amd64 emulation on Apple Silicon -- (Windows only) Install the Windows Subsystem for Linux along with a Linux distribution such as Ubuntu. You will need to copy the Resources.zip to your Linux home directory and unzip it there. **NOTE:** WSL on Windows ARM64 does not currently support running the Azure Function Core Tools in x86_64 emulation using QEMU so you will need to use a Codespace instead -- Open the root folder for the repo in Visual Studio Code -- You might get prompted to re-open the container in a Dev Container. You can do that by clicking the Yes button, but if you miss it or hit no, you can also use the Command Palette in VS Code and select Dev Containers: Reopen in Container +- (Windows only) Install the Windows Subsystem for Linux along with a Linux distribution such as Ubuntu. You will need to copy the `Resources.zip` to your Linux home directory and unzip it there. +- Open the root folder of the Student resource package in Visual Studio Code +- You should get prompted to re-open the folder in a Dev Container. You can do that by clicking the Yes button, but if you miss it or hit no, you can also use the Command Palette in VS Code and select `Dev Containers: Reopen in Container`
From 89c8b44481ca08ab30b57f11b678254484d5a733 Mon Sep 17 00:00:00 2001 From: "Peter C. Laudati" Date: Fri, 30 May 2025 16:58:20 -0400 Subject: [PATCH 21/24] Update Challenge-00-nolab.md wordsmithed win on arm issues. --- 068-AzureOpenAIApps/Student/Challenge-00-nolab.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/068-AzureOpenAIApps/Student/Challenge-00-nolab.md b/068-AzureOpenAIApps/Student/Challenge-00-nolab.md index 4e4f3bece1..4a9e25c932 100644 --- a/068-AzureOpenAIApps/Student/Challenge-00-nolab.md +++ b/068-AzureOpenAIApps/Student/Challenge-00-nolab.md @@ -82,18 +82,22 @@ If you want to setup your environment on your local workstation, expand the sect The sample application code, Azure deployment scripts, and sample data sources for this hack are available in a Student Resources package. -- [Download and unpack the Resources.zip](https://aka.ms/openaiapps/resources) package to your local workstation. +- [Download and unpack the Resources.zip](https://aka.ms/wth/openaiapps/resources) package to your local workstation. The rest of the challenges will refer to the relative paths inside the `Resources.zip` file where you can find the various resources to complete the challenges. -You will next be setting up your local workstation so that it can use dev containers. A dev container is a Docker-based environment designed to provide a consistent and reproducible development setup. The VS Code Dev Containers extension lets you easily open projects inside a containerized environment. +#### Set Up Local Dev Container + +You will next be setting up your local workstation so that it can use dev containers. A Dev Container is a Docker-based environment designed to provide a consistent and reproducible development setup. The VS Code Dev Containers extension lets you easily open projects inside a containerized environment. + +**NOTE:** On Windows, Dev Containers run in the Windows Subsystem for Linux (WSL). As of May 2025, WSL on Windows ARM64 does not currently support running the Azure Function Core Tools needed for this hackathon in x86_64 emulation using QEMU. IF you are using a Windows on ARM device, you will need to use a GitHub Codespace instead. On Windows and Mac OS (**NOTE:** only tested on Apple Silicon): - Download and install Docker Desktop - (Mac OS only) In Docker Desktop settings, choose Apple Virtualization Framework for the Virtual Machine Manager. Also, click the checkbox to use Rosetta for x86_64/amd64 emulation on Apple Silicon -- (Windows only) Install the Windows Subsystem for Linux along with a Linux distribution such as Ubuntu. You will need to copy the Resources.zip to your Linux home directory and unzip it there. **NOTE:** WSL on Windows ARM64 does not currently support running the Azure Function Core Tools in x86_64 emulation using QEMU so you will need to use a Codespace instead -- Open the root folder for the repo in Visual Studio Code -- You might get prompted to re-open the container in a Dev Container. You can do that by clicking the Yes button, but if you miss it or hit no, you can also use the Command Palette in VS Code and select Dev Containers: Reopen in Container +- (Windows only) Install the Windows Subsystem for Linux along with a Linux distribution such as Ubuntu. You will need to copy the `Resources.zip` to your Linux home directory and unzip it there. +- Open the root folder of the Student resource package in Visual Studio Code +- You should get prompted to re-open the folder in a Dev Container. You can do that by clicking the Yes button, but if you miss it or hit no, you can also use the Command Palette in VS Code and select `Dev Containers: Reopen in Container`
From c148d84c10dc8a9ee2f334d75abb2275dabb0b99 Mon Sep 17 00:00:00 2001 From: Pete Rodriguez Date: Fri, 30 May 2025 16:05:05 -0500 Subject: [PATCH 22/24] Updating student resource files from codespace --- .../Resources/ContosoAIAppsBackend/.gitignore | 2 + ...ument-intelligence-dictionary.json.example | 26 +++--- .../Resources/ContosoAIAppsFrontend/README.md | 2 +- .../ContosoAIAppsFrontend/package.json | 78 +++++++++--------- .../Student/Resources/infra/deploy.sh | 18 +++- .../Student/Resources/infra/fixperms.sh | 82 +++++++++++++++++++ .../Student/Resources/infra/functions.sh | 8 +- .../Student/Resources/infra/main.bicep | 13 ++- 8 files changed, 171 insertions(+), 58 deletions(-) create mode 100644 068-AzureOpenAIApps/Student/Resources/infra/fixperms.sh diff --git a/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsBackend/.gitignore b/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsBackend/.gitignore index cb579dbf46..7ed6ad5b6e 100644 --- a/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsBackend/.gitignore +++ b/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsBackend/.gitignore @@ -168,3 +168,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +.python_packages \ No newline at end of file diff --git a/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsBackend/document-intelligence-dictionary.json.example b/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsBackend/document-intelligence-dictionary.json.example index 8b6dd5c047..6c8cf28d3f 100644 --- a/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsBackend/document-intelligence-dictionary.json.example +++ b/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsBackend/document-intelligence-dictionary.json.example @@ -9,10 +9,10 @@ "school_district": "school_district", "school_name": "school_name", "exam_date": "exam_date", - "question_1": "q1", - "question_2": "q2", - "question_3": "q3", - "question_4": "q4" + "question_1": "question_1", + "question_2": "question_2", + "question_3": "question_3", + "question_4": "question_4" } }, { @@ -25,10 +25,10 @@ "school_district": "school_district", "school_name": "school_name", "exam_date": "exam_date", - "question_1": "q1", - "question_2": "q2", - "question_3": "q3", - "question_4": "q4" + "question_1": "question_1", + "question_2": "question_2", + "question_3": "question_3", + "question_4": "question_4" } }, { @@ -41,11 +41,11 @@ "school_district": "school_district", "school_name": "school_name", "exam_date": "exam_date", - "question_1": "q1", - "question_2": "q2", - "question_3": "q3", - "question_4": "q4", - "question_5": "q5" + "question_1": "question_1", + "question_2": "question_2", + "question_3": "question_3", + "question_4": "question_4", + "question_5": "question_5" } }, { diff --git a/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsFrontend/README.md b/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsFrontend/README.md index 53cc9e3a30..221de6773a 100644 --- a/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsFrontend/README.md +++ b/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsFrontend/README.md @@ -1,4 +1,4 @@ -# Contoso AI Apps +# ContosoAiApps This project was generated with [Angular CLI](https://github.com/angular/angular-cli) version 16.2.7. diff --git a/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsFrontend/package.json b/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsFrontend/package.json index a0806643c9..18aee13ae3 100644 --- a/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsFrontend/package.json +++ b/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsFrontend/package.json @@ -1,40 +1,40 @@ { - "name": "contoso-ai-apps", - "version": "0.0.0", - "scripts": { - "ng": "ng", - "start": "ng serve --port 4200", - "build": "ng build", - "watch": "ng build --watch --configuration development", - "test": "ng test" - }, - "private": true, - "dependencies": { - "@angular/animations": "^19.2.5", - "@angular/common": "^19.2.5", - "@angular/compiler": "^19.2.5", - "@angular/core": "^19.2.5", - "@angular/forms": "^19.2.5", - "@angular/platform-browser": "^19.2.5", - "@angular/platform-browser-dynamic": "^19.2.5", - "@angular/router": "^19.2.5", - "rxjs": "~7.8.2", - "tslib": "^2.8.1", - "uuid": "^11.1.0", - "zone.js": "~0.15.0" - }, - "devDependencies": { - "@angular-devkit/build-angular": "^19.2.6", - "@angular/cli": "^19.2.6", - "@angular/compiler-cli": "^19.2.5", - "@types/jasmine": "~5.1.7", - "@types/uuid": "^10.0.0", - "jasmine-core": "~5.6.0", - "karma": "~6.4.4", - "karma-chrome-launcher": "~3.2.0", - "karma-coverage": "~2.2.1", - "karma-jasmine": "~5.1.0", - "karma-jasmine-html-reporter": "~2.1.0", - "typescript": "~5.8.2" - } -} + "name": "contoso-ai-apps", + "version": "0.0.0", + "scripts": { + "ng": "ng", + "start": "ng serve --host 0.0.0.0 --port 4200", + "build": "ng build", + "watch": "ng build --watch --configuration development", + "test": "ng test" + }, + "private": true, + "dependencies": { + "@angular/animations": "^19.2.5", + "@angular/common": "^19.2.5", + "@angular/compiler": "^19.2.5", + "@angular/core": "^19.2.5", + "@angular/forms": "^19.2.5", + "@angular/platform-browser": "^19.2.5", + "@angular/platform-browser-dynamic": "^19.2.5", + "@angular/router": "^19.2.5", + "rxjs": "~7.8.2", + "tslib": "^2.8.1", + "uuid": "^11.1.0", + "zone.js": "~0.15.0" + }, + "devDependencies": { + "@angular-devkit/build-angular": "^19.2.6", + "@angular/cli": "^19.2.6", + "@angular/compiler-cli": "^19.2.5", + "@types/jasmine": "~5.1.7", + "@types/uuid": "^10.0.0", + "jasmine-core": "~5.6.0", + "karma": "~6.4.4", + "karma-chrome-launcher": "~3.2.0", + "karma-coverage": "~2.2.1", + "karma-jasmine": "~5.1.0", + "karma-jasmine-html-reporter": "~2.1.0", + "typescript": "~5.8.2" + } +} \ No newline at end of file diff --git a/068-AzureOpenAIApps/Student/Resources/infra/deploy.sh b/068-AzureOpenAIApps/Student/Resources/infra/deploy.sh index cc28d09605..0e2585e881 100755 --- a/068-AzureOpenAIApps/Student/Resources/infra/deploy.sh +++ b/068-AzureOpenAIApps/Student/Resources/infra/deploy.sh @@ -7,7 +7,10 @@ LOCATION="East US 2" DOCUMENT_INTELLIGENCE_LOCATION="East US" OPENAI_LOCATION="East US 2" RESOURCE_GROUP_NAME="openai-apps-wth" - +MODEL_NAME="gpt-4o" +MODEL_VERSION="2024-11-20" +EMBEDDING_MODEL="text-embedding-ada-002" +EMBEDDING_MODEL_VERSION="2" # Parse arguments while [[ "$#" -gt 0 ]]; do case $1 in @@ -22,6 +25,10 @@ while [[ "$#" -gt 0 ]]; do --document-intelligence-location) DOCUMENT_INTELLIGENCE_LOCATION="$2"; shift ;; --skip-local-settings-file) SKIP_LOCAL_SETTINGS_FILE=true; shift ;; --silent-install) SILENT_INSTALL=true; shift ;; + --model-name) MODEL_NAME="$2"; shift ;; + --model-version) MODEL_VERSION="$2"; shift ;; + --embedding-model) EMBEDDING_MODEL="$2"; shift ;; + --embedding-model-version) EMBEDDING_MODEL_VERSION="$2"; shift ;; *) error_exit "Unknown parameter passed: $1" ;; esac shift @@ -53,6 +60,10 @@ if [[ "$SILENT_INSTALL" == false ]]; then echo -e "\t Region: \e[33m$LOCATION\e[0m" echo -e "\t OpenAI Location: \e[33m$OPENAI_LOCATION\e[0m" echo -e "\t Azure DI Location: \e[33m$DOCUMENT_INTELLIGENCE_LOCATION\e[0m" + echo -e "\t Model Name: \e[33m$MODEL_NAME\e[0m" + echo -e "\tModel Version: \e[33m$MODEL_VERSION\e[0m" + echo -e "\tEmbedding Model: \e[33m$EMBEDDING_MODEL\e[0m" + echo -e "\tEmbedding Model Version: \e[33m$EMBEDDING_MODEL_VERSION\e[0m" echo -e "\e[31mIf any parameter is incorrect, abort this script, correct, and try again.\e[0m" echo -e "It will take around \e[32m15 minutes\e[0m to deploy all resources. You can monitor the progress from the deployments page in the resource group in Azure Portal.\n" @@ -71,7 +82,7 @@ az group create --name "$RESOURCE_GROUP_NAME" --location "$LOCATION" || error_ex # Deploy resources echo -e "\n- Deploying resources: " result=$(az deployment group create --resource-group "$RESOURCE_GROUP_NAME" --template-file ./main.bicep \ - --parameters openAILocation="$OPENAI_LOCATION" documentIntelligenceLocation="$DOCUMENT_INTELLIGENCE_LOCATION") || error_exit "Azure deployment failed." + --parameters openAILocation="$OPENAI_LOCATION" documentIntelligenceLocation="$DOCUMENT_INTELLIGENCE_LOCATION" modelName="$MODEL_NAME" modelVersion="$MODEL_VERSION" embeddingModel="$EMBEDDING_MODEL" embeddingModelVersion="$EMBEDDING_MODEL_VERSION") || error_exit "Azure deployment failed." # Extract outputs outputs=$(echo "$result" | jq -r '.properties.outputs') @@ -110,6 +121,9 @@ az cosmosdb update -g $RESOURCE_GROUP_NAME --name $COSMOS_DB_ACCOUNT --public-ne az resource update -g $RESOURCE_GROUP_NAME --name $COSMOS_DB_ACCOUNT --resource-type "Microsoft.DocumentDB/databaseAccounts" --set properties.disableLocalAuth=false az search service update --name $AZURE_AI_SEARCH --resource-group $RESOURCE_GROUP_NAME --public-access enabled --disable-local-auth false +cosmosdb=$(az cosmosdb show --name $COSMOS_DB_ACCOUNT --resource-group $RESOURCE_GROUP_NAME | jq -r '.id') +az resource update --ids $cosmosdb --set properties.disableLocalAuth=false --latest-include-preview + #End workaround diff --git a/068-AzureOpenAIApps/Student/Resources/infra/fixperms.sh b/068-AzureOpenAIApps/Student/Resources/infra/fixperms.sh new file mode 100644 index 0000000000..235dac60fd --- /dev/null +++ b/068-AzureOpenAIApps/Student/Resources/infra/fixperms.sh @@ -0,0 +1,82 @@ +#Workaround for FDPO MSFT internal subscriptions. +# Include functions +source ./functions.sh + +# Parse arguments +while [[ "$#" -gt 0 ]]; do + case $1 in + --resource-group-name) RESOURCE_GROUP_NAME="$2"; shift ;; + *) error_exit "Unknown parameter passed: $1" ;; + esac + shift +done + +# Validate mandatory parameters +if [[ -z "$RESOURCE_GROUP_NAME" ]]; then + error_exit "Resource Group Name is mandatory." +fi +# Try a command that requires login, suppress its output +if az account show > /dev/null 2>&1; then + echo "Azure CLI is already logged in." +else + echo "Need to run 'az login'." + az login +fi + +# Check if jq is installed +if ! command -v jq &> /dev/null; then + error_exit "jq is not installed. Please install it to proceed." +fi + +# Check if the resource group exists +echo "Checking if resource group '$RESOURCE_GROUP_NAME' exists..." +if [[ "$(az group exists --name "$RESOURCE_GROUP_NAME")" == "false" ]]; then + error_exit "Resource group '$RESOURCE_GROUP_NAME' not found. You may need to run deploy.sh instead." +fi +echo "Resource group '$RESOURCE_GROUP_NAME' found." + +# Extract outputs +outputs=$(az deployment group show \ + --resource-group $RESOURCE_GROUP_NAME \ + --name "main" \ + --query properties.outputs) +#echo $outputs + +AZURE_AI_SEARCH=$(echo "$outputs" | jq -r '.searchName.value') + +storageConnectionString=$(echo "$outputs" | jq -r '.storageConnectionString.value') +STORAGE_ACCOUNT=$(echo "$storageConnectionString" | sed -n 's/.*AccountName=\([^;]*\).*/\1/p') + +webjobsConnectionString=$(echo "$outputs" | jq -r '.webjobsConnectionString.value') +WEBJOBS_STORAGE_ACCOUNT=$(echo "$webjobsConnectionString" | sed -n 's/.*AccountName=\([^;]*\).*/\1/p') + +COSMOS_DB_ACCOUNT=$(echo "$outputs" | jq -r '.cosmosDBAccount.value') + +az storage account update -g $RESOURCE_GROUP_NAME --name $STORAGE_ACCOUNT --allow-shared-key-access true + + +az storage account update \ + --name ${STORAGE_ACCOUNT} \ + --resource-group $RESOURCE_GROUP_NAME \ + --public-network-access Enabled \ + --default-action Allow + +az storage account update -g $RESOURCE_GROUP_NAME --name $WEBJOBS_STORAGE_ACCOUNT --allow-shared-key-access true + +az storage account update \ + --name ${WEBJOBS_STORAGE_ACCOUNT} \ + --resource-group $RESOURCE_GROUP_NAME \ + --public-network-access Enabled \ + --default-action Allow + + # Enable Public Network Access +az cosmosdb update -g $RESOURCE_GROUP_NAME --name $COSMOS_DB_ACCOUNT --public-network-access Enabled + +# Enable Local Authentication with Keys +az resource update -g $RESOURCE_GROUP_NAME --name $COSMOS_DB_ACCOUNT --resource-type "Microsoft.DocumentDB/databaseAccounts" --set properties.disableLocalAuth=false + +az search service update --name $AZURE_AI_SEARCH --resource-group $RESOURCE_GROUP_NAME --public-access enabled --disable-local-auth false +cosmosdb=$(az cosmosdb show --name $COSMOS_DB_ACCOUNT --resource-group $RESOURCE_GROUP_NAME | jq -r '.id') +az resource update --ids $cosmosdb --set properties.disableLocalAuth=false --latest-include-preview + +#End workaround diff --git a/068-AzureOpenAIApps/Student/Resources/infra/functions.sh b/068-AzureOpenAIApps/Student/Resources/infra/functions.sh index dc74e27d8c..339976b891 100644 --- a/068-AzureOpenAIApps/Student/Resources/infra/functions.sh +++ b/068-AzureOpenAIApps/Student/Resources/infra/functions.sh @@ -36,6 +36,10 @@ function generate_local_settings_file { --arg storageConnection "$(echo "$outputs" | jq -r '.storageConnectionString.value')" \ --arg appInsightsConnection "$(echo "$outputs" | jq -r '.appInsightsConnectionString.value')" \ --arg serviceBusConnection "$(echo "$outputs" | jq -r '.serviceBusConnectionString.value')" \ + --arg modelName "$(echo "$outputs" | jq -r '.modelName.value')" \ + --arg modelVersion "$(echo "$outputs" | jq -r '.modelVersion.value')" \ + --arg embeddingModel "$(echo "$outputs" | jq -r '.embeddingModel.value')" \ + --arg embeddingModelVersion "$(echo "$outputs" | jq -r '.embeddingModelVersion.value')" \ '.Values.AZURE_OPENAI_API_KEY = $openAIKey | .Values.AZURE_OPENAI_ENDPOINT = $openAIEndpoint | .Values.AZURE_AI_SEARCH_ADMIN_KEY = $searchKey | @@ -49,7 +53,9 @@ function generate_local_settings_file { .Values.AzureWebJobsStorage = $webjobsConnection | .Values.DOCUMENT_STORAGE = $storageConnection | .Values.APPLICATIONINSIGHTS_CONNECTION_STRING = $appInsightsConnection | - .Values.SERVICE_BUS_CONNECTION_STRING = $serviceBusConnection' \ + .Values.SERVICE_BUS_CONNECTION_STRING = $serviceBusConnection | + .Values.AZURE_OPENAI_MODEL_DEPLOYMENT_NAME = $modelName | + .Values.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME = $embeddingModel' \ "$settings_file" > tmp.json && mv tmp.json "$settings_file" echo -e "\e[32mSettings file created successfully.\e[0m" diff --git a/068-AzureOpenAIApps/Student/Resources/infra/main.bicep b/068-AzureOpenAIApps/Student/Resources/infra/main.bicep index 7e24d3520f..7eae6b5c8a 100644 --- a/068-AzureOpenAIApps/Student/Resources/infra/main.bicep +++ b/068-AzureOpenAIApps/Student/Resources/infra/main.bicep @@ -5,6 +5,10 @@ var location = resourceGroup().location param openAILocation string param documentIntelligenceLocation string +param modelName string = 'gpt-4o' +param modelVersion string = '2024-11-20' +param embeddingModel string = 'text-embedding-ada-002' +param embeddingModelVersion string = '2' module redis 'modules/redis.bicep' = { name: 'redisDeployment' @@ -56,14 +60,19 @@ module openai 'modules/openai.bicep' = { location: openAILocation name: 'openai-${suffix}' deployments: [ - { name: 'gpt-4o', version: '2024-11-20' } - { name: 'text-embedding-ada-002', version: '2' } + { name: modelName, version: modelVersion } + { name: embeddingModel, version: embeddingModelVersion } ] } } output openAIKey string = openai.outputs.key1 output openAIEndpoint string = openai.outputs.endpoint +output modelName string = modelName +output modelVersion string = modelVersion +output embeddingModel string = embeddingModel +output embeddingModelVersion string = embeddingModelVersion + module search 'modules/search.bicep' = { name: 'searchDeployment' From 4c3ac41daba1ad6505f7a534be1ab60439fc5186 Mon Sep 17 00:00:00 2001 From: "Peter C. Laudati" Date: Fri, 30 May 2025 17:07:58 -0400 Subject: [PATCH 23/24] Update .wordlist.txt --- 068-AzureOpenAIApps/.wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/068-AzureOpenAIApps/.wordlist.txt b/068-AzureOpenAIApps/.wordlist.txt index 342bcfeaab..32205649d4 100644 --- a/068-AzureOpenAIApps/.wordlist.txt +++ b/068-AzureOpenAIApps/.wordlist.txt @@ -21,3 +21,4 @@ vectorized uploader Huachao queueing +QEMU From d4e013eb9d55153dfe72d222ec5c3ab0fef26765 Mon Sep 17 00:00:00 2001 From: "Peter C. Laudati" Date: Fri, 30 May 2025 17:08:35 -0400 Subject: [PATCH 24/24] Update README.md --- .../Student/Resources/ContosoAIAppsFrontend/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsFrontend/README.md b/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsFrontend/README.md index 221de6773a..53cc9e3a30 100644 --- a/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsFrontend/README.md +++ b/068-AzureOpenAIApps/Student/Resources/ContosoAIAppsFrontend/README.md @@ -1,4 +1,4 @@ -# ContosoAiApps +# Contoso AI Apps This project was generated with [Angular CLI](https://github.com/angular/angular-cli) version 16.2.7.