tutorials/Regression_testing_with_debugging.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4wXT98y_u6s-"
      },
      "outputs": [],
      "source": [
        "!pip install evidently[llm]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "from evidently.report import Report\n",
        "from evidently.test_suite import TestSuite\n",
        "from evidently.metric_preset import TextEvals\n",
        "from evidently.descriptors import *\n",
        "from evidently.tests import *\n",
        "from evidently.metrics import *\n",
        "\n",
        "from evidently.features.llm_judge import BinaryClassificationPromptTemplate"
      ],
      "metadata": {
        "id": "CVjuVaNfvACI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "To connect to Evidently Cloud:"
      ],
      "metadata": {
        "id": "O7lwoL4lmjfl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from evidently.ui.workspace.cloud import CloudWorkspace"
      ],
      "metadata": {
        "id": "LuckpHlgmmdQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Optional imports to create monitoring panels:"
      ],
      "metadata": {
        "id": "sXECWY60vCcS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from evidently.ui.dashboards import DashboardPanelPlot\n",
        "from evidently.ui.dashboards import DashboardPanelTestSuite\n",
        "from evidently.ui.dashboards import DashboardPanelTestSuiteCounter\n",
        "from evidently.ui.dashboards import TestSuitePanelType\n",
        "from evidently.ui.dashboards import ReportFilter\n",
        "from evidently.ui.dashboards import PanelValue\n",
        "from evidently.ui.dashboards import PlotType\n",
        "from evidently.ui.dashboards import CounterAgg\n",
        "from evidently.tests.base_test import TestStatus\n",
        "from evidently.renderers.html_widgets import WidgetSize"
      ],
      "metadata": {
        "id": "p19oWfaHvBwi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "os.environ[\"OPENAI_API_KEY\"] = \"YOUR KEY\""
      ],
      "metadata": {
        "id": "Cq5qLorfH75r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Connect to Evidently Cloud"
      ],
      "metadata": {
        "id": "DHzlvxKTvGeN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ws = CloudWorkspace(token=\"YOUR_API_TOKEN\", url=\"https://app.evidently.cloud\")"
      ],
      "metadata": {
        "id": "0kx9pYp5vGLS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Create a Project"
      ],
      "metadata": {
        "id": "7Nt3X9dmvKwL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "project = ws.create_project(\"Regression testing example\", team_id=\"YOUR_TEAM_ID\")\n",
        "project.description = \"My project description\"\n",
        "project.save()"
      ],
      "metadata": {
        "id": "Nv_QRvE2vMBE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Reference answers"
      ],
      "metadata": {
        "id": "vnilkhDtvP5D"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Prepare the dataset with inputs and approved answers you want to compare against."
      ],
      "metadata": {
        "id": "pqhUG-LavPJQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = [\n",
        "    [\"Why is the sky blue?\", \"The sky is blue because molecules in the air scatter blue light from the sun more than they scatter red light.\"],\n",
        "    [\"How do airplanes stay in the air?\", \"Airplanes stay in the air because their wings create lift by forcing air to move faster over the top of the wing than underneath, which creates lower pressure on top.\"],\n",
        "    [\"Why do we have seasons?\", \"We have seasons because the Earth is tilted on its axis, which causes different parts of the Earth to receive more or less sunlight throughout the year.\"],\n",
        "    [\"How do magnets work?\", \"Magnets work because they have a magnetic field that can attract or repel certain metals, like iron, due to the alignment of their atomic particles.\"],\n",
        "    [\"Why does the moon change shape?\", \"The moon changes shape, or goes through phases, because we see different portions of its illuminated half as it orbits the Earth.\"]\n",
        "]\n",
        "\n",
        "columns = [\"question\", \"target_response\"]\n",
        "\n",
        "ref_data = pd.DataFrame(data, columns=columns)"
      ],
      "metadata": {
        "id": "QKPLO4ln4S3X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pd.set_option('display.max_colwidth', None)"
      ],
      "metadata": {
        "id": "z72kQ8_T4fhE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ref_data.head()"
      ],
      "metadata": {
        "id": "lBvi2q9l4WX0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "report = Report(metrics=[\n",
        "    TextEvals(column_name=\"target_response\", descriptors=[\n",
        "        TextLength(),\n",
        "    ]),\n",
        "])\n",
        "\n",
        "report.run(reference_data=None,\n",
        "           current_data=ref_data)\n",
        "\n",
        "report"
      ],
      "metadata": {
        "id": "WhBwkYgm5h1Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "report.as_dict()"
      ],
      "metadata": {
        "id": "T4dbpZAQ6RhY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "report.json()"
      ],
      "metadata": {
        "id": "hEiv1TJJ6Xld"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Generate new answers"
      ],
      "metadata": {
        "id": "ABmpYXHV12ad"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's imitate. In practice, call your LLM app, get new answers, add them to the dataframe."
      ],
      "metadata": {
        "id": "GkyhudkI14DR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = [\n",
        "    [\"Why is the sky blue?\",\n",
        "     \"The sky is blue because molecules in the air scatter blue light from the sun more than they scatter red light.\",\n",
        "     \"The sky appears blue because air molecules scatter the sun’s blue light more than they scatter other colors.\"],\n",
        "\n",
        "    [\"How do airplanes stay in the air?\",\n",
        "     \"Airplanes stay in the air because their wings create lift by forcing air to move faster over the top of the wing than underneath, which creates lower pressure on top.\",\n",
        "     \"Airplanes stay airborne because the shape of their wings causes air to move faster over the top than the bottom, generating lift.\"],\n",
        "\n",
        "    [\"Why do we have seasons?\",\n",
        "     \"We have seasons because the Earth is tilted on its axis, which causes different parts of the Earth to receive more or less sunlight throughout the year.\",\n",
        "     \"Seasons occur because of the tilt of the Earth’s axis, leading to varying amounts of sunlight reaching different areas as the Earth orbits the sun.\"],\n",
        "\n",
        "    [\"How do magnets work?\",\n",
        "     \"Magnets work because they have a magnetic field that can attract or repel certain metals, like iron, due to the alignment of their atomic particles.\",\n",
        "     \"Magnets generate a magnetic field, which can attract metals like iron by causing the electrons in those metals to align in a particular way, creating an attractive or repulsive force.\"],\n",
        "\n",
        "    [\"Why does the moon change shape?\",\n",
        "     \"The moon changes shape, or goes through phases, because we see different portions of its illuminated half as it orbits the Earth.\",\n",
        "     \"The moon appears to change shape as it orbits Earth, which is because we see different parts of its lit-up half at different times. The sun lights up half of the moon, but as the moon moves around the Earth, we see varying portions of that lit-up side. So, the moon's shape in the sky seems to change gradually, from a thin crescent to a full circle and back to a crescent again.\"]\n",
        "]\n",
        "\n",
        "columns = [\"question\", \"target_response\", \"response\"]\n",
        "\n",
        "eval_data = pd.DataFrame(data, columns=columns)"
      ],
      "metadata": {
        "id": "ebzLUprM6PWN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "eval_data.head()"
      ],
      "metadata": {
        "id": "dXsNvaGD6gny"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Choose criteria"
      ],
      "metadata": {
        "id": "-qiIZ4oYy8DV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "- LLM-judged correctness based on reference: must be always correct.\n",
        "- LLM-judged style match to reference: must be always matching.\n",
        "- Text length is always between 80 and 200 symbols.\n"
      ],
      "metadata": {
        "id": "OQSA-6aBy99t"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Correctness LLM judge"
      ],
      "metadata": {
        "id": "w3ZpBa3G_EKK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "correctness_eval= LLMEval(\n",
        "    subcolumn=\"category\",\n",
        "    additional_columns={\"target_response\": \"target_response\"},\n",
        "    template = BinaryClassificationPromptTemplate(\n",
        "        criteria = \"\"\"\n",
        "An ANSWER is correct when it is the same as the REFERENCE in all facts and details, even if worded differently.\n",
        "The ANSWER is incorrect if it contradicts the REFERENCE, adds additional claims, omits or changes details.\n",
        "\n",
        "REFERENCE:\n",
        "\n",
        "=====\n",
        "{target_response}\n",
        "=====\n",
        "        \"\"\",\n",
        "        target_category=\"incorrect\",\n",
        "        non_target_category=\"correct\",\n",
        "        uncertainty=\"unknown\",\n",
        "        include_reasoning=True,\n",
        "        pre_messages=[(\"system\", \"You are an expert evaluator. will be given an ANSWER and REFERENCE.\")],\n",
        "        ),\n",
        "    provider = \"openai\",\n",
        "    model = \"gpt-4o-mini\",\n",
        "    display_name = \"Correctness\",\n",
        ")"
      ],
      "metadata": {
        "id": "ZbmSdbBO9pKU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Style LLM judge"
      ],
      "metadata": {
        "id": "6EIwNsV__G8e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "style_eval= LLMEval(\n",
        "    subcolumn=\"category\",\n",
        "    additional_columns={\"target_response\": \"target_response\"},\n",
        "    template = BinaryClassificationPromptTemplate(\n",
        "        criteria = \"\"\"\n",
        "An ANSWER is style-matching when it matches the REFERENCE answer in style.\n",
        "The ANSWER is style-mismatched when it diverges from the REFERENCE answer in style.\n",
        "\n",
        "Consider the following STYLE attributes:\n",
        "- tone (friendly, formal, casual, sarcastic, etc.)\n",
        "- sentence structure (simple, compound, complex, etc.)\n",
        "- verbosity level (relative length of answers)\n",
        "- and other similar attributes that may reflect difference in STYLE.\n",
        "\n",
        "You must focus only on STYLE. Ignore any differences in contents.\n",
        "\n",
        "=====\n",
        "{target_response}\n",
        "=====\n",
        "        \"\"\",\n",
        "        target_category=\"style-matching\",\n",
        "        non_target_category=\"style-mismatched\",\n",
        "        uncertainty=\"unknown\",\n",
        "        include_reasoning=True,\n",
        "        pre_messages=[(\"system\", \"You are an expert evaluator. will be given an ANSWER and REFERENCE.\")],\n",
        "        ),\n",
        "    provider = \"openai\",\n",
        "    model = \"gpt-4o-mini\",\n",
        "    display_name = \"Style\",\n",
        ")"
      ],
      "metadata": {
        "id": "y4bx2Inl_KoM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create a Test Suite"
      ],
      "metadata": {
        "id": "g9uPhOu_DCRj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "test_suite = TestSuite(tests=[\n",
        "    TestCategoryCount(\n",
        "        column_name=style_eval.on(\"response\"),\n",
        "        category=\"style-mismatched\",\n",
        "        eq=0,\n",
        "        is_critical=False),\n",
        "    TestCategoryCount(\n",
        "        column_name=correctness_eval.on(\"response\"),\n",
        "        category=\"incorrect\",\n",
        "        eq=0),\n",
        "    TestShareOfOutRangeValues(\n",
        "        column_name=TextLength(display_name=\"Response Length\")\n",
        "            .on(\"response\"),\n",
        "        left=80,\n",
        "        right=200,\n",
        "        eq=0),\n",
        "])"
      ],
      "metadata": {
        "id": "Ytj-v7-jDFwo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "For use without LLMs, you can try semantic similarity."
      ],
      "metadata": {
        "id": "drWqMBqXIW-O"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# test_suite = TestSuite(tests=[\n",
        "#    TestColumnValueMin(\n",
        "#        column_name=SemanticSimilarity(\n",
        "#        display_name=\"Response Similarity\").\n",
        "#        on([\"response\", \"target_response\"]),\n",
        "#        gte=0.9),\n",
        "#    TestShareOfOutRangeValues(\n",
        "#        column_name=TextLength(display_name=\"Response Length\")\n",
        "#            .on(\"response\"),\n",
        "#        left=80,\n",
        "#        right=200,\n",
        "#        eq=0),\n",
        "# ])"
      ],
      "metadata": {
        "id": "1kAcjP-dIV0H"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Run regression testing"
      ],
      "metadata": {
        "id": "WU32JGp22rL2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "test_suite.run(reference_data=None, current_data=eval_data)"
      ],
      "metadata": {
        "id": "8nXKnaEyBV9J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ws.add_test_suite(project.id, test_suite, include_data=True)"
      ],
      "metadata": {
        "id": "LH3nCj5CEgwt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# test_suite"
      ],
      "metadata": {
        "id": "Q0vTOXoIBW_P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# test_suite.datasets().current"
      ],
      "metadata": {
        "id": "sHTMl3TUBZ_i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Explore the results"
      ],
      "metadata": {
        "id": "384JViXb3JHC"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Head to the Evidently Platform UI."
      ],
      "metadata": {
        "id": "w_QNtSBN3Kju"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Next change? Test again."
      ],
      "metadata": {
        "id": "nzPLZcug3Sgv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = [\n",
        "    [\"Why is the sky blue?\",\n",
        "     \"The sky is blue because molecules in the air scatter blue light from the sun more than they scatter red light.\",\n",
        "     \"The sky looks blue because air molecules scatter the blue light from the sun more effectively than other colors.\"],\n",
        "\n",
        "    [\"How do airplanes stay in the air?\",\n",
        "     \"Airplanes stay in the air because their wings create lift by forcing air to move faster over the top of the wing than underneath, which creates lower pressure on top.\",\n",
        "     \"Airplanes fly by generating lift through the wings, which makes the air move faster above them, lowering the pressure.\"],\n",
        "\n",
        "    [\"Why do we have seasons?\",\n",
        "     \"We have seasons because the Earth is tilted on its axis, which causes different parts of the Earth to receive more or less sunlight throughout the year.\",\n",
        "     \"Seasons change because the distance between the Earth and the sun varies throughout the year.\"],  # This response contradicts the reference.\n",
        "\n",
        "    [\"How do magnets work?\",\n",
        "     \"Magnets work because they have a magnetic field that can attract or repel certain metals, like iron, due to the alignment of their atomic particles.\",\n",
        "     \"Magnets operate by creating a magnetic field, which interacts with certain metals like iron due to the specific alignment of atomic particles.\"],\n",
        "\n",
        "    [\"Why does the moon change shape?\",\n",
        "     \"The moon changes shape, or goes through phases, because we see different portions of its illuminated half as it orbits the Earth.\",\n",
        "     \"The moon's phases occur because we observe varying portions of its lit half as it moves around the Earth.\"]\n",
        "]\n",
        "\n",
        "columns = [\"question\", \"target_response\", \"response\"]\n",
        "\n",
        "eval_data_2 = pd.DataFrame(data, columns=columns)"
      ],
      "metadata": {
        "id": "-ROi4IikFJS-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_suite.run(reference_data=None, current_data=eval_data_2)"
      ],
      "metadata": {
        "id": "sZbMn-FJFReA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ws.add_test_suite(project.id, test_suite, include_data=True)"
      ],
      "metadata": {
        "id": "Z9vg4NW-FUHA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# test_suite"
      ],
      "metadata": {
        "id": "-xQIn-iTFWER"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# test_suite.datasets().current"
      ],
      "metadata": {
        "id": "orbPxsYIFqta"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Get a dashboard"
      ],
      "metadata": {
        "id": "tJ4U6_EI3PcB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Add a counter panel to show the SUCCESS rate of the latest test run:"
      ],
      "metadata": {
        "id": "1t4coiwMLx2Z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "project.dashboard.add_panel(\n",
        "     DashboardPanelTestSuiteCounter(\n",
        "        title=\"Latest Test run\",\n",
        "        filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),\n",
        "        size=WidgetSize.FULL,\n",
        "        statuses=[TestStatus.SUCCESS],\n",
        "        agg=CounterAgg.LAST,\n",
        "    ),\n",
        "    tab=\"Tests\"\n",
        ")\n"
      ],
      "metadata": {
        "id": "T79tMN0Y3IqH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Add a test monitoring panel to show all test results over time."
      ],
      "metadata": {
        "id": "CW8M_5EHL02P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "project.dashboard.add_panel(\n",
        "    DashboardPanelTestSuite(\n",
        "        title=\"Test results\",\n",
        "        filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),\n",
        "        size=WidgetSize.FULL,\n",
        "        panel_type=TestSuitePanelType.DETAILED,\n",
        "        time_agg=\"1min\",\n",
        "    ),\n",
        "    tab=\"Tests\"\n",
        ")\n"
      ],
      "metadata": {
        "id": "IQ8bWmCYLvyy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "project.save()"
      ],
      "metadata": {
        "id": "e5-7czDkMZM9"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}