Agenteval integration (#2672)

* first pass at offline agent eval integration

* Integrating AgentEval for offline scenarios

* removing old changes

* fixing notebook, updating docs

* fixing subcriteria bug

* updating class comment

* cleaning up agent constructors

* moving AgentEval agents to separate folder and adding a brief README

* fixing build breaks

* fixing formatting break

* fixing comments

* consolidating files in the agenteval folder under contrib and cleaning up imports

* fixing import ordering

* adding basic agenteval tests and fixing criteria parsing bug

* first try at adding openai agenteval tests to build process

* adding non-openai agenteval tests to build process

* updating test settings

* updating openai test

* Update test/agentchat/contrib/agent_eval/test_agent_eval.py

Co-authored-by: Wael Karkoub <wael.karkoub96@gmail.com>

* Update .github/workflows/contrib-openai.yml

Co-authored-by: Wael Karkoub <wael.karkoub96@gmail.com>

* test commit

* updating typing and converting to pydantic objects

* fixing test file

---------

Co-authored-by: Beibin Li <BeibinLi@users.noreply.github.com>
Co-authored-by: Chi Wang <wang.chi@microsoft.com>
Co-authored-by: Wael Karkoub <wael.karkoub96@gmail.com>

.github/workflows/contrib-openai.yml

@@ -74,7 +74,43 @@ jobs:
         with:
           file: ./coverage.xml
           flags: unittests
-
+  AgentEvalTest:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.10"]
+    runs-on: ${{ matrix.os }}
+    environment: openai1
+    steps:
+      # checkout to pr branch
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install packages and dependencies
+        run: |
+          docker --version
+          python -m pip install --upgrade pip wheel
+          pip install -e .
+          python -c "import autogen"
+          pip install pytest-cov>=5 pytest-asyncio
+      - name: Coverage
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
+          OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
+        run: |
+          pytest test/agentchat/contrib/agent_eval/test_agent_eval.py
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          flags: unittests
   CompressionTest:
     strategy:
       matrix:

.github/workflows/contrib-tests.yml

@@ -125,6 +125,35 @@ jobs:
           file: ./coverage.xml
           flags: unittests
 
+  AgentEvalTest:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.10"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install packages and dependencies for all tests
+        run: |
+          python -m pip install --upgrade pip wheel
+          pip install pytest-cov>=5
+      - name: Install packages and dependencies for AgentEval
+        run: |
+          pip install -e .
+      - name: Coverage
+        run: |
+          pytest test/agentchat/contrib/agent_eval/ --skip-openai
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          flags: unittests
+
   CompressionTest:
     runs-on: ${{ matrix.os }}
     strategy:

autogen/agentchat/contrib/agent_eval/README.md

@@ -0,0 +1,7 @@
+Agents for running the AgentEval pipeline.
+
+AgentEval is a process for evaluating a LLM-based system's performance on a given task.
+
+When given a task to evaluate and a few example runs, the critic and subcritic agents create evaluation criteria for evaluating a system's solution. Once the criteria has been created, the quantifier agent can evaluate subsequent task solutions based on the generated criteria.
+
+For more information see: [AgentEval Integration Roadmap](https://github.com/microsoft/autogen/issues/2162)

autogen/agentchat/contrib/agent_eval/agent_eval.py

@@ -0,0 +1,101 @@
+from typing import Dict, List, Literal, Optional, Union
+
+import autogen
+from autogen.agentchat.contrib.agent_eval.criterion import Criterion
+from autogen.agentchat.contrib.agent_eval.critic_agent import CriticAgent
+from autogen.agentchat.contrib.agent_eval.quantifier_agent import QuantifierAgent
+from autogen.agentchat.contrib.agent_eval.subcritic_agent import SubCriticAgent
+from autogen.agentchat.contrib.agent_eval.task import Task
+
+
+def generate_criteria(
+    llm_config: Optional[Union[Dict, Literal[False]]] = None,
+    task: Task = None,
+    additional_instructions: str = "",
+    max_round=2,
+    use_subcritic: bool = False,
+):
+    """
+    Creates a list of criteria for evaluating the utility of a given task.
+    Args:
+        llm_config (dict or bool): llm inference configuration.
+        task (Task): The task to evaluate.
+        additional_instructions (str): Additional instructions for the criteria agent.
+        max_round (int): The maximum number of rounds to run the conversation.
+        use_subcritic (bool): Whether to use the subcritic agent to generate subcriteria.
+    Returns:
+        list: A list of Criterion objects for evaluating the utility of the given task.
+    """
+    critic = CriticAgent(
+        system_message=CriticAgent.DEFAULT_SYSTEM_MESSAGE + "\n" + additional_instructions,
+        llm_config=llm_config,
+    )
+
+    critic_user = autogen.UserProxyAgent(
+        name="critic_user",
+        max_consecutive_auto_reply=0,  # terminate without auto-reply
+        human_input_mode="NEVER",
+        code_execution_config={"use_docker": False},
+    )
+
+    agents = [critic_user, critic]
+
+    if use_subcritic:
+        subcritic = SubCriticAgent(
+            llm_config=llm_config,
+        )
+        agents.append(subcritic)
+
+    groupchat = autogen.GroupChat(
+        agents=agents, messages=[], max_round=max_round, speaker_selection_method="round_robin"
+    )
+    critic_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=llm_config)
+
+    critic_user.initiate_chat(critic_manager, message=task.get_sys_message())
+    criteria = critic_user.last_message()
+    content = criteria["content"]
+    # need to strip out any extra code around the returned json
+    content = content[content.find("[") : content.rfind("]") + 1]
+    criteria = Criterion.parse_json_str(content)
+    return criteria
+
+
+def quantify_criteria(
+    llm_config: Optional[Union[Dict, Literal[False]]] = None,
+    criteria: List[Criterion] = None,
+    task: Task = None,
+    test_case: str = "",
+    ground_truth: str = "",
+):
+    """
+    Quantifies the performance of a system using the provided criteria.
+    Args:
+        llm_config (dict or bool): llm inference configuration.
+        criteria ([Criterion]): A list of criteria for evaluating the utility of a given task.
+        task (Task): The task to evaluate.
+        test_case (str): The test case to evaluate.
+        ground_truth (str): The ground truth for the test case.
+    Returns:
+        dict: A dictionary where the keys are the criteria and the values are the assessed performance based on accepted values for each criteria.
+    """
+    quantifier = QuantifierAgent(
+        llm_config=llm_config,
+    )
+
+    quantifier_user = autogen.UserProxyAgent(
+        name="quantifier_user",
+        max_consecutive_auto_reply=0,  # terminate without auto-reply
+        human_input_mode="NEVER",
+        code_execution_config={"use_docker": False},
+    )
+
+    quantifier_user.initiate_chat(  # noqa: F841
+        quantifier,
+        message=task.get_sys_message()
+        + "Evaluation dictionary: "
+        + Criterion.write_json(criteria)
+        + "actual test case to evaluate: "
+        + test_case,
+    )
+    quantified_results = quantifier_user.last_message()
+    return {"actual_success": ground_truth, "estimated_performance": quantified_results["content"]}

autogen/agentchat/contrib/agent_eval/criterion.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import json
+from typing import List
+
+import pydantic_core
+from pydantic import BaseModel
+from pydantic.json import pydantic_encoder
+
+
+class Criterion(BaseModel):
+    """
+    A class that represents a criterion for agent evaluation.
+    """
+
+    name: str
+    description: str
+    accepted_values: List[str]
+    sub_criteria: List[Criterion] = list()
+
+    @staticmethod
+    def parse_json_str(criteria: str):
+        """
+        Create a list of Criterion objects from a json string.
+        Args:
+            criteria (str): Json string that represents the criteria
+        returns:
+            [Criterion]: A list of Criterion objects that represents the json criteria information.
+        """
+        return [Criterion(**crit) for crit in json.loads(criteria)]
+
+    @staticmethod
+    def write_json(criteria):
+        """
+        Create a json string from a list of Criterion objects.
+        Args:
+            criteria ([Criterion]): A list of Criterion objects.
+        Returns:
+            str: A json string that represents the list of Criterion objects.
+        """
+        return json.dumps([crit.model_dump() for crit in criteria], indent=2)

autogen/agentchat/contrib/agent_eval/critic_agent.py

@@ -0,0 +1,41 @@
+from typing import Optional
+
+from autogen.agentchat.conversable_agent import ConversableAgent
+
+
+class CriticAgent(ConversableAgent):
+    """
+    An agent for creating list of criteria for evaluating the utility of a given task.
+    """
+
+    DEFAULT_SYSTEM_MESSAGE = """You are a helpful assistant. You suggest criteria for evaluating different tasks. They should be distinguishable, quantifiable and not redundant.
+    Convert the evaluation criteria into a list where each item is a criteria which consists of the following dictionary as follows
+    {"name": name of the criterion, "description": criteria description , "accepted_values": possible accepted inputs for this key}
+    Make sure "accepted_values" include the acceptable inputs for each key that are fine-grained and preferably multi-graded levels and "description" includes the criterion description.
+    Output just the criteria string you have created, no code.
+    """
+
+    DEFAULT_DESCRIPTION = "An AI agent for creating list criteria for evaluating the utility of a given task."
+
+    def __init__(
+        self,
+        name="critic",
+        system_message: Optional[str] = DEFAULT_SYSTEM_MESSAGE,
+        description: Optional[str] = DEFAULT_DESCRIPTION,
+        **kwargs,
+    ):
+        """
+        Args:
+            name (str): agent name.
+            system_message (str): system message for the ChatCompletion inference.
+                Please override this attribute if you want to reprogram the agent.
+            description (str): The description of the agent.
+            **kwargs (dict): Please refer to other kwargs in
+                [ConversableAgent](../../conversable_agent#__init__).
+        """
+        super().__init__(
+            name=name,
+            system_message=system_message,
+            description=description,
+            **kwargs,
+        )

autogen/agentchat/contrib/agent_eval/quantifier_agent.py

@@ -0,0 +1,36 @@
+from typing import Optional
+
+from autogen.agentchat.conversable_agent import ConversableAgent
+
+
+class QuantifierAgent(ConversableAgent):
+    """
+    An agent for quantifying the performance of a system using the provided criteria.
+    """
+
+    DEFAULT_SYSTEM_MESSAGE = """"You are a helpful assistant. You quantify the output of different tasks based on the given criteria.
+    The criterion is given in a json list format where each element is a distinct criteria.
+    The each element is a dictionary as follows {"name": name of the criterion, "description": criteria description , "accepted_values": possible accepted inputs for this key}
+    You are going to quantify each of the crieria for a given task based on the task description.
+    Return a dictionary where the keys are the criteria and the values are the assessed performance based on accepted values for each criteria.
+    Return only the dictionary, no code."""
+
+    DEFAULT_DESCRIPTION = "An AI agent for quantifing the performance of a system using the provided criteria."
+
+    def __init__(
+        self,
+        name="quantifier",
+        system_message: Optional[str] = DEFAULT_SYSTEM_MESSAGE,
+        description: Optional[str] = DEFAULT_DESCRIPTION,
+        **kwargs,
+    ):
+        """
+        Args:
+            name (str): agent name.
+            system_message (str): system message for the ChatCompletion inference.
+                Please override this attribute if you want to reprogram the agent.
+            description (str): The description of the agent.
+            **kwargs (dict): Please refer to other kwargs in
+                [ConversableAgent](../../conversable_agent#__init__).
+        """
+        super().__init__(name=name, system_message=system_message, description=description, **kwargs)

autogen/agentchat/contrib/agent_eval/subcritic_agent.py

@@ -0,0 +1,42 @@
+from typing import Optional
+
+from autogen.agentchat.conversable_agent import ConversableAgent
+
+
+class SubCriticAgent(ConversableAgent):
+    """
+    An agent for creating subcriteria from a given list of criteria for evaluating the utility of a given task.
+    """
+
+    DEFAULT_SYSTEM_MESSAGE = """You are a helpful assistant to the critic agent. You suggest sub criteria for evaluating different tasks based on the criteria provided by the critic agent (if you feel it is needed).
+        They should be distinguishable, quantifiable, and related to the overall theme of the critic's provided criteria.
+        You operate by taking in the description of the criteria. You then create a new key called sub criteria where you provide the sub criteria for the given criteria.
+        The value of the sub_criteria is a dictionary where the keys are the subcriteria and each value is as follows {"description": sub criteria description , "accepted_values": possible accepted inputs for this key}
+        Do this for each criteria provided by the critic (removing the criteria's accepted values). "accepted_values" include the acceptable inputs for each key that are fine-grained and preferably multi-graded levels. "description" includes the criterion description.
+        Once you have created the sub criteria for the given criteria, you return the json (make sure to include the contents of the critic's dictionary in the final dictionary as well).
+        Make sure to return a valid json and no code"""
+
+    DEFAULT_DESCRIPTION = "An AI agent for creating subcriteria from a given list of criteria."
+
+    def __init__(
+        self,
+        name="subcritic",
+        system_message: Optional[str] = DEFAULT_SYSTEM_MESSAGE,
+        description: Optional[str] = DEFAULT_DESCRIPTION,
+        **kwargs,
+    ):
+        """
+        Args:
+            name (str): agent name.
+            system_message (str): system message for the ChatCompletion inference.
+                Please override this attribute if you want to reprogram the agent.
+            description (str): The description of the agent.
+            **kwargs (dict): Please refer to other kwargs in
+                [ConversableAgent](../../conversable_agent#__init__).
+        """
+        super().__init__(
+            name=name,
+            system_message=system_message,
+            description=description,
+            **kwargs,
+        )

autogen/agentchat/contrib/agent_eval/task.py

@@ -0,0 +1,37 @@
+import json
+
+from pydantic import BaseModel
+
+
+class Task(BaseModel):
+    """
+    Class representing a task for agent completion, includes example agent execution for criteria generation.
+    """
+
+    name: str
+    description: str
+    successful_response: str
+    failed_response: str
+
+    def get_sys_message(self):
+        return f"""Task: {self.name}.
+        Task description: {self.description}
+        Task successful example: {self.successful_response}
+        Task failed example: {self.failed_response}
+        """
+
+    @staticmethod
+    def parse_json_str(task: str):
+        """
+        Create a Task object from a json object.
+        Args:
+            json_data (dict): A dictionary that represents the task.
+        Returns:
+            Task: A Task object that represents the json task information.
+        """
+        json_data = json.loads(task)
+        name = json_data.get("name")
+        description = json_data.get("description")
+        successful_response = json_data.get("successful_response")
+        failed_response = json_data.get("failed_response")
+        return Task(name, description, successful_response, failed_response)

test/agentchat/contrib/agent_eval/test_agent_eval.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3 -m pytest
+
+import json
+
+import pytest
+from conftest import reason, skip_openai  # noqa: E402
+
+import autogen
+from autogen.agentchat.contrib.agent_eval.agent_eval import generate_criteria, quantify_criteria
+from autogen.agentchat.contrib.agent_eval.criterion import Criterion
+from autogen.agentchat.contrib.agent_eval.task import Task
+
+KEY_LOC = "notebook"
+OAI_CONFIG_LIST = "OAI_CONFIG_LIST"
+
+
+def remove_ground_truth(test_case: str):
+    test_details = json.loads(test_case)
+    # need to remove the ground truth from the test details
+    correctness = test_details.pop("is_correct", None)
+    test_details.pop("correct_ans", None)
+    test_details.pop("check_result", None)
+    return str(test_details), correctness
+
+
+if not skip_openai:
+    openai_config_list = autogen.config_list_from_json(
+        OAI_CONFIG_LIST,
+        file_location=KEY_LOC,
+        # The Retrieval tool requires at least gpt-3.5-turbo-1106 (newer versions are supported) or gpt-4-turbo-preview models.
+        # https://platform.openai.com/docs/models/overview
+        filter_dict={
+            "api_type": ["openai"],
+            "model": [
+                "gpt-4-turbo",
+                "gpt-4-turbo-preview",
+                "gpt-4-0125-preview",
+                "gpt-4-1106-preview",
+                "gpt-3.5-turbo",
+                "gpt-3.5-turbo-0125",
+                "gpt-3.5-turbo-1106",
+            ],
+        },
+    )
+
+    aoai_config_list = autogen.config_list_from_json(
+        OAI_CONFIG_LIST,
+        file_location=KEY_LOC,
+        filter_dict={"api_type": ["azure"]},
+    )
+
+    success_str = open("test/test_files/agenteval-in-out/samples/sample_math_response_successful.txt", "r").read()
+    response_successful = remove_ground_truth(success_str)[0]
+    failed_str = open("test/test_files/agenteval-in-out/samples/sample_math_response_failed.txt", "r").read()
+    response_failed = remove_ground_truth(failed_str)[0]
+    task = Task(
+        **{
+            "name": "Math problem solving",
+            "description": "Given any question, the system needs to solve the problem as consisely and accurately as possible",
+            "successful_response": response_successful,
+            "failed_response": response_failed,
+        }
+    )
+
+
+@pytest.mark.skipif(
+    skip_openai,
+    reason=reason,
+)
+def test_generate_criteria():
+    criteria = generate_criteria(task=task, llm_config={"config_list": aoai_config_list})
+    assert criteria
+    assert len(criteria) > 0
+    assert criteria[0].description
+    assert criteria[0].name
+    assert criteria[0].accepted_values
+
+
+@pytest.mark.skipif(
+    skip_openai,
+    reason=reason,
+)
+def test_quantify_criteria():
+    criteria_file = "test/test_files/agenteval-in-out/samples/sample_math_criteria.json"
+    criteria = open(criteria_file, "r").read()
+    criteria = Criterion.parse_json_str(criteria)
+
+    test_case = open("test/test_files/agenteval-in-out/samples/sample_test_case.json", "r").read()
+    test_case, ground_truth = remove_ground_truth(test_case)
+
+    quantified = quantify_criteria(
+        llm_config={"config_list": aoai_config_list},
+        criteria=criteria,
+        task=task,
+        test_case=test_case,
+        ground_truth=ground_truth,
+    )
+    assert quantified
+    assert quantified["actual_success"]
+    assert quantified["estimated_performance"]

test/agentchat/contrib/agent_eval/test_criterion.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3 -m pytest
+
+from autogen.agentchat.contrib.agent_eval.criterion import Criterion
+
+
+def test_parse_json_str():
+    criteria_file = "test/test_files/agenteval-in-out/samples/sample_math_criteria.json"
+    criteria = open(criteria_file, "r").read()
+    criteria = Criterion.parse_json_str(criteria)
+    assert criteria
+    assert len(criteria) == 6
+    assert criteria[0].name == "Problem Interpretation"
+    assert criteria[0].description == "Ability to correctly interpret the problem."
+    assert len(criteria[0].accepted_values) == 5
+
+
+def test_write_json():
+    criteria1 = Criterion(name="test1", description="test1 description", accepted_values=["test1", "test2"])
+    criteria2 = Criterion(name="test2", description="test2 description", accepted_values=["test1", "test2"])
+    output = Criterion.write_json([criteria1, criteria2])
+    assert (
+        output
+        == """[
+  {
+    "name": "test1",
+    "description": "test1 description",
+    "accepted_values": [
+      "test1",
+      "test2"
+    ],
+    "sub_criteria": []
+  },
+  {
+    "name": "test2",
+    "description": "test2 description",
+    "accepted_values": [
+      "test1",
+      "test2"
+    ],
+    "sub_criteria": []
+  }
+]"""
+    )
+
+
+def test_write_parse_compatibility():
+    criterion1 = Criterion(name="test1", description="test1 description", accepted_values=["test1", "test2"])
+    criterion2 = Criterion(name="test2", description="test2 description", accepted_values=["test1", "test2"])
+
+    output = Criterion.write_json([criterion1, criterion2])
+    criteria = Criterion.parse_json_str(output)
+    assert criteria
+    assert len(criteria) == 2
+    assert criteria[0].name == "test1"
+    assert criteria[0].description == "test1 description"
+    assert len(criteria[0].accepted_values) == 2
+    assert criteria[1].name == "test2"
+    assert criteria[1].description == "test2 description"
+    assert len(criteria[1].accepted_values) == 2

test/agentchat/contrib/agent_eval/test_task.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3 -m pytest
+
+from autogen.agentchat.contrib.agent_eval.task import Task
+
+
+def test_parse_json_str():
+    task = Task(
+        **{
+            "name": "Math problem solving",
+            "description": "Given any question, the system needs to solve the problem as consisely and accurately as possible",
+            "successful_response": '{"message": "The answer is 5", "is_correct": True}',
+            "failed_response": '{"message": "I don\'t know the answer", "is_correct": False}',
+        }
+    )
+    assert task
+    assert task.name == "Math problem solving"
+    assert (
+        task.description
+        == "Given any question, the system needs to solve the problem as consisely and accurately as possible"
+    )
+    assert task.successful_response == '{"message": "The answer is 5", "is_correct": True}'
+    assert task.failed_response == '{"message": "I don\'t know the answer", "is_correct": False}'

test/test_files/agenteval-in-out/samples/sample_math_criteria.json

@@ -1,26 +1,26 @@
-{
-    "Problem Interpretation": {
-      "description": "Ability to correctly interpret the problem.",
-      "accepted_values": ["completely off", "slightly relevant", "relevant", "mostly accurate", "completely accurate"]
-    },
-    "Mathematical Methodology": {
-      "description": "Adequacy of the chosen mathematical or algorithmic methodology for the question",
-      "accepted_values": ["inappropriate", "barely adequate", "adequate", "mostly effective", "completely effective"]
-    },
-    "Calculation Correctness": {
-      "description": "Accuracy of calculations made and solutions given",
-      "accepted_values": ["completely incorrect", "mostly incorrect", "neither", "mostly correct", "completely correct"]
-    },
-    "Explanation Clarity": {
-      "description": "Clarity and comprehensibility of explanations, including language use and structure",
-      "accepted_values": ["not at all clear", "slightly clear", "moderately clear", "very clear", "completely clear"]
-    },
-    "Code Efficiency": {
-      "description": "Quality of code in terms of  efficiency and elegance",
-      "accepted_values": ["not at all efficient", "slightly efficient", "moderately efficient", "very efficient", "extremely efficient"]
-    },
-    "Code Correctness": {
-      "description": "Correctness of the provided code",
-      "accepted_values": ["completely incorrect", "mostly incorrect", "partly correct", "mostly correct", "completely correct"]
-    }
+[
+  { "name": "Problem Interpretation",
+    "description": "Ability to correctly interpret the problem.",
+    "accepted_values": ["completely off", "slightly relevant", "relevant", "mostly accurate", "completely accurate"]
+  },
+  { "name": "Mathematical Methodology",
+    "description": "Adequacy of the chosen mathematical or algorithmic methodology for the question",
+    "accepted_values": ["inappropriate", "barely adequate", "adequate", "mostly effective", "completely effective"]
+  },
+  { "name": "Calculation Correctness",
+    "description": "Accuracy of calculations made and solutions given",
+    "accepted_values": ["completely incorrect", "mostly incorrect", "neither", "mostly correct", "completely correct"]
+  },
+  { "name": "Explanation Clarity",
+    "description": "Clarity and comprehensibility of explanations, including language use and structure",
+    "accepted_values": ["not at all clear", "slightly clear", "moderately clear", "very clear", "completely clear"]
+  },
+  { "name": "Code Efficiency",
+    "description": "Quality of code in terms of  efficiency and elegance",
+    "accepted_values": ["not at all efficient", "slightly efficient", "moderately efficient", "very efficient", "extremely efficient"]
+  },
+  { "name": "Code Correctness",
+    "description": "Correctness of the provided code",
+    "accepted_values": ["completely incorrect", "mostly incorrect", "partly correct", "mostly correct", "completely correct"]
   }
+]