Merge pull request #2399 from microsoft/gagb-ct_webarena

Improve documentation of MMWebsurfer

autogen/agentchat/contrib/mmagent.py

@@ -0,0 +1,43 @@
+# ruff: noqa: E722
+import autogen
+from autogen.code_utils import content_str
+
+
+class MultimodalAgent(autogen.ConversableAgent):
+    def __init__(
+        self,
+        name,
+        **kwargs,
+    ):
+        super().__init__(
+            name=name,
+            **kwargs,
+        )
+        self._reply_func_list = []
+        self.register_reply([autogen.Agent, None], MultimodalAgent.generate_mlm_reply)
+        self.register_reply([autogen.Agent, None], autogen.ConversableAgent.generate_code_execution_reply)
+        self.register_reply([autogen.Agent, None], autogen.ConversableAgent.generate_function_call_reply)
+        self.register_reply([autogen.Agent, None], autogen.ConversableAgent.check_termination_and_human_reply)
+
+    def generate_mlm_reply(
+        self,
+        messages=None,
+        sender=None,
+        config=None,
+    ):
+        """Generate a reply using autogen.oai."""
+        if messages is None:
+            messages = self._oai_messages[sender]
+
+        # Clone the messages to give context, but remove old screenshots
+        history = []
+        for i in range(0, len(messages) - 1):
+            message = {}
+            message.update(messages[i])
+            message["content"] = content_str(message["content"])
+            history.append(message)
+        history.append(messages[-1])
+
+        response = self.client.create(messages=self._oai_system_message + history)
+        completion = self.client.extract_text_or_completion_object(response)[0]
+        return True, completion

autogen/agentchat/contrib/multimodal_web_surfer/multimodal_web_surfer.py

@@ -51,6 +51,8 @@ class MultimodalWebSurferAgent(ConversableAgent):
 
     DEFAULT_DESCRIPTION = "A helpful assistant with access to a web browser. Ask them to perform web searches, open pages, and interact with content (e.g., clicking links, scrolling the viewport, etc., filling in form fields, etc.)"
 
+    DEFAULT_START_PAGE = "https://www.bing.com/"
+
     def __init__(
         self,
         name: str,
@@ -62,14 +64,34 @@ class MultimodalWebSurferAgent(ConversableAgent):
         function_map: Optional[Dict[str, Callable]] = None,
         code_execution_config: Union[Dict, Literal[False]] = False,
         llm_config: Optional[Union[Dict, Literal[False]]] = None,
-        mlm_config: Optional[Union[Dict, Literal[False]]] = None,
+        mlm_config: Optional[Union[Dict, Literal[False]]] = None,  # TODO: Remove this
         default_auto_reply: Optional[Union[str, Dict, None]] = "",
-        headless=True,
+        headless: bool = True,
         chromium_channel=DEFAULT_CHANNEL,
-        chromium_data_dir=None,
-        start_page="https://www.bing.com/",
-        debug_dir=os.getcwd(),
+        chromium_data_dir: Optional[str] = None,
+        start_page: Optional[str] = None,
+        debug_dir: Optional[str] = None,
     ):
+        """
+        Create a new MultimodalWebSurferAgent.
+
+        Args:
+            name: The name of the agent.
+            system_message: system message prompt.
+            description: The description of the agent.
+            is_termination_msg: A function that determines if a received message is a termination message.
+            max_consecutive_auto_reply: The maximum number of consecutive auto-replies the agent can make.
+            human_input_mode: The mode for human input.
+            function_map: A dictionary of functions to register.
+            code_execution_config: The configuration for code execution.
+            llm_config: The configuration for the LLM.
+            default_auto_reply: The default auto-reply.
+            headless: Whether to run the browser in headless mode.
+            chromium_channel: The Chromium channel to use.
+            chromium_data_dir: The Chromium data directory. If None, a new context is created.
+            start_page: The start page for the browser.
+            debug_dir: The directory to store debug information. TODO: Clarify behavior on None.
+        """
         super().__init__(
             name=name,
             system_message=system_message,
@@ -84,8 +106,8 @@ class MultimodalWebSurferAgent(ConversableAgent):
         )
         # self._mlm_config = mlm_config
         # self._mlm_client = OpenAIWrapper(**self._mlm_config)
-        self.start_page = start_page
-        self.debug_dir = debug_dir
+        self.start_page = start_page or self.DEFAULT_START_PAGE
+        self.debug_dir = debug_dir or os.getcwd()
 
         # Create the playwright instance
         launch_args = {"headless": headless}

autogen/agentchat/contrib/orchestrator.py

@@ -0,0 +1,356 @@
+# ruff: noqa: E722
+import json
+import traceback
+import copy
+import sys
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Union, Callable, Literal, Tuple
+from autogen import Agent, ConversableAgent, GroupChatManager, GroupChat, OpenAIWrapper
+from autogen.code_utils import extract_code
+
+
+class Orchestrator(ConversableAgent):
+    def __init__(
+        self,
+        name: str,
+        agents: List[ConversableAgent] = [],
+        is_termination_msg: Optional[Callable[[Dict], bool]] = None,
+        max_consecutive_auto_reply: Optional[int] = None,
+        human_input_mode: Optional[str] = "TERMINATE",
+        function_map: Optional[Dict[str, Callable]] = None,
+        code_execution_config: Union[Dict, Literal[False]] = False,
+        llm_config: Optional[Union[Dict, Literal[False]]] = False,
+        default_auto_reply: Optional[Union[str, Dict, None]] = "",
+    ):
+        super().__init__(
+            name=name,
+            system_message="",
+            is_termination_msg=is_termination_msg,
+            max_consecutive_auto_reply=max_consecutive_auto_reply,
+            human_input_mode=human_input_mode,
+            function_map=function_map,
+            code_execution_config=code_execution_config,
+            llm_config=llm_config,
+            default_auto_reply=default_auto_reply,
+        )
+
+        self._agents = agents
+
+        self.orchestrated_messages = []
+
+        # NOTE: Async reply functions are not yet supported with this contrib agent
+        self._reply_func_list = []
+        self.register_reply([Agent, None], Orchestrator.run_chat)
+        self.register_reply([Agent, None], ConversableAgent.generate_code_execution_reply)
+        self.register_reply([Agent, None], ConversableAgent.generate_function_call_reply)
+        self.register_reply([Agent, None], ConversableAgent.check_termination_and_human_reply)
+
+    def _print_thought(self, message):
+        print(self.name + " (thought)\n")
+        print(message.strip() + "\n")
+        print("\n", "-" * 80, flush=True, sep="")
+
+    def _broadcast(self, message, out_loud=[], exclude=[]):
+        m = copy.deepcopy(message)
+        m["role"] = "user"
+        for a in self._agents:
+            if a in exclude or a.name in exclude:
+                continue
+            if a in out_loud or a.name in out_loud:
+                self.send(message, a, request_reply=False, silent=False)
+            else:
+                self.send(message, a, request_reply=False, silent=True)
+
+    def run_chat(
+        self,
+        messages: Optional[List[Dict]] = None,
+        sender: Optional[Agent] = None,
+        config: Optional[OpenAIWrapper] = None,
+    ) -> Tuple[bool, Union[str, Dict, None]]:
+        # We should probably raise an error in this case.
+        if self.client is None:
+            return False, None
+
+        if messages is None:
+            messages = self._oai_messages[sender]
+
+        # Work with a copy of the messages
+        _messages = copy.deepcopy(messages)
+
+        ##### Memory ####
+
+        # Pop the last message, which is the task
+        task = _messages.pop()["content"]
+
+        # A reusable description of the team
+        team = "\n".join([a.name + ": " + a.description for a in self._agents])
+        names = ", ".join([a.name for a in self._agents])
+
+        execution_enabled_agents = [a for a in self._agents if isinstance(a._code_execution_config, dict)]
+
+        # A place to store relevant facts
+        facts = ""
+
+        # A place to store the plan
+        plan = ""
+
+        #################
+
+        # Start by writing what we know
+        closed_book_prompt = f"""Below I will present you a request. Before we begin addressing the request, please answer the following pre-survey to the best of your ability. Keep in mind that you are Ken Jennings-level with trivia, and Mensa-level with puzzles, so there should be a deep well to draw from.
+
+Here is the request:
+
+{task}
+
+Here is the pre-survey:
+
+    1. Please list any specific facts or figures that are GIVEN in the request itself. It is possible that there are none.
+    2. Please list any facts that may need to be looked up, and WHERE SPECIFICALLY they might be found. In some cases, authoritative sources are mentioned in the request itself.
+    3. Please list any facts that may need to be derived (e.g., via logical deduction, simulation, or computation)
+    4. Please list any facts that are recalled from memory, hunches, well-reasoned guesses, etc.
+
+When answering this survey, keep in mind that "facts" will typically be specific names, dates, statistics, etc. Your answer should use headings:
+
+    1. GIVEN OR VERIFIED FACTS
+    2. FACTS TO LOOK UP
+    3. FACTS TO DERIVE
+    4. EDUCATED GUESSES
+""".strip()
+
+        _messages.append({"role": "user", "content": closed_book_prompt, "name": sender.name})
+
+        response = self.client.create(
+            messages=_messages,
+            cache=self.client_cache,
+        )
+        extracted_response = self.client.extract_text_or_completion_object(response)[0]
+        _messages.append({"role": "assistant", "content": extracted_response, "name": self.name})
+        facts = extracted_response
+
+        # Make an initial plan
+        plan_prompt = f"""Fantastic. To address this request we have assembled the following team:
+
+{team}
+
+Based on the team composition, and known and unknown facts, please devise a short bullet-point plan for addressing the original request. Remember, there is no requirement to involve all team members -- a team member's particular expertise may not be needed for this task.""".strip()
+        _messages.append({"role": "user", "content": plan_prompt, "name": sender.name})
+
+        response = self.client.create(
+            messages=_messages,
+            cache=self.client_cache,
+        )
+
+        extracted_response = self.client.extract_text_or_completion_object(response)[0]
+        _messages.append({"role": "assistant", "content": extracted_response, "name": self.name})
+        plan = extracted_response
+
+        # Main loop
+        total_turns = 0
+        max_turns = 30
+        while total_turns < max_turns:
+
+            # Populate the message histories
+            self.orchestrated_messages = []
+            for a in self._agents:
+                a.reset()
+
+            self.orchestrated_messages.append(
+                {
+                    "role": "assistant",
+                    "content": f"""
+We are working to address the following user request:
+
+{task}
+
+
+To answer this request we have assembled the following team:
+
+{team}
+
+Some additional points to consider:
+
+{facts}
+
+{plan}
+""".strip(),
+                    "name": self.name,
+                }
+            )
+            self._broadcast(self.orchestrated_messages[-1])
+            self._print_thought(self.orchestrated_messages[-1]["content"])
+
+            # Inner loop
+            stalled_count = 0
+            while total_turns < max_turns:
+                total_turns += 1
+
+                prev_message = self.orchestrated_messages[-1]["content"]
+                code_blocks = [t for t in extract_code(prev_message) if t[0] in ["python", "sh"]]
+
+                data = None
+                if len(code_blocks) > 0 and len(execution_enabled_agents) > 0:
+                    step_prompt = f"""
+Recall we are working on the following request:
+
+{task}
+
+To make progress on the request, please answer the following questions, including necessary reasoning:
+
+    - Is the request fully satisfied? (True if complete, or False if the original request has yet to be SUCCESSFULLY addressed)
+    - Are we making forward progress? (True if just starting, or recent messages are adding value. False if recent messages show evidence of being stuck in a reasoning or action loop, or there is evidence of significant barriers to success such as the inability to read from a required file)
+
+Please output an answer in pure JSON format according to the following schema. The JSON object must be parsable as-is. DO NOT OUTPUT ANYTHING OTHER THAN JSON, AND DO NOT DEVIATE FROM THIS SCHEMA:
+
+    {{
+        "is_request_satisfied": {{
+            "reason": string,
+            "answer": boolean
+        }},
+        "is_progress_being_made": {{
+            "reason": string,
+            "answer": boolean
+        }}
+    }}
+""".strip()
+
+                    # This is a temporary message we will immediately pop
+                    self.orchestrated_messages.append({"role": "user", "content": step_prompt, "name": sender.name})
+                    response = self.client.create(
+                        messages=self.orchestrated_messages,
+                        cache=self.client_cache,
+                        response_format={"type": "json_object"},
+                    )
+                    self.orchestrated_messages.pop()
+
+                    extracted_response = self.client.extract_text_or_completion_object(response)[0]
+                    try:
+                        data = json.loads(extracted_response)
+                        data["next_speaker"] = {
+                            "reason": "Assigning to an agent that can execute the script.",
+                            "answer": execution_enabled_agents[0].name,
+                        }
+                        data["instruction_or_question"] = {
+                            "reason": "Assigning to an agent that can execute the script.",
+                            "answer": "Please execute the above script.",
+                        }
+                    except json.decoder.JSONDecodeError as e:
+                        # Something went wrong. Restart this loop.
+                        self._print_thought(str(e))
+                        break
+                else:
+                    step_prompt = f"""
+Recall we are working on the following request:
+
+{task}
+
+And we have assembled the following team:
+
+{team}
+
+To make progress on the request, please answer the following questions, including necessary reasoning:
+
+    - Is the request fully satisfied? (True if complete, or False if the original request has yet to be SUCCESSFULLY addressed)
+    - Are we making forward progress? (True if just starting, or recent messages are adding value. False if recent messages show evidence of being stuck in a reasoning or action loop, or there is evidence of significant barriers to success such as the inability to read from a required file)
+    - Who should speak next? (select from: {names})
+    - What instruction or question would you give this team member? (Phrase as if speaking directly to them, and include any specific information they may need)
+
+Please output an answer in pure JSON format according to the following schema. The JSON object must be parsable as-is. DO NOT OUTPUT ANYTHING OTHER THAN JSON, AND DO NOT DEVIATE FROM THIS SCHEMA:
+
+    {{
+        "is_request_satisfied": {{
+            "reason": string,
+            "answer": boolean
+        }},
+        "is_progress_being_made": {{
+            "reason": string,
+            "answer": boolean
+        }},
+        "next_speaker": {{
+            "reason": string,
+            "answer": string (select from: {names})
+        }},
+        "instruction_or_question": {{
+            "reason": string,
+            "answer": string
+        }}
+    }}
+""".strip()
+                    # This is a temporary message we will immediately pop
+                    self.orchestrated_messages.append({"role": "user", "content": step_prompt, "name": sender.name})
+                    response = self.client.create(
+                        messages=self.orchestrated_messages,
+                        cache=self.client_cache,
+                        response_format={"type": "json_object"},
+                    )
+                    self.orchestrated_messages.pop()
+
+                    extracted_response = self.client.extract_text_or_completion_object(response)[0]
+                    try:
+                        data = json.loads(extracted_response)
+                    except json.decoder.JSONDecodeError as e:
+                        # Something went wrong. Restart this loop.
+                        self._print_thought(str(e))
+                        break
+
+                self._print_thought(json.dumps(data, indent=4))
+
+                if data["is_request_satisfied"]["answer"]:
+                    return True, "TERMINATE"
+
+                if data["is_progress_being_made"]["answer"]:
+                    stalled_count -= 1
+                    stalled_count = max(stalled_count, 0)
+                else:
+                    stalled_count += 1
+
+                if stalled_count >= 3:
+                    self._print_thought("We aren't making progress. Let's reset.")
+                    new_facts_prompt = f"""It's clear we aren't making as much progress as we would like, but we may have learned something new. Please rewrite the following fact sheet, updating it to include anything new we have learned. This is also a good time to update educated guesses (please add or update at least one educated guess or hunch, and explain your reasoning).
+
+{facts}
+""".strip()
+                    self.orchestrated_messages.append(
+                        {"role": "user", "content": new_facts_prompt, "name": sender.name}
+                    )
+                    response = self.client.create(
+                        messages=self.orchestrated_messages,
+                        cache=self.client_cache,
+                    )
+                    facts = self.client.extract_text_or_completion_object(response)[0]
+                    self.orchestrated_messages.append({"role": "assistant", "content": facts, "name": self.name})
+
+                    new_plan_prompt = f"""Please come up with a new plan expressed in bullet points. Keep in mind the following team composition, and do not involve any other outside people in the plan -- we cannot contact anyone else.
+
+Team membership:
+{team}
+""".strip()
+                    self.orchestrated_messages.append({"role": "user", "content": new_plan_prompt, "name": sender.name})
+                    response = self.client.create(
+                        messages=self.orchestrated_messages,
+                        cache=self.client_cache,
+                    )
+
+                    plan = self.client.extract_text_or_completion_object(response)[0]
+                    break
+
+                # Broadcast the message to all agents
+                m = {"role": "user", "content": data["instruction_or_question"]["answer"], "name": self.name}
+                if m["content"] is None:
+                    m["content"] = ""
+                self._broadcast(m, out_loud=[data["next_speaker"]["answer"]])
+
+                # Keep a copy
+                m["role"] = "assistant"
+                self.orchestrated_messages.append(m)
+
+                # Request a reply
+                for a in self._agents:
+                    if a.name == data["next_speaker"]["answer"]:
+                        reply = {"role": "user", "name": a.name, "content": a.generate_reply(sender=self)}
+                        self.orchestrated_messages.append(reply)
+                        a.send(reply, self, request_reply=False)
+                        self._broadcast(reply, exclude=[a])
+                        break
+
+        return True, "TERMINATE"

samples/apps/try_orc/README.md

@@ -0,0 +1,34 @@
+# GAIA Orchestrator Agent Demo
+
+This repository contains a sample application designed to demonstrate the capabilities of the orchestrator developed for the GAIA benchmark.
+
+## Getting Started
+
+These instructions will guide you through the process of setting up and running the application.
+
+### Prerequisites
+
+The application should be run in a virtual environment to ensure that its dependencies do not interfere with any existing Python packages you may have. You can use any virtual environment manager you prefer, such as venv or conda.
+
+### Installation
+
+1. Clone this repository to your local machine.
+2. Navigate to the `try_orc` directory within the cloned repository:
+
+    ```bash
+    cd try_orc
+    ```
+
+3. Install the required Python packages:
+
+    ```bash
+    pip install -r requirements.txt
+    ```
+
+### Running the Application
+
+Once you've installed the required packages, you can run the application with the following command:
+
+```bash
+python main.py
+```

samples/apps/try_orc/config_manager.py

@@ -0,0 +1,61 @@
+import os
+from typing import Optional, List, Dict
+
+import autogen
+
+
+class ConfigManager:
+
+    DEFAULT_LLM_MODEL = "gpt-4-turbo"
+    DEFAULT_MLM_MODEL = "gpt-4-1106-vision-preview"
+    DEFAULT_TIMEOUT = 300
+
+    def __init__(self):
+        self.llm_config = None
+        self.client = None
+
+        self.mlm_config = None
+        self.mlm_client = None
+
+        self.bing_api_key = None
+
+    def _get_config_list(self, config_path_or_env: Optional[str] = None) -> List[Dict[str, str]]:
+        config_list = None
+
+        try:
+            config_list = autogen.config_list_from_json(config_path_or_env)
+        except Exception as e:  # config list may not be available
+            api_key = os.environ.get("OPENAI_API_KEY", None)
+            if api_key is None:
+                raise Exception("No config list or OPENAI_API_KEY found", e)
+
+            config_list = [
+                {"model": self.DEFAULT_LLM_MODEL, "api_key": api_key, "tags": ["llm"]},
+                {"model": self.DEFAULT_MLM_MODEL, "api_key": api_key, "tags": ["mlm"]},
+            ]
+        return config_list
+
+    def _get_bing_api_key(self) -> str:
+        bing_api_key = os.environ.get("BING_API_KEY", None)
+        if bing_api_key is None:
+            raise Exception("Please set BING_API_KEY environment variable.")
+        return bing_api_key
+
+    def initialize(self, config_path_or_env: Optional[str] = "OAI_CONFIG_LIST") -> None:
+
+        config_list = self._get_config_list(config_path_or_env)
+
+        llm_config_list = autogen.filter_config(config_list, {"tags": ["llm"]})
+        assert len(llm_config_list) > 0, "No API key with 'llm' tag found in config list."
+
+        mlm_config_list = autogen.filter_config(config_list, {"tags": ["mlm"]})
+        assert len(mlm_config_list) > 0, "No API key with 'mlm' tag found in config list."
+
+        self.llm_config = {"config_list": llm_config_list, "timeout": self.DEFAULT_TIMEOUT, "temperature": 0.1}
+
+        self.mlm_config = {"config_list": mlm_config_list, "timeout": self.DEFAULT_TIMEOUT, "temperature": 0.1}
+
+        self.client = autogen.OpenAIWrapper(**self.llm_config)
+        self.mlm_client = autogen.OpenAIWrapper(**self.mlm_config)
+
+        self.bing_api_key = self._get_bing_api_key()

samples/apps/try_orc/main.py

@@ -0,0 +1,69 @@
+import autogen
+from autogen.agentchat.contrib.web_surfer import WebSurferAgent
+from autogen.browser_utils import RequestsMarkdownBrowser, BingMarkdownSearch
+
+from autogen.agentchat.contrib.orchestrator import Orchestrator
+
+from config_manager import ConfigManager
+from misc_utils import response_preparer
+
+# setup LLM config and clients
+config_manager = ConfigManager()
+config_manager.initialize()
+
+assistant = autogen.AssistantAgent(
+    "assistant",
+    is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
+    code_execution_config=False,
+    llm_config=config_manager.llm_config,
+)
+
+user_proxy_name = "computer_terminal"
+user_proxy = autogen.UserProxyAgent(
+    user_proxy_name,
+    human_input_mode="NEVER",
+    description="A computer terminal that performs no other action than running Python scripts (provided to it quoted in ```python code blocks), or sh shell scripts (provided to it quoted in ```sh code blocks)",
+    is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
+    code_execution_config={
+        "work_dir": "coding",
+        "use_docker": False,
+    },
+    default_auto_reply=f'Invalid {user_proxy_name} input: no code block detected.\nPlease provide {user_proxy_name} a complete Python script or a shell (sh) script to run. Scripts should appear in code blocks beginning "```python" or "```sh" respectively.',
+    max_consecutive_auto_reply=15,
+)
+
+browser = RequestsMarkdownBrowser(
+    viewport_size=1024 * 5,
+    downloads_folder="coding",
+    search_engine=BingMarkdownSearch(bing_api_key=config_manager.bing_api_key, interleave_results=False),
+)
+
+web_surfer = WebSurferAgent(
+    "web_surfer",
+    llm_config=config_manager.llm_config,
+    summarizer_llm_config=config_manager.llm_config,
+    is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
+    code_execution_config=False,
+    browser=browser,
+)
+
+maestro = Orchestrator(
+    "orchestrator",
+    agents=[assistant, user_proxy, web_surfer],
+    llm_config=config_manager.llm_config,
+)
+
+# read the task from standard input
+task = input("Enter the task: ")
+# task = "Find 10 highest cited publications written by Gagan Bansal"
+
+user_proxy.initiate_chat(maestro, message=task, clear_history=True)
+
+
+final_response = response_preparer(
+    inner_messages=maestro.chat_messages[user_proxy],
+    PROMPT=task,
+    client=assistant.client,
+)
+
+print(final_response)

samples/apps/try_orc/misc_utils.py

@@ -0,0 +1,77 @@
+import re
+import copy
+from typing import List, Dict
+
+import autogen
+
+
+def response_preparer(
+    inner_messages: List[Dict[str, str]],
+    PROMPT: str,
+    client: autogen.OpenAIWrapper,
+) -> str:
+
+    messages = [
+        {
+            "role": "user",
+            "content": f"""Earlier you were asked the following:
+
+{PROMPT}
+
+Your team then worked diligently to address that request. Here is a transcript of that conversation:""",
+        }
+    ]
+
+    for message in inner_messages:
+        if not message.get("content"):
+            continue
+        message = copy.deepcopy(message)
+        message["role"] = "user"
+        messages.append(message)
+
+    # ask for the final answer
+    messages.append(
+        {
+            "role": "user",
+            "content": f"""
+Read the above conversation and output a FINAL ANSWER to the question. The question is repeated here for convenience:
+
+{PROMPT}
+
+""",
+        }
+    )
+
+    response = client.create(context=None, messages=messages)
+    if "finish_reason='content_filter'" in str(response):
+        raise Exception(str(response))
+    extracted_response = client.extract_text_or_completion_object(response)[0]
+
+    # No answer
+    if "unable to determine" in extracted_response.lower():
+        print("\n>>>Making an educated guess.\n")
+        messages.append({"role": "assistant", "content": extracted_response})
+        messages.append(
+            {
+                "role": "user",
+                "content": """
+I understand that a definitive answer could not be determined. Please make a well-informed EDUCATED GUESS based on the conversation.
+
+To output the educated guess, use the following template: EDUCATED GUESS: [YOUR EDUCATED GUESS]
+Your EDUCATED GUESS should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. DO NOT OUTPUT 'I don't know', 'Unable to determine', etc.
+ADDITIONALLY, your EDUCATED GUESS MUST adhere to any formatting instructions specified in the original question (e.g., alphabetization, sequencing, units, rounding, decimal places, etc.)
+If you are asked for a number, express it numerically (i.e., with digits rather than words), don't use commas, and don't include units such as $ or percent signs unless specified otherwise.
+If you are asked for a string, don't use articles or abbreviations (e.g. for cities), unless specified otherwise. Don't output any final sentence punctuation such as '.', '!', or '?'.
+If you are asked for a comma separated list, apply the above rules depending on whether the elements are numbers or strings.
+""".strip(),
+            }
+        )
+
+        response = client.create(context=None, messages=messages)
+        if "finish_reason='content_filter'" in str(response):
+            raise Exception(str(response))
+        extracted_response = client.extract_text_or_completion_object(response)[0]
+
+        return re.sub(r"EDUCATED GUESS:", "FINAL ANSWER:", extracted_response)
+    else:
+        return extracted_response

samples/apps/try_orc/mm_basic.py

@@ -0,0 +1,40 @@
+import os
+from autogen.agentchat.contrib.multimodal_web_surfer import MultimodalWebSurferAgent
+from autogen.agentchat.contrib.mmagent import MultimodalAgent
+
+from config_manager import ConfigManager
+
+config = ConfigManager()
+config.initialize()
+
+
+web_surfer = MultimodalWebSurferAgent(
+    "web_surfer",
+    llm_config=config.llm_config,
+    is_termination_msg=lambda x: str(x).find("TERMINATE") >= 0 or str(x).find("FINAL ANSWER") >= 0,
+    human_input_mode="NEVER",
+    headless=True,
+    chromium_channel="chromium",
+    chromium_data_dir=None,
+    start_page="https://bing.com",
+    debug_dir=os.path.join(os.path.dirname(__file__), "debug"),
+)
+
+user_proxy = MultimodalAgent(
+    "user_proxy",
+    system_message="""You are a general-purpose AI assistant and can handle many questions -- but you don't have access to a web browser. However, the user you are talking to does have a browser, and you can see the screen. Provide short direct instructions to them to take you where you need to go to answer the initial question posed to you.
+
+Once the user has taken the final necessary action to complete the task, and you have fully addressed the initial request, reply with the word TERMINATE.""",
+    llm_config=config.llm_config,
+    human_input_mode="NEVER",
+    is_termination_msg=lambda x: False,
+    max_consecutive_auto_reply=20,
+)
+
+task = input("Enter the task: ")
+
+web_surfer.initiate_chat(
+    user_proxy,
+    message=task,
+    clear_history=True,
+)

samples/apps/try_orc/requirements.txt

@@ -0,0 +1,10 @@
+../../../../autogen
+pathvalidate
+markdownify
+puremagic
+binaryornot
+mammoth
+python-pptx
+pandas
+pdfminer.six
+playwright

samples/tools/autogenbench/scenarios/WebArena/.gitignore

@@ -0,0 +1,2 @@
+Results
+ENV.json