Added a link to AutoGenBench

* Add initial example

* Add decorator for yt utils

* Rename submodule; Add ability to specify package version

* Add support for specifying pip package name

* Fix bugs in requires

* Add more file utils; improve nb

* Add a new template that uses two agents + function calling

* Improve decorator name

* Add decorator for secrets

* Improve the decorator

* Improve gpt4v function

* Fix documentation

* Improve functions and add tests

* Improve test

autogen/agentchat/contrib/functions/file_utils.py

@@ -0,0 +1,229 @@
+from typing import Optional
+from .functions_utils import FunctionWithRequirements
+
+
+@FunctionWithRequirements(python_packages=["pdfminer.six", "requests"])
+def read_text_from_pdf(file_path: str) -> str:
+    """
+    Reads text from a PDF file and returns it as a string.
+
+    Args:
+        file_path (str): The path to the PDF file.
+
+    Returns:
+        str: The extracted text from the PDF file.
+    """
+    import io
+    import requests
+    from pdfminer.high_level import PDFResourceManager, PDFPageInterpreter
+    from pdfminer.converter import TextConverter
+    from pdfminer.pdfpage import PDFPage
+
+    resource_manager = PDFResourceManager()
+    text_stream = io.StringIO()
+    converter = TextConverter(resource_manager, text_stream)
+    interpreter = PDFPageInterpreter(resource_manager, converter)
+
+    if file_path.startswith("http://") or file_path.startswith("https://"):
+        response = requests.get(file_path)
+        file = io.BytesIO(response.content)
+    else:
+        file = open(file_path, "rb")
+
+    for page in PDFPage.get_pages(file):
+        interpreter.process_page(page)
+
+    text = text_stream.getvalue()
+    converter.close()
+    text_stream.close()
+
+    return text
+
+
+@FunctionWithRequirements(python_packages=["python-docx"])
+def read_text_from_docx(file_path: str) -> str:
+    """
+    Reads text from a DOCX file and returns it as a string.
+
+    Args:
+        file_path (str): The path to the DOCX file.
+
+    Returns:
+        str: The extracted text from the DOCX file.
+    """
+    from docx import Document
+
+    doc = Document(file_path)
+    paragraphs = [p.text for p in doc.paragraphs]
+    text = "\n".join(paragraphs)
+
+    return text
+
+
+@FunctionWithRequirements(python_packages=["easyocr"])
+def read_text_from_image(file_path: str) -> str:
+    """
+    Reads text from an image file or URL and returns it as a string.
+
+    Warning: EasyOCR requires torch, which is slow to download and install.
+    TODO: is there a better way to handle large dependencies?
+
+    Args:
+        file_path (str): The path to the image file or URL.
+
+    Returns:
+        str: The extracted text from the image file or URL.
+    """
+    import easyocr
+
+    reader = easyocr.Reader(["en"])  # specify the language(s)
+    output = reader.readtext(file_path)
+    # The output is a list of tuples, each containing the coordinates of the text and the text itself.
+    # We join all the text pieces together to get the final text.
+    text = " ".join([item[1] for item in output])
+    return text
+
+
+@FunctionWithRequirements(python_packages=["python-pptx"])
+def read_text_from_pptx(file_path: str) -> str:
+    """
+    Reads text from a PowerPoint file and returns it as a string.
+
+    Args:
+        file_path (str): The path to the PowerPoint file.
+
+    Returns:
+        str: The extracted text from the PowerPoint file.
+    """
+    from pptx import Presentation
+
+    presentation = Presentation(file_path)
+    text = ""
+
+    slide_num = 0
+    for slide in presentation.slides:
+        slide_num += 1
+
+        text += f"\n\n<!-- Slide number: {slide_num} -->\n"
+
+        for shape in slide.shapes:
+            if shape.has_text_frame:
+                text += shape.text + " "
+
+        text = text.strip()
+
+    return text
+
+
+@FunctionWithRequirements(python_packages=["pandas", "openpyxl"])
+def read_text_from_xlsx(file_path: str) -> str:
+    """
+    Reads text from an Excel file and returns it as a string.
+
+    Args:
+        file_path (str): The path to the Excel file.
+
+    Returns:
+        str: The extracted text from the Excel file.
+    """
+    import pandas as pd
+
+    df = pd.read_excel(file_path)
+    text = df.to_string(index=False)
+
+    return text
+
+
+@FunctionWithRequirements(python_packages=["speechrecognition", "requests", "pydub"])
+def read_text_from_audio(file_path: str) -> str:
+    """
+    Reads text from an audio file or a URL and returns it as a string.
+
+    Args:
+        file_path (str): The path to the audio file or the URL.
+
+    Returns:
+        str: The extracted text from the audio file or the URL.
+    """
+    import requests
+    import speech_recognition as sr
+    import tempfile
+
+    recognizer = sr.Recognizer()
+
+    if file_path.startswith("http"):
+        response = requests.get(file_path)
+        with tempfile.NamedTemporaryFile(delete=True, suffix=".wav") as temp_audio:
+            temp_audio.write(response.content)
+            temp_audio.seek(0)
+            with sr.AudioFile(temp_audio.name) as source:
+                audio = recognizer.record(source)
+    else:
+        with sr.AudioFile(file_path) as source:
+            audio = recognizer.record(source)
+
+    text = recognizer.recognize_google(audio)
+
+    return text
+
+
+@FunctionWithRequirements(python_packages=["openai"], secrets=["OPENAI_API_KEY"])
+def caption_image_using_gpt4v(file_path_or_url: str, prompt: Optional[str] = None) -> str:
+    """
+    Generates a caption for an image using the GPT-4 Vision model from OpenAI.
+
+    Args:
+        file_path_or_url (str): The path to the image file or the URL.
+        prompt (str, optional): The prompt to use for generating the caption. Defaults to "What’s in this image?".
+
+
+    Returns:
+        str: The caption generated for the image.
+    """
+    import os
+    import base64
+    import openai
+    from openai import OpenAI
+
+    prompt = prompt or "What’s in this image?"
+    caption = ""
+
+    openai.api_key = os.environ["OPENAI_API_KEY"]
+    client = OpenAI()
+
+    # check if the file_path_or_url is a local file that exists
+    if os.path.exists(file_path_or_url):
+        image_path = file_path_or_url
+        with open(image_path, "rb") as image_file:
+            image_base64 = base64.b64encode(image_file.read()).decode("utf-8")
+            file_path_or_url = f"data:image/jpeg;base64,{image_base64}"
+
+    # check if the file_path_or_url is a URL
+    if (
+        file_path_or_url.startswith("http://")
+        or file_path_or_url.startswith("https://")
+        or file_path_or_url.startswith("data:")
+    ):
+        image_url = file_path_or_url
+        response = client.chat.completions.create(
+            model="gpt-4-vision-preview",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": prompt},
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": image_url,
+                            },
+                        },
+                    ],
+                }
+            ],
+            max_tokens=300,
+        )
+        caption = response.choices[0].message.content
+    else:
+        raise ValueError("Invalid file path or URL")
+    return caption

autogen/agentchat/contrib/functions/functions_utils.py

@@ -0,0 +1,71 @@
+import os
+import subprocess
+import sys
+import functools
+import pkg_resources
+from typing import List, Optional, Tuple
+
+
+class FunctionWithRequirements:
+    """Decorator class that adds requirements and setup functionality to a function."""
+
+    def __init__(self, python_packages: Optional[List[str]] = None, secrets: Optional[List[str]] = None):
+        self.python_packages = python_packages or []
+        self.secrets = secrets or []
+
+    def __call__(self, func: callable) -> callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            self.setup()
+            return func(*args, **kwargs)
+
+        wrapper.setup = self.setup
+        wrapper.get_requirements = self.get_requirements
+        return wrapper
+
+    def get_requirements(self) -> Tuple[List[str], List[str]]:
+        """Returns the Python packages and secrets required by the function."""
+        return self.python_packages, self.secrets
+
+    def setup(self) -> None:
+        """Installs the required Python packages and checks the required secrets."""
+        # Install Python packages
+        all_packages = {pkg: None if "==" not in pkg else pkg.split("==")[1] for pkg in self.python_packages}
+        for name, version in all_packages.items():
+            if "==" in name:
+                name, version = name.split("==")
+            print("requested package:", name, version)
+            try:
+                try:
+                    installed_package = pkg_resources.get_distribution(name)
+                except pkg_resources.DistributionNotFound:
+                    print(f"Package {name} not found")
+                    installed_package = None
+                    raise ImportError
+
+                print("found package", installed_package)
+                if version is not None and installed_package.parsed_version != pkg_resources.parse_version(version):
+                    print("Package mismatch detected")
+                    raise ImportError
+            except ImportError or pkg_resources.DistributionNotFound:
+                print(f"Installing {name}{'==' + version if version else ''}...")
+                subprocess.check_call(
+                    [
+                        sys.executable,
+                        "-m",
+                        "pip",
+                        "install",
+                        name + "==" + version if version else name,
+                        "--upgrade",
+                        "--quiet",
+                    ]
+                )
+            except Exception as e:
+                print(f"Error: {e}")
+
+        # Check secrets
+        for name in self.secrets:
+            if name not in os.environ:
+                raise EnvironmentError(f"Environment variable {name} is not set")
+            else:
+                print(f"Environment variable {name} is set")

autogen/agentchat/contrib/functions/youtube_utils.py

@@ -0,0 +1,29 @@
+from .functions_utils import FunctionWithRequirements
+
+
+@FunctionWithRequirements(python_packages=["youtube_transcript_api==0.6.0"])
+def get_youtube_transcript(youtube_link: str) -> str:
+    """
+    Gets the transcript of a YouTube video.
+
+    Args:
+        youtube_link (str): The link to the YouTube video.
+
+    Returns:
+        str: The transcript of the YouTube video.
+    """
+    from youtube_transcript_api import YouTubeTranscriptApi
+
+    # Extract video ID from the YouTube link
+    video_id = youtube_link.split("v=")[1]
+
+    try:
+        # Get the transcript for the video
+        transcript_list = YouTubeTranscriptApi.get_transcript(video_id)
+
+        # Combine all parts of the transcript into a single string
+        transcript = " ".join([part["text"] for part in transcript_list])
+
+        return transcript
+    except Exception as e:
+        return str(e)

autogen/agentchat/contrib/web_surfer.py

@@ -2,6 +2,7 @@ import json
 import copy
 import logging
 import re
+import time
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Union, Callable, Literal, Tuple
 from typing_extensions import Annotated
@@ -23,7 +24,7 @@ class WebSurferAgent(ConversableAgent):
         + datetime.now().date().isoformat()
     )
 
-    DEFAULT_DESCRIPTION = "A helpful assistant with access to a web browser. Ask them to perform web searches, open pages, navigate to Wikipedia, answer questions from pages, and or generate summaries."
+    DEFAULT_DESCRIPTION = "A helpful assistant with access to a web browser. Ask them to perform web searches, open pages, navigate to Wikipedia, download files, etc. Once on a desired page, ask them to answer questions by reading the page, generate summaries, find specific words or phrases on the page (ctrl+f), or even just scroll up or down in the viewport."
 
     def __init__(
         self,
@@ -124,7 +125,14 @@ class WebSurferAgent(ConversableAgent):
             current_page = self.browser.viewport_current_page
             total_pages = len(self.browser.viewport_pages)
 
+            address = self.browser.address
+            for i in range(len(self.browser.history)-2,-1,-1): # Start from the second last
+                if self.browser.history[i][0] == address:
+                    header += f"You previously visited this page {round(time.time() - self.browser.history[i][1])} seconds ago.\n"
+                    break
+
             header += f"Viewport position: Showing page {current_page+1} of {total_pages}.\n"
+
             return (header, self.browser.viewport)
 
         @self._user_proxy.register_for_execution()
@@ -163,6 +171,15 @@ class WebSurferAgent(ConversableAgent):
             header, content = _browser_state()
             return header.strip() + "\n=======================\n" + content
 
+        @self._user_proxy.register_for_execution()
+        @self._assistant.register_for_llm(
+            name="download_file", description="Download a file at a given URL and, if possible, return its text."
+        )
+        def _visit_page(url: Annotated[str, "The relative or absolute url of the file to be downloaded."]) -> str:
+            self.browser.visit_page(url)
+            header, content = _browser_state()
+            return header.strip() + "\n=======================\n" + content
+
         @self._user_proxy.register_for_execution()
         @self._assistant.register_for_llm(
             name="page_up",
@@ -183,14 +200,51 @@ class WebSurferAgent(ConversableAgent):
             header, content = _browser_state()
             return header.strip() + "\n=======================\n" + content
 
+        @self._user_proxy.register_for_execution()
+        @self._assistant.register_for_llm(
+            name="find_on_page_ctrl_f",
+            description="Scroll the viewport to the first occurrence of the search string. This is equivalent to Ctrl+F.",
+        )
+        def _find_on_page_ctrl_f(
+            search_string: Annotated[
+                str, "The string to search for on the page. This search string supports wildcards like '*'"
+            ]
+        ) -> str:
+            find_result = self.browser.find_on_page(search_string)
+            header, content = _browser_state()
+
+            if find_result is None:
+                return (
+                    header.strip()
+                    + "\n=======================\nThe search string '"
+                    + search_string
+                    + "' was not found on this page."
+                )
+            else:
+                return header.strip() + "\n=======================\n" + content
+
+        @self._user_proxy.register_for_execution()
+        @self._assistant.register_for_llm(
+            name="find_next",
+            description="Scroll the viewport to next occurrence of the search string.",
+        )
+        def _find_next() -> str:
+            find_result = self.browser.find_next()
+            header, content = _browser_state()
+
+            if find_result is None:
+                return header.strip() + "\n=======================\nThe search string was not found on this page."
+            else:
+                return header.strip() + "\n=======================\n" + content
+
         if self.summarization_client is not None:
 
             @self._user_proxy.register_for_execution()
             @self._assistant.register_for_llm(
-                name="answer_from_page",
+                name="read_page_and_answer",
                 description="Uses AI to read the page and directly answer a given question based on the content.",
             )
-            def _answer_from_page(
+            def _read_page_and_answer(
                 question: Annotated[Optional[str], "The question to directly answer."],
                 url: Annotated[Optional[str], "[Optional] The url of the page. (Defaults to the current page)"] = None,
             ) -> str:
@@ -198,18 +252,20 @@ class WebSurferAgent(ConversableAgent):
                     self.browser.visit_page(url)
 
                 # We are likely going to need to fix this later, but summarize only as many tokens that fit in the buffer
-                limit = 4096
-                try:
-                    limit = get_max_token_limit(self.summarizer_llm_config["config_list"][0]["model"])  # type: ignore[index]
-                except ValueError:
-                    pass  # limit is unknown
-                except TypeError:
-                    pass  # limit is unknown
-
-                if limit < 16000:
-                    logger.warning(
-                        f"The token limit ({limit}) of the WebSurferAgent.summarizer_llm_config, is below the recommended 16k."
-                    )
+#                limit = 4096
+#                try:
+#                    limit = get_max_token_limit(self.summarizer_llm_config["config_list"][0]["model"])  # type: ignore[index]
+#                except ValueError:
+#                    pass  # limit is unknown
+#                except TypeError:
+#                    pass  # limit is unknown
+#
+#                if limit < 16000:
+#                    logger.warning(
+#                        f"The token limit ({limit}) of the WebSurferAgent.summarizer_llm_config, is below the recommended 16k."
+#                    )
+
+                limit = 32000
 
                 buffer = ""
                 for line in re.split(r"([\r\n]+)", self.browser.page_content):
@@ -251,7 +307,7 @@ class WebSurferAgent(ConversableAgent):
                     Optional[str], "[Optional] The url of the page to summarize. (Defaults to current page)"
                 ] = None
             ) -> str:
-                return _answer_from_page(url=url, question=None)
+                return _read_page_and_answer(url=url, question=None)
 
     def generate_surfer_reply(
         self,

autogen/agentchat/groupchat.py

@@ -5,7 +5,6 @@ import sys
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional, Union, Tuple
 
-
 from ..code_utils import content_str
 from ..exception_utils import AgentNameConflict
 from .agent import Agent
@@ -58,7 +57,8 @@ class GroupChat:
         Must be supplied if `allowed_or_disallowed_speaker_transitions` is not None.
     - enable_clear_history: enable possibility to clear history of messages for agents manually by providing
         "clear history" phrase in user prompt. This is experimental feature.
-        See description of `GroupChatManager.clear_agents_history` function for more info.
+        See description of GroupChatManager.clear_agents_history function for more info.
+    - send_introductions: send a round of introductions at the start of the group chat, so agents know who they can speak to (default: False)
     """
 
     agents: List[Agent]
@@ -71,6 +71,7 @@ class GroupChat:
     allowed_or_disallowed_speaker_transitions: Optional[Dict] = None
     speaker_transitions_type: Optional[str] = None
     enable_clear_history: Optional[bool] = False
+    send_introductions: Optional[bool] = False
 
     _VALID_SPEAKER_SELECTION_METHODS = ["auto", "manual", "random", "round_robin"]
     _VALID_SPEAKER_TRANSITIONS_TYPE = ["allowed", "disallowed", None]
@@ -229,6 +230,16 @@ Then select the next role from {[agent.name for agent in agents]} to play. Only
             agents = self.agents
         return f"Read the above conversation. Then select the next role from {[agent.name for agent in agents]} to play. Only return the role."
 
+    def introductions_msg(self, agents: Optional[List[Agent]] = None) -> str:
+        """Return the system message for selecting the next speaker. This is always the *first* message in the context."""
+        if agents is None:
+            agents = self.agents
+
+        return f"""Hello everyone. We have assembled a great team today to answer questions and solve tasks. In attendance are:
+
+{self._participant_roles(agents)}
+"""
+
     def manual_select_speaker(self, agents: Optional[List[Agent]] = None) -> Union[Agent, None]:
         """Manually select the next speaker."""
         if agents is None:
@@ -535,6 +546,16 @@ class GroupChatManager(ConversableAgent):
         message = messages[-1]
         speaker = sender
         groupchat = config
+        send_introductions = getattr(groupchat, "send_introductions", False)
+
+        if send_introductions:
+            # Broadcast the intro
+            intro = groupchat.introductions_msg()
+            for agent in groupchat.agents:
+                self.send(intro, agent, request_reply=False, silent=True)
+            # NOTE: We do not also append to groupchat.messages,
+            # since groupchat handles its own introductions
+
         if self.client_cache is not None:
             for a in groupchat.agents:
                 a.previous_cache = a.client_cache
@@ -598,6 +619,16 @@ class GroupChatManager(ConversableAgent):
         message = messages[-1]
         speaker = sender
         groupchat = config
+        send_introductions = getattr(groupchat, "send_introductions", False)
+
+        if send_introductions:
+            # Broadcast the intro
+            intro = groupchat.introductions_msg()
+            for agent in groupchat.agents:
+                self.a_send(intro, agent, request_reply=False, silent=True)
+            # NOTE: We do not also append to groupchat.messages,
+            # since groupchat handles its own introductions
+
         if self.client_cache is not None:
             for a in groupchat.agents:
                 a.previous_cache = a.client_cache

autogen/browser_utils.py

@@ -1,30 +1,18 @@
+# ruff: noqa: E722
 import json
 import os
 import requests
 import re
-import markdownify
 import io
 import uuid
 import mimetypes
-from urllib.parse import urljoin, urlparse
-from bs4 import BeautifulSoup
+import time
+import pathlib
+import pathvalidate
+from urllib.parse import urljoin, urlparse, unquote, parse_qs
+from urllib.request import url2pathname
 from typing import Any, Dict, List, Optional, Union, Tuple
-
-# Optional PDF support
-IS_PDF_CAPABLE = False
-try:
-    import pdfminer
-    import pdfminer.high_level
-
-    IS_PDF_CAPABLE = True
-except ModuleNotFoundError:
-    pass
-
-# Other optional dependencies
-try:
-    import pathvalidate
-except ModuleNotFoundError:
-    pass
+from .mdconvert import MarkdownConverter, UnsupportedFormatException, FileConversionException
 
 
 class SimpleTextBrowser:
@@ -41,23 +29,27 @@ class SimpleTextBrowser:
         self.start_page: str = start_page if start_page else "about:blank"
         self.viewport_size = viewport_size  # Applies only to the standard uri types
         self.downloads_folder = downloads_folder
-        self.history: List[str] = list()
+        self.history: List[Tuple[str, float]] = list()
         self.page_title: Optional[str] = None
         self.viewport_current_page = 0
         self.viewport_pages: List[Tuple[int, int]] = list()
         self.set_address(self.start_page)
         self.bing_api_key = bing_api_key
         self.request_kwargs = request_kwargs
+        self._mdconvert = MarkdownConverter()
+        self._page_content: str = ""
 
-        self._page_content = ""
+        self._find_on_page_query: Union[str, None] = None
+        self._find_on_page_last_result: Union[int, None] = None  # Location of the last result
 
     @property
     def address(self) -> str:
         """Return the address of the current page."""
-        return self.history[-1]
+        return self.history[-1][0]
 
     def set_address(self, uri_or_path: str) -> None:
-        self.history.append(uri_or_path)
+        # TODO: Handle anchors
+        self.history.append((uri_or_path, time.time()))
 
         # Handle special URIs
         if uri_or_path == "about:blank":
@@ -65,12 +57,21 @@ class SimpleTextBrowser:
         elif uri_or_path.startswith("bing:"):
             self._bing_search(uri_or_path[len("bing:") :].strip())
         else:
-            if not uri_or_path.startswith("http:") and not uri_or_path.startswith("https:"):
-                uri_or_path = urljoin(self.address, uri_or_path)
-                self.history[-1] = uri_or_path  # Update the address with the fully-qualified path
+            if (
+                not uri_or_path.startswith("http:")
+                and not uri_or_path.startswith("https:")
+                and not uri_or_path.startswith("file:")
+            ):
+                if len(self.history) > 1:
+                    prior_address = self.history[-2][0]
+                    uri_or_path = urljoin(prior_address, uri_or_path)
+                    # Update the address with the fully-qualified path
+                    self.history[-1] = (uri_or_path, self.history[-1][1])
             self._fetch_page(uri_or_path)
 
         self.viewport_current_page = 0
+        self.find_on_page_query = None
+        self.find_on_page_viewport = None
 
     @property
     def viewport(self) -> str:
@@ -96,14 +97,86 @@ class SimpleTextBrowser:
     def page_up(self) -> None:
         self.viewport_current_page = max(self.viewport_current_page - 1, 0)
 
+    def find_on_page(self, query: str) -> Union[str, None]:
+        """Searches for the query from the current viewport forward, looping back to the start if necessary."""
+
+        # Did we get here via a previous find_on_page search with the same query?
+        # If so, map to find_next
+        if query == self._find_on_page_query and self.viewport_current_page == self._find_on_page_last_result:
+            return self.find_next()
+
+        # Ok it's a new search start from the current viewport
+        self._find_on_page_query = query
+        viewport_match = self._find_next_viewport(query, self.viewport_current_page)
+        if viewport_match is None:
+            self._find_on_page_last_result = None
+            return None
+        else:
+            self.viewport_current_page = viewport_match
+            self._find_on_page_last_result = viewport_match
+            return self.viewport
+
+    def find_next(self) -> None:
+        """Scroll to the next viewport that matches the query"""
+
+        if self._find_on_page_query is None:
+            return None
+
+        starting_viewport = self._find_on_page_last_result
+        if starting_viewport is None:
+            starting_viewport = 0
+        else:
+            starting_viewport += 1
+            if starting_viewport >= len(self.viewport_pages):
+                starting_viewport = 0
+
+        viewport_match = self._find_next_viewport(self._find_on_page_query, starting_viewport)
+        if viewport_match is None:
+            self._find_on_page_last_result = None
+            return None
+        else:
+            self.viewport_current_page = viewport_match
+            self._find_on_page_last_result = viewport_match
+            return self.viewport
+
+    def _find_next_viewport(self, query: str, starting_viewport: int) -> Union[int, None]:
+        """Search for matches between the starting viewport looping when reaching the end."""
+
+        if query is None:
+            return None
+
+        # Normalize the query, and convert to a regular expression
+        nquery = re.sub(r"\*", "__STAR__", query)
+        nquery = " " + (" ".join(re.split(r"\W+", nquery))).strip() + " "
+        nquery = nquery.replace(" __STAR__ ", "__STAR__ ")  # Merge isolated stars with prior word
+        nquery = nquery.replace("__STAR__", ".*").lower()
+
+        if nquery.strip() == "":
+            return None
+
+        idxs = list()
+        idxs.extend(range(starting_viewport, len(self.viewport_pages)))
+        idxs.extend(range(0, starting_viewport))
+
+        for i in idxs:
+            bounds = self.viewport_pages[i]
+            content = self.page_content[bounds[0] : bounds[1]]
+
+            # TODO: Remove markdown links and images
+            ncontent = " " + (" ".join(re.split(r"\W+", content))).strip().lower() + " "
+            if re.search(nquery, ncontent):
+                return i
+
+        return None
+
     def visit_page(self, path_or_uri: str) -> str:
         """Update the address, visit the page, and return the content of the viewport."""
         self.set_address(path_or_uri)
         return self.viewport
 
     def _split_pages(self) -> None:
-        # Split only regular pages
-        if not self.address.startswith("http:") and not self.address.startswith("https:"):
+        # Do not split search results
+        if self.address.startswith("bing:"):
             self.viewport_pages = [(0, len(self._page_content))]
             return
 
@@ -153,105 +226,106 @@ class SimpleTextBrowser:
     def _bing_search(self, query: str) -> None:
         results = self._bing_api_call(query)
 
+        def _prev_visit(url):
+            for i in range(len(self.history) - 1, -1, -1):
+                if self.history[i][0] == url:
+                    # Todo make this more human-friendly
+                    return f"You previously visited this page {round(time.time() - self.history[i][1])} seconds ago.\n"
+            return ""
+
         web_snippets: List[str] = list()
         idx = 0
-        for page in results["webPages"]["value"]:
-            idx += 1
-            web_snippets.append(f"{idx}. [{page['name']}]({page['url']})\n{page['snippet']}")
-            if "deepLinks" in page:
-                for dl in page["deepLinks"]:
-                    idx += 1
-                    web_snippets.append(
-                        f"{idx}. [{dl['name']}]({dl['url']})\n{dl['snippet'] if 'snippet' in dl else ''}"  # type: ignore[index]
-                    )
+        if "webPages" in results:
+            for page in results["webPages"]["value"]:
+                idx += 1
+                web_snippets.append(
+                    f"{idx}. [{page['name']}]({page['url']})\n{_prev_visit(page['url'])}{page['snippet']}"
+                )
+                if "deepLinks" in page:
+                    for dl in page["deepLinks"]:
+                        idx += 1
+                        web_snippets.append(
+                            f"{idx}. [{dl['name']}]({dl['url']})\n{_prev_visit(dl['url'])}{dl['snippet'] if 'snippet' in dl else ''}"
+                        )
 
         news_snippets = list()
         if "news" in results:
             for page in results["news"]["value"]:
                 idx += 1
-                news_snippets.append(f"{idx}. [{page['name']}]({page['url']})\n{page['description']}")
+                datePublished = ""
+                if "datePublished" in page:
+                    datePublished = "\nDate published: " + page["datePublished"].split("T")[0]
+                news_snippets.append(
+                    f"{idx}. [{page['name']}]({page['url']})\n{_prev_visit(page['url'])}{page['description']}{datePublished}"
+                )
+
+        video_snippets = list()
+        if "videos" in results:
+            for page in results["videos"]["value"]:
+                if not page["contentUrl"].startswith("https://www.youtube.com/watch?v="):
+                    continue
+                idx += 1
+                datePublished = ""
+                if "datePublished" in page:
+                    datePublished = "\nDate published: " + page["datePublished"].split("T")[0]
+                video_snippets.append(
+                    f"{idx}. [{page['name']}]({page['contentUrl']})\n{_prev_visit(page['contentUrl'])}{page['description']}{datePublished}"
+                )
 
         self.page_title = f"{query} - Search"
 
         content = (
-            f"A Bing search for '{query}' found {len(web_snippets) + len(news_snippets)} results:\n\n## Web Results\n"
+            f"A Bing search for '{query}' found {len(web_snippets) + len(news_snippets) + len(video_snippets)} results:\n\n## Web Results\n"
             + "\n\n".join(web_snippets)
         )
         if len(news_snippets) > 0:
             content += "\n\n## News Results:\n" + "\n\n".join(news_snippets)
+        if len(video_snippets) > 0:
+            content += "\n\n## Video Results:\n" + "\n\n".join(video_snippets)
+
         self._set_page_content(content)
 
     def _fetch_page(self, url: str) -> None:
+        download_path = ""
         try:
-            # Prepare the request parameters
-            request_kwargs = self.request_kwargs.copy() if self.request_kwargs is not None else {}
-            request_kwargs["stream"] = True
+            if url.startswith("file://"):
+                download_path = os.path.normcase(os.path.normpath(unquote(url[7:])))
+                res = self._mdconvert.convert_local(download_path)
+                self.page_title = res.title
+                self._set_page_content(res.text_content)
+            else:
+                # Prepare the request parameters
+                request_kwargs = self.request_kwargs.copy() if self.request_kwargs is not None else {}
+                request_kwargs["stream"] = True
 
-            # Send a HTTP request to the URL
-            response = requests.get(url, **request_kwargs)
-            response.raise_for_status()
+                # Send a HTTP request to the URL
+                response = requests.get(url, **request_kwargs)
+                response.raise_for_status()
 
-            # If the HTTP request returns a status code 200, proceed
-            if response.status_code == 200:
+                # If the HTTP request was successful
                 content_type = response.headers.get("content-type", "")
-                for ct in ["text/html", "text/plain", "application/pdf"]:
-                    if ct in content_type.lower():
-                        content_type = ct
-                        break
-
-                if content_type == "text/html":
-                    # Get the content of the response
-                    html = ""
-                    for chunk in response.iter_content(chunk_size=512, decode_unicode=True):
-                        html += chunk
-
-                    soup = BeautifulSoup(html, "html.parser")
-
-                    # Remove javascript and style blocks
-                    for script in soup(["script", "style"]):
-                        script.extract()
-
-                    # Convert to markdown -- Wikipedia gets special attention to get a clean version of the page
-                    if url.startswith("https://en.wikipedia.org/"):
-                        body_elm = soup.find("div", {"id": "mw-content-text"})
-                        title_elm = soup.find("span", {"class": "mw-page-title-main"})
-
-                        if body_elm:
-                            # What's the title
-                            main_title = soup.title.string
-                            if title_elm and len(title_elm) > 0:
-                                main_title = title_elm.string
-                            webpage_text = (
-                                "# " + main_title + "\n\n" + markdownify.MarkdownConverter().convert_soup(body_elm)
-                            )
-                        else:
-                            webpage_text = markdownify.MarkdownConverter().convert_soup(soup)
-                    else:
-                        webpage_text = markdownify.MarkdownConverter().convert_soup(soup)
-
-                    # Convert newlines
-                    webpage_text = re.sub(r"\r\n", "\n", webpage_text)
-
-                    # Remove excessive blank lines
-                    self.page_title = soup.title.string
-                    self._set_page_content(re.sub(r"\n{2,}", "\n\n", webpage_text).strip())
-                elif content_type == "text/plain":
-                    # Get the content of the response
-                    plain_text = ""
-                    for chunk in response.iter_content(chunk_size=512, decode_unicode=True):
-                        plain_text += chunk
-
-                    self.page_title = None
-                    self._set_page_content(plain_text)
-                elif IS_PDF_CAPABLE and content_type == "application/pdf":
-                    pdf_data = io.BytesIO(response.raw.read())
-                    self.page_title = None
-                    self._set_page_content(pdfminer.high_level.extract_text(pdf_data))
-                elif self.downloads_folder is not None:
+
+                # Text or HTML
+                if "text/" in content_type.lower():
+                    res = self._mdconvert.convert_response(response)
+                    self.page_title = res.title
+                    self._set_page_content(res.text_content)
+                # A download
+                else:
                     # Try producing a safe filename
                     fname = None
+                    download_path = None
                     try:
                         fname = pathvalidate.sanitize_filename(os.path.basename(urlparse(url).path)).strip()
+                        download_path = os.path.abspath(os.path.join(self.downloads_folder, fname))
+
+                        suffix = 0
+                        while os.path.exists(download_path) and suffix < 1000:
+                            suffix += 1
+                            base, ext = os.path.splitext(fname)
+                            new_fname = f"{base}__{suffix}{ext}"
+                            download_path = os.path.abspath(os.path.join(self.downloads_folder, new_fname))
+
                     except NameError:
                         pass
 
@@ -261,22 +335,130 @@ class SimpleTextBrowser:
                         if extension is None:
                             extension = ".download"
                         fname = str(uuid.uuid4()) + extension
+                        download_path = os.path.abspath(os.path.join(self.downloads_folder, fname))
 
                     # Open a file for writing
-                    download_path = os.path.abspath(os.path.join(self.downloads_folder, fname))
                     with open(download_path, "wb") as fh:
                         for chunk in response.iter_content(chunk_size=512):
                             fh.write(chunk)
 
-                    # Return a page describing what just happened
-                    self.page_title = "Download complete."
-                    self._set_page_content(f"Downloaded '{url}' to '{download_path}'.")
-                else:
-                    self.page_title = f"Error - Unsupported Content-Type '{content_type}'"
-                    self._set_page_content(self.page_title)
+                    # Render it
+                    local_uri = pathlib.Path(download_path).as_uri()
+                    self.set_address(local_uri)
+
+        except UnsupportedFormatException:
+            self.page_title = ("Download complete.",)
+            self._set_page_content(f"# Download complete\n\nSaved file to '{download_path}'")
+        except FileConversionException:
+            self.page_title = ("Download complete.",)
+            self._set_page_content(f"# Download complete\n\nSaved file to '{download_path}'")
+        except FileNotFoundError:
+            self.page_title = "Error 404"
+            self._set_page_content(f"## Error 404\n\nFile not found: {download_path}")
+        except requests.exceptions.RequestException:
+            self.page_title = f"Error {response.status_code}"
+
+            # If the error was rendered in HTML we might as well render it
+            content_type = response.headers.get("content-type", "")
+            if content_type is not None and "text/html" in content_type.lower():
+                res = self._mdconvert.convert(response)
+                self.page_title = f"Error {response.status_code}"
+                self._set_page_content(f"## Error {response.status_code}\n\n{res.text_content}")
             else:
-                self.page_title = "Error"
-                self._set_page_content("Failed to retrieve " + url)
-        except requests.exceptions.RequestException as e:
-            self.page_title = "Error"
-            self._set_page_content(str(e))
+                text = ""
+                for chunk in response.iter_content(chunk_size=512, decode_unicode=True):
+                    text += chunk
+                self.page_title = f"Error {response.status_code}"
+                self._set_page_content(f"## Error {response.status_code}\n\n{text}")
+
+
+# #https://stackoverflow.com/questions/10123929/fetch-a-file-from-a-local-url-with-python-requests
+# class LocalFileAdapter(requests.adapters.BaseAdapter):
+#     """Protocol Adapter to allow Requests to GET file:// URLs"""
+#
+#     @staticmethod
+#     def _chkpath(method, path):
+#         """Return an HTTP status for the given filesystem path."""
+#         if method.lower() in ("put", "delete"):
+#             return 501, "Not Implemented"
+#         elif method.lower() not in ("get", "head"):
+#             return 405, "Method Not Allowed"
+#         elif not os.path.exists(path):
+#             return 404, "File Not Found"
+#         elif not os.access(path, os.R_OK):
+#             return 403, "Access Denied"
+#         else:
+#             return 200, "OK"
+#
+#     def send(self, req, **kwargs):
+#         """Return the file specified by the given request"""
+#         path = os.path.normcase(os.path.normpath(url2pathname(req.path_url)))
+#         response = requests.Response()
+#
+#         response.status_code, response.reason = self._chkpath(req.method, path)
+#         if response.status_code == 200 and req.method.lower() != "head":
+#             try:
+#                 if os.path.isfile(path):
+#                     response.raw = open(path, "rb")
+#                 else:  # List the directory
+#                     response.headers["content-type"] = "text/html"
+#                     pardir = os.path.normpath(os.path.join(path, os.pardir))
+#                     pardir_uri = pathlib.Path(pardir).as_uri()
+#                     listing = f"""
+# <!DOCTYPE html>
+# <html>
+#   <head>
+#     <title>Index of {html.escape(path)}</title>
+#   </head>
+#   <body>
+#     <h1>Index of {html.escape(path)}</h1>
+#
+#     <a href="{html.escape(pardir_uri, quote=True)}">.. (parent directory)</a>
+#
+#     <table>
+#     <tr>
+#        <th>Name</th><th>Size</th><th>Date modified</th>
+#     </tr>
+# """
+#
+#                     for entry in os.listdir(path):
+#                         full_path = os.path.normpath(os.path.join(path, entry))
+#                         full_path_uri = pathlib.Path(full_path).as_uri()
+#                         size = ""
+#
+#                        if os.path.isdir(full_path):
+#                            entry = entry + os.path.sep
+#                        else:
+#                            size = str(os.path.getsize(full_path))
+#
+#                        listing += (
+#                            "<tr>\n"
+#                            + f'<td><a href="{html.escape(full_path_uri, quote=True)}">{html.escape(entry)}</a></td>'
+#                            + f"<td>{html.escape(size)}</td>"
+#                            + f"<td>{html.escape(entry)}</td>"
+#                            + "</tr>"
+#                        )
+#
+#                    listing += """
+#    </table>
+#  </body>
+# </html>
+# """
+#
+#                    response.raw = io.StringIO(listing)
+#            except (OSError, IOError) as err:
+#                response.status_code = 500
+#                response.reason = str(err)
+#
+#        if isinstance(req.url, bytes):
+#            response.url = req.url.decode("utf-8")
+#        else:
+#            response.url = req.url
+#
+#        response.request = req
+#        response.connection = self
+#
+#        return response
+#
+#    def close(self):
+#        pass

autogen/mdconvert.py

@@ -0,0 +1,760 @@
+# ruff: noqa: E722
+import json
+import os
+import requests
+import re
+import markdownify
+import io
+import uuid
+import mimetypes
+import html
+import pathlib
+import puremagic
+import tempfile
+import copy
+import mammoth
+import pptx
+import pydub
+import pandas as pd
+import speech_recognition as sr
+import sys
+import traceback
+
+import PIL
+import shutil
+import subprocess
+import easyocr
+import numpy as np
+
+import base64
+
+from urllib.parse import urljoin, urlparse, parse_qs
+from urllib.request import url2pathname
+from bs4 import BeautifulSoup
+from typing import Any, Dict, List, Optional, Union, Tuple
+
+# Optional PDF support
+IS_PDF_CAPABLE = False
+try:
+    import pdfminer
+    import pdfminer.high_level
+
+    IS_PDF_CAPABLE = True
+except ModuleNotFoundError:
+    pass
+
+# Optional YouTube transcription support
+IS_YOUTUBE_TRANSCRIPT_CAPABLE = False
+try:
+    from youtube_transcript_api import YouTubeTranscriptApi
+
+    IS_YOUTUBE_TRANSCRIPT_CAPABLE = True
+except ModuleNotFoundError:
+    pass
+
+
+class DocumentConverterResult:
+    """The result of converting a document to text."""
+
+    def __init__(self, title: Union[str, None] = None, text_content: str = ""):
+        self.title = title
+        self.text_content = text_content
+
+
+class DocumentConverter:
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        raise NotImplementedError()
+
+
+class PlainTextConverter(DocumentConverter):
+    """Anything with content type text/plain"""
+
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        extension = kwargs.get("file_extension", "")
+        if extension == "":
+            return None
+
+        content_type, encoding = mimetypes.guess_type("__placeholder" + extension)
+        if content_type is None:
+            return None
+
+        if "text/" not in content_type.lower():
+            return None
+
+        text_content = ""
+        with open(local_path, "rt") as fh:
+            text_content = fh.read()
+
+        return DocumentConverterResult(
+            title=None,
+            text_content=text_content,
+        )
+
+
+class HtmlConverter(DocumentConverter):
+    """Anything with content type text/html"""
+
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not html
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() not in [".html", ".htm"]:
+            return None
+
+        result = None
+        with open(local_path, "rt") as fh:
+            result = self._convert(fh.read())
+
+        return result
+
+    def _convert(self, html_content) -> Union[None, DocumentConverterResult]:
+        """Helper function that converts and HTML string."""
+
+        # Parse the string
+        soup = BeautifulSoup(html_content, "html.parser")
+
+        # Remove javascript and style blocks
+        for script in soup(["script", "style"]):
+            script.extract()
+
+        # Print only the main content
+        body_elm = soup.find("body")
+        webpage_text = ""
+        if body_elm:
+            webpage_text = markdownify.MarkdownConverter().convert_soup(body_elm)
+        else:
+            webpage_text = markdownify.MarkdownConverter().convert_soup(soup)
+
+        return DocumentConverterResult(
+            title=None if soup.title is None else soup.title.string,
+            text_content=webpage_text,
+        )
+
+
+class WikipediaConverter(DocumentConverter):
+    """Handle Wikipedia pages separately, focusing only on the main document content."""
+
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not Wikipedia
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() not in [".html", ".htm"]:
+            return None
+        url = kwargs.get("url", "")
+        if not re.search(r"^https?:\/\/[a-zA-Z]{2,3}\.wikipedia.org\/", url):
+            return None
+
+        # Parse the file
+        soup = None
+        with open(local_path, "rt") as fh:
+            soup = BeautifulSoup(fh.read(), "html.parser")
+
+        # Remove javascript and style blocks
+        for script in soup(["script", "style"]):
+            script.extract()
+
+        # Print only the main content
+        body_elm = soup.find("div", {"id": "mw-content-text"})
+        title_elm = soup.find("span", {"class": "mw-page-title-main"})
+
+        webpage_text = ""
+        if body_elm:
+            # What's the title
+            main_title = soup.title.string
+            if title_elm and len(title_elm) > 0:
+                main_title = title_elm.string
+
+            # Convert the page
+            webpage_text = "# " + main_title + "\n\n" + markdownify.MarkdownConverter().convert_soup(body_elm)
+        else:
+            webpage_text = markdownify.MarkdownConverter().convert_soup(soup)
+
+        return DocumentConverterResult(
+            title=soup.title.string,
+            text_content=webpage_text,
+        )
+
+
+class YouTubeConverter(DocumentConverter):
+    """Handle YouTube specially, focusing on the video title, description, and transcript."""
+
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not YouTube
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() not in [".html", ".htm"]:
+            return None
+        url = kwargs.get("url", "")
+        if not url.startswith("https://www.youtube.com/watch?"):
+            return None
+
+        # Parse the file
+        soup = None
+        with open(local_path, "rt") as fh:
+            soup = BeautifulSoup(fh.read(), "html.parser")
+
+        # Read the meta tags
+        metadata = {"title": soup.title.string}
+        for meta in soup(["meta"]):
+            for a in meta.attrs:
+                if a in ["itemprop", "property", "name"]:
+                    metadata[meta[a]] = meta.get("content", "")
+                    break
+
+        # We can also try to read the full description. This is more prone to breaking, since it reaches into the page implementation
+        try:
+            for script in soup(["script"]):
+                content = script.text
+                if "ytInitialData" in content:
+                    lines = re.split(r"\r?\n", content)
+                    obj_start = lines[0].find("{")
+                    obj_end = lines[0].rfind("}")
+                    if obj_start >= 0 and obj_end >= 0:
+                        data = json.loads(lines[0][obj_start : obj_end + 1])
+                        attrdesc = self._findKey(data, "attributedDescriptionBodyText")
+                        if attrdesc:
+                            metadata["description"] = attrdesc["content"]
+                    break
+        except:
+            pass
+
+        # Start preparing the page
+        webpage_text = "# YouTube\n"
+
+        title = self._get(metadata, ["title", "og:title", "name"])
+        if title:
+            webpage_text += f"\n## {title}\n"
+
+        stats = ""
+        views = self._get(metadata, ["interactionCount"])
+        if views:
+            stats += f"- **Views:** {views}\n"
+
+        keywords = self._get(metadata, ["keywords"])
+        if keywords:
+            stats += f"- **Keywords:** {keywords}\n"
+
+        runtime = self._get(metadata, ["duration"])
+        if runtime:
+            stats += f"- **Runtime:** {runtime}\n"
+
+        if len(stats) > 0:
+            webpage_text += f"\n### Video Metadata\n{stats}\n"
+
+        description = self._get(metadata, ["description", "og:description"])
+        if description:
+            webpage_text += f"\n### Description\n{description}\n"
+
+        if IS_YOUTUBE_TRANSCRIPT_CAPABLE:
+            transcript_text = ""
+            parsed_url = urlparse(url)
+            params = parse_qs(parsed_url.query)
+            if "v" in params:
+                video_id = params["v"][0]
+                try:
+                    # Must be a single transcript.
+                    transcript = YouTubeTranscriptApi.get_transcript(video_id)
+                    transcript_text = " ".join([part["text"] for part in transcript])
+                    # Alternative formatting:
+                    # formatter = TextFormatter()
+                    # formatter.format_transcript(transcript)
+                except:
+                    pass
+            if transcript_text:
+                webpage_text += f"\n### Transcript\n{transcript_text}\n"
+
+        return DocumentConverterResult(
+            title=title if title else soup.title.string,
+            text_content=webpage_text,
+        )
+
+    def _get(self, json, keys, default=None):
+        for k in keys:
+            if k in json:
+                return json[k]
+        return default
+
+    def _findKey(self, json, key):
+        if isinstance(json, list):
+            for elm in json:
+                ret = self._findKey(elm, key)
+                if ret is not None:
+                    return ret
+        elif isinstance(json, dict):
+            for k in json:
+                if k == key:
+                    return json[k]
+                else:
+                    ret = self._findKey(json[k], key)
+                    if ret is not None:
+                        return ret
+        return None
+
+
+class PdfConverter(DocumentConverter):
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a PDF
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() != ".pdf":
+            return None
+
+        return DocumentConverterResult(
+            title=None,
+            text_content=pdfminer.high_level.extract_text(local_path),
+        )
+
+
+class DocxConverter(HtmlConverter):
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a DOCX
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() != ".docx":
+            return None
+
+        result = None
+        with open(local_path, "rb") as docx_file:
+            result = mammoth.convert_to_html(docx_file)
+            html_content = result.value
+            result = self._convert(html_content)
+
+        return result
+
+
+class XlsxConverter(HtmlConverter):
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a XLSX
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() != ".xlsx":
+            return None
+
+        sheets = pd.read_excel(local_path, sheet_name=None)
+        md_content = ""
+        for s in sheets:
+            md_content += f"## {s}\n"
+            html_content = sheets[s].to_html(index=False)
+            md_content += self._convert(html_content).text_content.strip() + "\n\n"
+
+        return DocumentConverterResult(
+            title=None,
+            text_content=md_content.strip(),
+        )
+
+
+class PptxConverter(HtmlConverter):
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a PPTX
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() != ".pptx":
+            return None
+
+        md_content = ""
+
+        presentation = pptx.Presentation(local_path)
+        slide_num = 0
+        for slide in presentation.slides:
+            slide_num += 1
+
+            md_content += f"\n\n<!-- Slide number: {slide_num} -->\n"
+
+            title = slide.shapes.title
+            for shape in slide.shapes:
+                # Pictures
+                if self._is_picture(shape):
+                    # https://github.com/scanny/python-pptx/pull/512#issuecomment-1713100069
+                    alt_text = ""
+                    try:
+                        alt_text = shape._element._nvXxPr.cNvPr.attrib.get("descr", "")
+                    except:
+                        pass
+
+                    # A placeholder name
+                    filename = re.sub(r"\W", "", shape.name) + ".jpg"
+                    # try:
+                    #    filename = shape.image.filename
+                    # except:
+                    #    pass
+
+                    md_content += "\n![" + (alt_text if alt_text else shape.name) + "](" + filename + ")\n"
+
+                # Tables
+                if self._is_table(shape):
+                    html_table = "<html><body><table>"
+                    first_row = True
+                    for row in shape.table.rows:
+                        html_table += "<tr>"
+                        for cell in row.cells:
+                            if first_row:
+                                html_table += "<th>" + html.escape(cell.text) + "</th>"
+                            else:
+                                html_table += "<td>" + html.escape(cell.text) + "</td>"
+                        html_table += "</tr>"
+                        first_row = False
+                    html_table += "</table></body></html>"
+                    md_content += "\n" + self._convert(html_table).text_content.strip() + "\n"
+
+                # Text areas
+                elif shape.has_text_frame:
+                    if shape == title:
+                        md_content += "# " + shape.text.lstrip() + " "
+                    else:
+                        md_content += shape.text + " "
+
+            md_content = md_content.strip()
+
+            if slide.has_notes_slide:
+                md_content += "\n\n### Notes:\n"
+                notes_frame = slide.notes_slide.notes_text_frame
+                if notes_frame is not None:
+                    md_content += notes_frame.text
+                md_content = md_content.strip()
+
+        return DocumentConverterResult(
+            title=None,
+            text_content=md_content.strip(),
+        )
+
+    def _is_picture(self, shape):
+        if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.PICTURE:
+            return True
+        if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.PLACEHOLDER:
+            if hasattr(shape, "image"):
+                return True
+        return False
+
+    def _is_table(self, shape):
+        if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.TABLE:
+            return True
+        return False
+
+
+class WavConverter(DocumentConverter):
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a XLSX
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() != ".wav":
+            return None
+
+        recognizer = sr.Recognizer()
+        with sr.AudioFile(local_path) as source:
+            audio = recognizer.record(source)
+            text_content = recognizer.recognize_google(audio).strip()
+
+        return DocumentConverterResult(
+            title=None,
+            text_content="### Audio Transcript:\n" + ("[No speech detected]" if text_content == "" else text_content),
+        )
+
+
+class Mp3Converter(WavConverter):
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a MP3
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() != ".mp3":
+            return None
+
+        handle, temp_path = tempfile.mkstemp(suffix=".wav")
+        os.close(handle)
+        try:
+            sound = pydub.AudioSegment.from_mp3(local_path)
+            sound.export(temp_path, format="wav")
+
+            _args = dict()
+            _args.update(kwargs)
+            _args["file_extension"] = ".wav"
+
+            result = super().convert(temp_path, **_args)
+        finally:
+            os.unlink(temp_path)
+
+        return result
+
+
+class ImageConverter(DocumentConverter):
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a XLSX
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() not in [".jpg", ".jpeg", ".png"]:
+            return None
+
+        ocr_min_confidence = kwargs.get("ocr_min_confidence", 0.25)
+
+        md_content = ""
+
+        # Add metadata
+        metadata = self._get_metadata(local_path)
+        if metadata:
+            for f in [
+                "Title",
+                "Caption",
+                "Description",
+                "Keywords",
+                "Artist",
+                "DateTimeOriginal",
+                "CreateDate",
+                "GPSPosition",
+            ]:
+                if f in metadata:
+                    md_content += f"{f}: {metadata[f]}\n"
+
+        # Try describing the image with GPTV
+        mlm_client = kwargs.get("mlm_client")
+        if mlm_client is not None:
+            md_content += (
+                "\n# Description:\n"
+                + self._get_mlm_description(local_path, extension, mlm_client, prompt=kwargs.get("mlm_prompt")).strip()
+                + "\n"
+            )
+
+        image = PIL.Image.open(local_path)
+        # Remove transparency
+        if image.mode in ("RGBA", "P"):
+            image = image.convert("RGB")
+
+        reader = easyocr.Reader(["en"])  # specify the language(s)
+        output = reader.readtext(np.array(image))  # local_path)
+        # The output is a list of tuples, each containing the coordinates of the text and the text itself.
+        # We join all the text pieces together to get the final text.
+        ocr_text = " "
+        for item in output:
+            if item[2] >= ocr_min_confidence:
+                ocr_text += item[1] + " "
+        ocr_text = ocr_text.strip()
+
+        if len(ocr_text) > 0:
+            md_content += "\n# Text detected by OCR:\n" + ocr_text
+
+        return DocumentConverterResult(
+            title=None,
+            text_content=md_content,
+        )
+
+    def _get_metadata(self, local_path):
+        exiftool = shutil.which("exiftool")
+        if not exiftool:
+            return None
+        else:
+            try:
+                result = subprocess.run([exiftool, "-json", local_path], capture_output=True, text=True).stdout
+                return json.loads(result)[0]
+            except:
+                return None
+
+    def _get_mlm_description(self, local_path, extension, client, prompt=None):
+        if prompt is None or prompt.strip() == "":
+            prompt = "Write a detailed caption for this image."
+
+        sys.stderr.write(f"MLM Prompt:\n{prompt}\n")
+
+        data_uri = ""
+        with open(local_path, "rb") as image_file:
+            content_type, encoding = mimetypes.guess_type("_dummy" + extension)
+            if content_type is None:
+                content_type = "image/jpeg"
+            image_base64 = base64.b64encode(image_file.read()).decode("utf-8")
+            data_uri = f"data:{content_type};base64,{image_base64}"
+
+            messages = [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": prompt},
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": data_uri,
+                            },
+                        },
+                    ],
+                }
+            ]
+
+            response = client.create(messages=messages)
+            return client.extract_text_or_completion_object(response)[0]
+
+class FileConversionException(BaseException):
+    pass
+
+class UnsupportedFormatException(BaseException):
+    pass
+
+class MarkdownConverter:
+    """(In preview) An extremely simple text-based document reader, suitable for LLM use.
+    This reader will convert common file-types or webpages to Markdown."""
+
+    def __init__(
+        self,
+        requests_session: Optional[requests.Session] = None,
+        mlm_client: Optional[Any] = None,
+    ):
+        if requests_session is None:
+            self._requests_session = requests.Session()
+        else:
+            self._requests_session = requests_session
+
+        self._mlm_client = mlm_client
+
+        self._page_converters: List[DocumentConverter] = []
+
+        # Register converters for successful browsing operations
+        # Later registrations are tried first / take higher priority than earlier registrations
+        # To this end, the most specific converters should appear below the most generic converters
+        self.register_page_converter(PlainTextConverter())
+        self.register_page_converter(HtmlConverter())
+        self.register_page_converter(WikipediaConverter())
+        self.register_page_converter(YouTubeConverter())
+        self.register_page_converter(DocxConverter())
+        self.register_page_converter(XlsxConverter())
+        self.register_page_converter(PptxConverter())
+        self.register_page_converter(WavConverter())
+        self.register_page_converter(Mp3Converter())
+        self.register_page_converter(ImageConverter())
+
+        if IS_PDF_CAPABLE:
+            self.register_page_converter(PdfConverter())
+
+    def convert(self, source, **kwargs):
+        """
+        Args:
+            - source: can be a string representing a path or url, or a requests.response object
+            - extension: specifies the file extension to use when interpreting the file. If None, infer from source (path, uri, content-type, etc.)
+        """
+
+        # Local path or url
+        if isinstance(source, str):
+            if source.startswith("http://") or source.startswith("https://") or source.startswith("file://"):
+                return self.convert_url(source, **kwargs)
+            else:
+                return self.convert_local(source, **kwargs)
+        # Request response
+        elif isinstance(source, requests.Response):
+            return self.convert_response(source, **kwargs)
+
+    def convert_local(self, path, **kwargs):
+        # Prepare a list of extensions to try (in order of priority)
+        ext = kwargs.get("file_extension")
+        extensions = [ext] if ext is not None else []
+
+        # Get extension alternatives from the path and puremagic
+        base, ext = os.path.splitext(path)
+        self._append_ext(extensions, ext)
+        self._append_ext(extensions, self._guess_ext_magic(path))
+
+        # Convert
+        return self._convert(path, extensions, **kwargs)
+
+    def convert_url(self, url, **kwargs):
+        # Send a HTTP request to the URL
+        response = self._requests_session.get(url, stream=True)
+        response.raise_for_status()
+        return self.convert_response(response, **kwargs)
+
+    def convert_response(self, response, **kwargs):
+        # Prepare a list of extensions to try (in order of priority)
+        ext = kwargs.get("file_extension")
+        extensions = [ext] if ext is not None else []
+
+        # Guess from the mimetype
+        content_type = response.headers.get("content-type", "").split(";")[0]
+        self._append_ext(extensions, mimetypes.guess_extension(content_type))
+
+        # Read the content disposition if there is one
+        content_disposition = response.headers.get("content-disposition", "")
+        m = re.search(r"filename=([^;]+)", content_disposition)
+        if m:
+            base, ext = os.path.splitext(m.group(1).strip("\"'"))
+            self._append_ext(extensions, ext)
+
+        # Read from the extension from the path
+        base, ext = os.path.splitext(urlparse(response.url).path)
+        self._append_ext(extensions, ext)
+
+        # Save the file locally to a temporary file. It will be deleted before this method exits
+        handle, temp_path = tempfile.mkstemp()
+        fh = os.fdopen(handle, "wb")
+        result = None
+        try:
+            # Download the file
+            for chunk in response.iter_content(chunk_size=512):
+                fh.write(chunk)
+            fh.close()
+
+            # Use puremagic to check for more extension options
+            self._append_ext(extensions, self._guess_ext_magic(temp_path))
+
+            # Convert
+            result = self._convert(temp_path, extensions, url=response.url)
+
+        # Clean up
+        finally:
+            try:
+                fh.close()
+            except:
+                pass
+            os.unlink(temp_path)
+
+        return result
+
+    def _convert(self, local_path, extensions, **kwargs):
+        error_trace = ""
+        for ext in extensions:
+            for converter in self._page_converters:
+                _kwargs = copy.deepcopy(kwargs)
+                _kwargs.update({"file_extension": ext})
+
+                # Copy any additional global options
+                if "mlm_client" not in _kwargs and self._mlm_client is not None:
+                    _kwargs["mlm_client"] = self._mlm_client
+
+                # If we hit an error log it and keep trying
+                try:
+                    res = converter.convert(local_path, **_kwargs)
+                except Exception as e:
+                    error_trace = ("\n\n" + traceback.format_exc()).strip()
+
+                if res is not None:
+                    # Normalize the content
+                    res.text_content = "\n".join([line.rstrip() for line in re.split(r"\r?\n", res.text_content)])
+                    res.text_content = re.sub(r"\n{3,}", "\n\n", res.text_content)
+
+                    # Todo
+                    return res
+
+        # If we got this far without success, report any exceptions
+        if len(error_trace) > 0:
+            raise FileConversionException(
+                f"Could not convert '{local_path}' to Markdown. File type was recognized as {extensions}. While converting the file, the following error was encountered:\n\n{error_trace}"
+            )
+
+        # Nothing can handle it!
+        raise UnsupportedFormatException(
+            f"Could not convert '{local_path}' to Markdown. The formats {extensions} are not supported."
+        )
+
+    def _append_ext(self, extensions, ext):
+        """Append a unique non-None, non-empty extension to a list of extensions."""
+        if ext is None:
+            return
+        ext = ext.strip()
+        if ext == "":
+            return
+        # if ext not in extensions:
+        if True:
+            extensions.append(ext)
+
+    def _guess_ext_magic(self, path):
+        """Use puremagic (a Python implementation of libmagic) to guess a file's extension based on the first few bytes."""
+        # Use puremagic to guess
+        try:
+            guesses = puremagic.magic_file(path)
+            if len(guesses) > 0:
+                ext = guesses[0].extension.strip()
+                if len(ext) > 0:
+                    return ext
+        except FileNotFoundError:
+            pass
+        except IsADirectoryError:
+            pass
+        except PermissionError:
+            pass
+        return None
+
+    def register_page_converter(self, converter: DocumentConverter) -> None:
+        """Register a page text converter."""
+        self._page_converters.insert(0, converter)

autogen/oai/openai_utils.py

@@ -50,6 +50,7 @@ OAI_PRICE1K = {
     "gpt-4-0125-preview": (0.01, 0.03),
     "gpt-4-turbo-preview": (0.01, 0.03),
     "gpt-4-1106-vision-preview": (0.01, 0.03),  # TODO: support vision pricing of images
+    "gpt-4-turbo-v": (0.01, 0.03),  # TODO: support vision pricing of images
 }
 
 

autogen/token_count_utils.py

@@ -31,6 +31,7 @@ def get_max_token_limit(model: str = "gpt-3.5-turbo-0613") -> int:
         "gpt-4-0125-preview": 128000,
         "gpt-4-turbo-preview": 128000,
         "gpt-4-vision-preview": 128000,
+        "gpt-4-turbo-v": 128000,  # Azure
     }
     return max_token_limit[model]
 

samples/tools/autogenbench/.gitignore

@@ -0,0 +1,3 @@
+scenarios/*/Downloads
+scenarios/*/Tasks
+*/Results

samples/tools/autogenbench/autogenbench/cli.py

@@ -1,4 +1,5 @@
 import sys
+from .version import __version__
 from .run_cmd import run_cli
 from .clone_cmd import clone_cli
 from .tabulate_cmd import tabulate_cli
@@ -9,6 +10,7 @@ def main(args=None):
         args = sys.argv[:]  # Shallow copy
 
     invocation_cmd = "autogenbench"
+    version_string = f"AutoGenBench version {__version__}"
 
     commands = [
         {
@@ -26,6 +28,11 @@ def main(args=None):
             "description": "tabulate the results of a previous run",
             "function": tabulate_cli,
         },
+        {
+            "command": "--version",
+            "description": f"print the version of {invocation_cmd}",
+            "function": lambda _args: print(f"{version_string}"),
+        },
         {"command": "--help", "description": "print this message", "function": None},
     ]
 
@@ -40,6 +47,8 @@ def main(args=None):
         commands_details += f"    {padded_cmd}: {c['description']}\n"
 
     usage_text = f"""
+{version_string}
+
 usage: {invocation_cmd} COMMAND ARGS
 
 Where, COMMAND is one of: {commands_list}
@@ -49,6 +58,8 @@ and ARGS are specific to the command.
 """.strip()
 
     help_text = f"""
+{version_string}
+
 usage: {invocation_cmd} COMMAND ARGS
 
 {invocation_cmd} is a tool for running and managing AutoGen benchmark scenarios. A typically session might resemble:

samples/tools/autogenbench/autogenbench/res/Dockerfile

@@ -1,16 +1,24 @@
-# Host a jsPsych experiment in Azure
 FROM python:3.11
 MAINTAINER AutoGen
 
-# Upgrade pip
-RUN pip install --upgrade pip
+# Install packages
+RUN apt-get update && apt-get install ffmpeg exiftool -y
 
 # Set the image to the Pacific Timezone
 RUN ln -snf /usr/share/zoneinfo/US/Pacific /etc/localtime && echo "US/Pacific" > /etc/timezone
 
+# Upgrade pip
+RUN pip install --upgrade pip
+
 # Pre-load autogen dependencies, but not autogen itself since we'll often want to install the latest from source
 RUN pip install pyautogen[teachable,lmm,graphs,websurfer]
 RUN pip uninstall --yes pyautogen
 
 # Pre-load popular packages as per https://learnpython.com/blog/most-popular-python-packages/
 RUN pip install numpy pandas matplotlib seaborn scikit-learn requests urllib3 nltk pillow pytest
+
+# Pre-load packages needed for complex_task file utils
+RUN pip install python-docx pdfminer.six requests pillow easyocr python-pptx SpeechRecognition pandas openpyxl pydub mammoth puremagic youtube_transcript_api==0.6.0
+
+# Pre-load the OCR model
+RUN /usr/bin/echo -e "import easyocr\nreader = easyocr.Reader(['en'])" | python

samples/tools/autogenbench/autogenbench/run_cmd.py

@@ -11,13 +11,14 @@ import docker
 import random
 from autogen import config_list_from_json
 from autogen.oai.openai_utils import filter_config
+from .version import __version__
 
 # Figure out where everything is
 SCRIPT_PATH = os.path.realpath(__file__)
 SCRIPT_NAME = os.path.basename(SCRIPT_PATH)
 SCRIPT_DIR = os.path.dirname(SCRIPT_PATH)
 
-TASK_TIMEOUT = 60 * 30  # 30 minutes
+TASK_TIMEOUT = 60 * 60 * 2 # 2 hours
 
 BASE_TEMPLATE_PATH = os.path.join(SCRIPT_DIR, "template")
 RESOURCES_PATH = os.path.join(SCRIPT_DIR, "res")
@@ -247,17 +248,25 @@ def get_scenario_env(config_list, env_file=DEFAULT_ENV_FILE):
     Returns: A dictionary of keys and values that need to be added to the system environment.
     """
     env = dict()
+
+    # Populate with commonly needed keys
+    openai_api_key = os.environ.get("OPENAI_API_KEY")
+    if openai_api_key is not None and len(openai_api_key.strip()) > 0:
+        env["OPENAI_API_KEY"] = openai_api_key
+
+    bing_api_key = os.environ.get("BING_API_KEY")
+    if bing_api_key is not None and len(bing_api_key.strip()) > 0:
+        env["BING_API_KEY"] = bing_api_key
+
+    # Update with any values from the ENV.json file
     if os.path.isfile(env_file):
         with open(env_file, "rt") as fh:
-            env = json.loads(fh.read())
+            env.update(json.loads(fh.read()))
 
+    # Include the config_list that we are using
     config_list_json = json.dumps(config_list)
     env["OAI_CONFIG_LIST"] = config_list_json
 
-    openai_api_key = os.environ.get("OPENAI_API_KEY")
-    if openai_api_key is not None and len(openai_api_key.strip()) > 0:
-        env["OPENAI_API_KEY"] = openai_api_key
-
     return env
 
 
@@ -286,6 +295,12 @@ def run_scenario_natively(work_dir, env, timeout=TASK_TIMEOUT):
             f"""#
 echo RUN.SH STARTING !#!#
 export AUTOGEN_TESTBED_SETTING="Native"
+echo "autogenbench version: {__version__}" > timestamp.txt
+
+# Create and activate the virtual environment
+# This is called in a subprocess, and will not impact the parent
+{sys.executable} -m venv .autogenbench_venv
+. .autogenbench_venv/bin/activate
 
 # Run the global init script if it exists
 if [ -f global_init.sh ] ; then
@@ -298,6 +313,7 @@ if [ -f scenario_init.sh ] ; then
 fi
 
 # Run the scenario
+pip install -r requirements.txt
 echo SCENARIO.PY STARTING !#!#
 timeout --preserve-status --kill-after {timeout  + 30}s {timeout}s python scenario.py
 EXIT_CODE=$?
@@ -312,6 +328,10 @@ if [ -d .cache ] ; then
     rm -Rf .cache
 fi
 
+if [ -d __pycache__ ] ; then
+    rm -Rf __pycache__
+fi
+
 # Run the scenario finalize script if it exists
 if [ -f scenario_finalize.sh ] ; then
     . ./scenario_finalize.sh
@@ -322,6 +342,12 @@ if [ -f global_finalize.sh ] ; then
     . ./global_finalize.sh
 fi
 
+# We don't need to deactivate the venv because it's
+# contained in the subprocess; but we should clean it up
+if [ -d .autogenbench_venv ] ; then
+    rm -Rf .autogenbench_venv
+fi
+
 echo RUN.SH COMPLETE !#!#
 """
         )
@@ -387,7 +413,9 @@ def run_scenario_in_docker(work_dir, env, timeout=TASK_TIMEOUT, docker_image=Non
             f"""#
 echo RUN.SH STARTING !#!#
 export AUTOGEN_TESTBED_SETTING="Docker"
+
 umask 000
+echo "autogenbench version: {__version__}" > timestamp.txt
 
 # Run the global init script if it exists
 if [ -f global_init.sh ] ; then
@@ -415,6 +443,10 @@ if [ -d .cache ] ; then
     rm -Rf .cache
 fi
 
+if [ -d __pycache__ ] ; then
+    rm -Rf __pycache__
+fi
+
 # Run the scenario finalize script if it exists
 if [ -f scenario_finalize.sh ] ; then
     . ./scenario_finalize.sh
@@ -429,18 +461,31 @@ echo RUN.SH COMPLETE !#!#
 """
         )
 
-    print("\n\n" + work_dir + "\n===================================================================")
+    # Figure out what folders to mount
+    volumes = {str(pathlib.Path(work_dir).absolute()): {"bind": "/workspace", "mode": "rw"}}
+
+    # Add the autogen repo if we can find it
+    autogen_repo_base = os.environ.get("AUTOGENBENCH_REPO_BASE")
+    if autogen_repo_base is None:
+        autogen_repo_base = find_autogen_repo(os.getcwd())
+    elif not os.path.isdir(autogen_repo_base):
+        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), autogen_repo_base)
+
+    if autogen_repo_base is not None:
+        volumes[str(pathlib.Path(autogen_repo_base).absolute())] = {"bind": "/autogen", "mode": "rw"}
+
+    print("Mounting:")
+    for k in volumes:
+        bind = volumes[k]["bind"]
+        mode = volumes[k]["mode"].upper()
+        if bind == "/workspace":
+            k = os.path.relpath(k)
+        print(f"[{mode}]\t'{k}' => '{bind}'")
+    print("===================================================================")
 
     # Create and run the container
-    abs_path = str(pathlib.Path(work_dir).absolute())
     container = client.containers.run(
-        image,
-        command=["sh", "run.sh"],
-        working_dir="/workspace",
-        environment=env,
-        detach=True,
-        # get absolute path to the working directory
-        volumes={abs_path: {"bind": "/workspace", "mode": "rw"}},
+        image, command=["sh", "run.sh"], working_dir="/workspace", environment=env, detach=True, volumes=volumes
     )
 
     # Read the logs in a streaming fashion. Keep an eye on the time to make sure we don't need to stop.
@@ -485,6 +530,34 @@ def build_default_docker_image(docker_client, image_tag):
             sys.stdout.write(segment["stream"])
 
 
+def find_autogen_repo(path):
+    """
+    Utility for identifying if the path is a subdirectory of the autogen repo.
+
+    Returns: the path to the root of the autogen repo if one is found, otherwise None
+    """
+
+    # Normalize the path (we expect a directory)
+    path = os.path.abspath(path)
+    if os.path.isfile(path):
+        path = os.path.dirname(path)
+
+    while True:
+        test_path = os.path.join(path, "autogen", "agentchat", "conversable_agent.py")  # We found autogen
+        if os.path.isfile(test_path):
+            return path
+
+        # Stop if we hit the root
+        parent_dir = os.path.abspath(os.path.join(path, os.pardir))
+        if parent_dir == path:
+            break
+
+        # Keep searching
+        path = parent_dir
+
+    return None
+
+
 def run_cli(args):
     invocation_cmd = args[0]
     args = args[1:]
@@ -581,12 +654,23 @@ def run_cli(args):
         if parsed_args.requirements is not None:
             sys.exit("--requirements is not compatible with --native. Exiting.")
 
-        choice = input(
-            'WARNING: Running natively, without Docker, not only poses the usual risks of executing arbitrary AI generated code on your machine, it also makes it impossible to ensure that each test starts from a known and consistent set of initial conditions. For example, if the agents spend time debugging and installing Python libraries to solve the task, then those libraries will be available to all other runs. In other words, earlier runs can influence later runs, leading to many confounds in testing.\n\nAre you absolutely sure you want to continue with native execution? Type "Yes" exactly, and in full, to proceed: '
+        sys.stderr.write(
+            "WARNING: Running natively, without Docker, not only poses the usual risks of executing arbitrary AI generated code on your machine, it also makes it impossible to ensure that each test starts from a known and consistent set of initial conditions. For example, if the agents spend time debugging and installing Python libraries to solve the task, then those libraries will be available to all other runs. In other words, earlier runs can influence later runs, leading to many confounds in testing.\n\n"
         )
 
-        if choice.strip().lower() != "yes":
-            sys.exit("Received '" + choice + "'. Exiting.")
+        # Does an environment variable override the prompt?
+        allow_native = os.environ.get("AUTOGENBENCH_ALLOW_NATIVE")
+        if allow_native is None or allow_native == "":
+            choice = input(
+                'Are you absolutely sure you want to continue with native execution? Type "Yes" exactly, and in full, to proceed: '
+            )
+            if choice.strip().lower() != "yes":
+                sys.exit("Received '" + choice + "'. Exiting.")
+        elif allow_native.strip().lower() != "yes":
+            sys.exit(f"Exiting because AUTOGENBENCH_ALLOW_NATIVE is '{allow_native}'\n")
+        else:
+            sys.stderr.write(f"Continuing because AUTOGENBENCH_ALLOW_NATIVE is '{allow_native}'\n")
+            time.sleep(0.75)  # Pause very briefly so the message isn't lost in the noise
 
     # Parse the subsample
     subsample = None

samples/tools/autogenbench/autogenbench/tabulate_cmd.py

@@ -3,6 +3,7 @@ import sys
 import argparse
 import tabulate as tb
 from .load_module import load_module
+from copy import deepcopy
 
 # Figure out where everything is
 SCRIPT_PATH = os.path.realpath(__file__)
@@ -88,6 +89,10 @@ def default_tabulate(args, scorer=default_scorer, exclude_dir_names=EXCLUDE_DIR_
         help="Output the results in CSV format.",
     )
 
+    parser.add_argument(
+        "-e", "--excel", help="Output the results in Excel format. Please specify a path for the Excel file.", type=str
+    )
+
     parsed_args = parser.parse_args(args)
 
     all_results = list()
@@ -145,15 +150,17 @@ def default_tabulate(args, scorer=default_scorer, exclude_dir_names=EXCLUDE_DIR_
         def _count_equals(value, trial):
             count = 0
             for row in all_results:
+                is_answer_matched = row[trial + 1][0] if isinstance(row[trial + 1], tuple) else row[trial + 1]
+
                 # Count missing
                 if value is None:
                     if trial + 1 < len(row):
-                        if row[trial + 1] is None:
+                        if is_answer_matched is None:
                             count += 1
                     else:
                         count += 1
                 # Count match
-                elif trial + 1 < len(row) and row[trial + 1] == value:
+                elif trial + 1 < len(row) and is_answer_matched == value:
                     count += 1
             return count
 
@@ -178,7 +185,12 @@ def default_tabulate(args, scorer=default_scorer, exclude_dir_names=EXCLUDE_DIR_
             footer_row.append(footer[0][i + 1] + footer[1][i + 1] + footer[2][i + 1])
         footer.append(footer_row)
 
-        table = all_results.copy()
+        table = deepcopy(all_results)
+        for row in table:
+            for trial in range(0, max_instances):
+                if isinstance(row[trial + 1], tuple):
+                    row[trial + 1] = row[trial + 1][0]
+
         table.append(tb.SEPARATING_LINE)
         table.extend(footer)
 
@@ -186,6 +198,7 @@ def default_tabulate(args, scorer=default_scorer, exclude_dir_names=EXCLUDE_DIR_
 
         # Print out alpha-version warning
         sys.stderr.write("\n" + warning + "\n\n")
+    return parsed_args, all_results
 
 
 def tabulate_cli(args):

samples/tools/autogenbench/autogenbench/template/testbed_utils.py

@@ -6,6 +6,15 @@ import json
 
 AUTOGEN_VERSION = packaging.version.parse(autogen.__version__)
 
+# Try importing the runtime_logging module (only available in some branches)
+TELEMETRY_ENABLED = False
+try:
+    import autogen.runtime_logging
+
+    TELEMETRY_ENABLED = True
+except ImportError:
+    pass
+
 
 def default_llm_config(config_list, timeout=180):
     """Return a default config list with a given timeout, and with caching disabled.
@@ -57,6 +66,10 @@ def init():
     if AUTOGEN_VERSION < packaging.version.parse("0.2.0b1"):
         autogen.Completion.start_logging(compact=False)
 
+    # Start logging
+    if TELEMETRY_ENABLED:
+        autogen.runtime_logging.start(config={"dbname": "telemetry.db"})
+
 
 def finalize(agents):
     """Helper function to finalize logging in a testbed scenario.
@@ -89,3 +102,7 @@ def finalize(agents):
         with open(os.path.join(script_dir, "completion_log.json"), "wt") as fh:
             fh.write(json.dumps(autogen.Completion.logged_history, indent=4))
         autogen.Completion.stop_logging()
+
+    # Stop logging
+    if TELEMETRY_ENABLED:
+        autogen.runtime_logging.stop()

samples/tools/autogenbench/autogenbench/version.py

@@ -1 +1 @@
-__version__ = "0.0.1"
+__version__ = "0.0.2a3"

samples/tools/autogenbench/pyproject.toml

@@ -22,6 +22,8 @@ dependencies = [
     "docker",
     "huggingface_hub",
     "tabulate",
+    "pandas",
+    "openpyxl"
 ]
 
 dynamic = ["version"]
@@ -47,3 +49,8 @@ exclude = ["*.tests*"]
 
 [project.scripts]
 autogenbench = "autogenbench.cli:main"
+
+[tool.black]
+# https://github.com/psf/black
+line-length = 120
+exclude = "(.eggs|.git|.hg|.mypy_cache|.venv|_build|buck-out|build|dist)"

samples/tools/autogenbench/scenarios/AutoGPT/MANIFEST.json

@@ -8,6 +8,7 @@
         "Templates/TwoAgents/should_not_contain.json.txt": "Templates/TwoAgents/should_not_contain.json.txt",
         "Templates/TwoAgents/scenario.py": "Templates/TwoAgents/scenario.py",
         "Templates/TwoAgents/scenario_init.sh": "Templates/TwoAgents/scenario_init.sh",
+        "Templates/TwoAgents/requirements.txt": "Templates/TwoAgents/requirements.txt",
         "Challenges/1_sort_csv/data.json": "Challenges/1_sort_csv/data.json",
         "Challenges/1_sort_csv/artifacts_in/input.csv": "Challenges/1_sort_csv/artifacts_in/input.csv",
         "Challenges/2_combine_csv/data.json": "Challenges/2_combine_csv/data.json",

samples/tools/autogenbench/scenarios/AutoGPT/Scripts/init_tasks.py

@@ -8,6 +8,7 @@ import os
 import sys
 import glob
 import base64
+import re
 from huggingface_hub import snapshot_download
 
 SCRIPT_PATH = os.path.realpath(__file__)
@@ -88,7 +89,12 @@ def create_jsonl(name, template):
 
 ###############################################################################
 def main():
-    templates = {"two_agents": os.path.join(TEMPLATES_DIR, "TwoAgents")}
+    # list all directories in the Templates directory
+    # and populate a dictionary with the name and path
+    templates = {}
+    for entry in os.scandir(TEMPLATES_DIR):
+        if entry.is_dir():
+            templates[re.sub(r"\s", "", entry.name)] = entry.path
 
     # Add coding directories if needed (these are usually empty and left out of the repo)
     for template in templates.values():

samples/tools/autogenbench/scenarios/AutoGPT/Templates/TwoAgents/requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/microsoft/autogen.git@complex_tasks

samples/tools/autogenbench/scenarios/Examples/MANIFEST.json

@@ -3,6 +3,7 @@
         "Templates/TwoAgents/scenario_finalize.sh": "Templates/TwoAgents/scenario_finalize.sh",
         "Templates/TwoAgents/scenario.py": "Templates/TwoAgents/scenario.py",
         "Templates/TwoAgents/scenario_init.sh": "Templates/TwoAgents/scenario_init.sh",
+        "Templates/TwoAgents/requirements.txt": "Templates/TwoAgents/requirements.txt",
         "Tasks/default_two_agents.jsonl": "Tasks/default_two_agents.jsonl",
         "README.md": "README.md"
     }

samples/tools/autogenbench/scenarios/Examples/Templates/ThreeAgents/requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/microsoft/autogen.git@complex_tasks

samples/tools/autogenbench/scenarios/Examples/Templates/TwoAgents/requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/microsoft/autogen.git@complex_tasks

samples/tools/autogenbench/scenarios/GAIA/MANIFEST.json

@@ -3,10 +3,19 @@
         "README.md": "README.md",
         "Scripts/init_tasks.py": "Scripts/init_tasks.py",
         "Scripts/custom_tabulate.py": "Scripts/custom_tabulate.py",
+        "Scripts/collate_results.py": "Scripts/collate_results.py",
         "Templates/BasicTwoAgents/expected_answer.txt": "Templates/BasicTwoAgents/expected_answer.txt",
+        "Templates/BasicTwoAgents/prompt.txt": "Templates/BasicTwoAgents/prompt.txt",
         "Templates/BasicTwoAgents/scenario.py": "Templates/BasicTwoAgents/scenario.py",
-        "Templates/SocietyOfMind/scenario.py": "Templates/SocietyOfMind/scenario.py",
+        "Templates/BasicTwoAgents/requirements.txt": "Templates/BasicTwoAgents/requirements.txt",
+        "Templates/BasicTwoAgentsFunctionCalling/expected_answer.txt": "Templates/BasicTwoAgentsFunctionCalling/expected_answer.txt",
+        "Templates/BasicTwoAgentsFunctionCalling/prompt.txt": "Templates/BasicTwoAgentsFunctionCalling/prompt.txt",
+        "Templates/BasicTwoAgentsFunctionCalling/scenario.py": "Templates/BasicTwoAgentsFunctionCalling/scenario.py",
+        "Templates/BasicTwoAgentsFunctionCalling/requirements.txt": "Templates/BasicTwoAgentsFunctionCalling/requirements.txt",
         "Templates/SocietyOfMind/expected_answer.txt": "Templates/SocietyOfMind/expected_answer.txt",
+        "Templates/SocietyOfMind/prompt.txt": "Templates/SocietyOfMind/prompt.txt",
+        "Templates/SocietyOfMind/scenario.py": "Templates/SocietyOfMind/scenario.py",
+        "Templates/SocietyOfMind/group_chat_moderator.py": "Templates/SocietyOfMind/group_chat_moderator.py",
         "Templates/SocietyOfMind/requirements.txt": "Templates/SocietyOfMind/requirements.txt"
     }
 }

samples/tools/autogenbench/scenarios/GAIA/README.md

@@ -33,6 +33,12 @@ autogenbench tabulate Results/gaia_test_level_1__soc
 
 And similarly for level 2 and 3.
 
+
+## Export the metrics in Excel format
+```sh
+autogenbench tabulate Results/gaia_test_level_1__soc -e EXCEL_REPORT_PATH
+```
+
 ## References
 **GAIA: a benchmark for General AI Assistants**<br/>
 Grégoire Mialon, Clémentine Fourrier, Craig Swift, Thomas Wolf, Yann LeCun, Thomas Scialom<br/>

samples/tools/autogenbench/scenarios/GAIA/Scripts/collate_results.py

@@ -0,0 +1,102 @@
+import os
+import json
+import re
+import sys
+import argparse
+
+
+def normalize_answer(a):
+    # Trim (left and right)
+    # Replace multiple spaces with one space
+    # Remove trailing punctuation
+    # Trim again
+    return re.sub(r"[\.\!\?]+$", "", re.sub(r"\s+", " ", a.strip())).strip()
+
+
+def collate(results_dir):
+    """
+    Collate the results of running GAIA. Print the results in the format accepted by the leaderboard.
+
+    Args:
+        results_dir (path): The folder where results were be saved.
+    """
+
+    for test_id in os.listdir(results_dir):
+        test_path = os.path.join(results_dir, test_id)
+
+        for instance in os.listdir(test_path):
+            instance_dir = os.path.join(test_path, str(instance))
+            console_log_file = os.path.join(instance_dir, "console_log.txt")
+
+            final_answer = ""
+            console_log = ""
+            if os.path.isfile(console_log_file):
+                with open(console_log_file, "rt") as fh:
+                    console_log = fh.read()
+
+                    # Trim the console log
+                    m = re.search(
+                        r"SCENARIO.PY STARTING !#!#(.*?)SCENARIO.PY (COMPLETE|EXITED .*?) !#!#", console_log, re.DOTALL
+                    )
+                    if m:
+                        console_log = m.group(1).strip()
+
+                    # Extract the final answer
+                    final_answer = ""
+                    m = re.search(r"FINAL ANSWER:(.*?)\n", console_log, re.DOTALL)
+                    if m:
+                        final_answer = m.group(1).strip()
+
+            expected_answer_file = os.path.join(instance_dir, "expected_answer.txt")
+            expected_answer = "NOT PROVIDED !#!#"
+            if os.path.isfile(expected_answer_file):
+                with open(expected_answer_file, "rt") as fh:
+                    expected_answer = fh.read().strip()
+
+            prompt_file = os.path.join(instance_dir, "prompt.txt")
+            prompt = None
+            if os.path.isfile(prompt_file):
+                with open(prompt_file, "rt") as fh:
+                    prompt = fh.read().strip()
+
+            # Apply approximate string matching
+            is_correct = normalize_answer(final_answer) == normalize_answer(expected_answer)
+
+            # Parse the steps
+            steps = [s.strip() for s in re.split(r"\-\-\-\-\-\-\-\-+", console_log) if len(s) > 0]
+
+            print(
+                json.dumps(
+                    {
+                        "task_id": test_id,
+                        "trial": instance,
+                        "question": prompt,
+                        "is_correct": is_correct,
+                        "model_answer": final_answer,
+                        "expected_answer": expected_answer,
+                        "reasoning_trace": steps,
+                    },
+                    indent=4,
+                )
+            )
+
+
+###############################################################################
+if __name__ == "__main__":
+    script_path = os.path.realpath(__file__)
+    script_name = os.path.basename(script_path)
+    script_dir = os.path.dirname(script_path)
+
+    parser = argparse.ArgumentParser(
+        description=f"""
+{script_name} will collate the results of the GAIA scenarios into the jsonl format that can be submit to AgentEval.
+""".strip(),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+
+    parser.add_argument(
+        "scenario",
+        help="Path to the scenario results.",
+    )
+    args = parser.parse_args()
+    collate(args.scenario)

samples/tools/autogenbench/scenarios/GAIA/Scripts/custom_tabulate.py

@@ -1,16 +1,25 @@
 import os
 import sys
-import json
 import re
 from autogenbench.tabulate_cmd import default_tabulate
+import json
+import pandas as pd
+import sqlite3
+import glob
+import numpy as np
+
+EXCLUDE_DIR_NAMES = ["__pycache__"]
 
 
 def normalize_answer(a):
     # Lower case
     # Trim (left and right)
+    # standardize comma separated values
     # Replace multiple spaces with one space
     # Remove trailing punctuation
-    return re.sub(r"[\.\!\?]+$", "", re.sub(r"\s+", " ", a.strip().lower()))
+    norm_answer = ", ".join(a.strip().lower().split(","))
+    norm_answer = re.sub(r"[\.\!\?]+$", "", re.sub(r"\s+", " ", norm_answer))
+    return norm_answer
 
 
 def scorer(instance_dir):
@@ -32,17 +41,156 @@ def scorer(instance_dir):
     with open(console_log_file, "rt") as fh:
         console_log = fh.read()
 
-        final_answer = ""
+        final_answer = None 
         m = re.search(r"FINAL ANSWER:(.*?)\n", console_log, re.DOTALL)
         if m:
             final_answer = m.group(1).strip()
 
+        # Missing the final answer line
+        if final_answer is None:
+            return None
+
         # Return true if they are equal after normalization
-        return normalize_answer(expected_answer) == normalize_answer(final_answer)
+        n_ex = normalize_answer(expected_answer)
+        n_final = normalize_answer(final_answer)
+        return (
+            (n_ex != "" and n_ex == n_final),
+            n_ex,
+            n_final
+        )
+
+
+def get_number_of_chat_messages(chat_messages_dir):
+    result = 0
+    for file in glob.glob(f"{chat_messages_dir}/*_messages.json"):
+        with open(file, "r") as f:
+            content = json.load(f)
+            for agent, messages in content.items():
+                result += len(messages)
+    return result
 
 
 def main(args):
-    default_tabulate(args, scorer=scorer)
+    parsed_args, all_results = default_tabulate(args, scorer=scorer)
+    excel_path = parsed_args.excel
+
+    if excel_path:
+        excel_dir = os.path.dirname(excel_path) or "."
+        if not os.path.exists(excel_dir):
+            os.makedirs(excel_dir, exist_ok=True)
+
+        if not excel_path.endswith((".xlsx", ".xls")):
+            excel_path += ".xlsx"
+
+        runlogs = parsed_args.runlogs if parsed_args.runlogs.endswith("/") else parsed_args.runlogs + "/"
+
+        if os.path.isdir(runlogs):
+            task_ids = sorted(
+                [task_id for task_id in os.listdir(runlogs) if task_id not in EXCLUDE_DIR_NAMES],
+                key=lambda s: os.path.getmtime(os.path.join(parsed_args.runlogs, s)),
+            )
+        else:
+            raise ValueError("please input a valid directory to tabulate result")
+
+        trials = sorted(os.listdir(f"{runlogs}{task_ids[0]}"), key=lambda x: int(x)) if len(task_ids) > 0 else []
+        dbnames = [[f"{runlogs}{task_id}/{trial}/telemetry.db" for task_id in task_ids] for trial in trials]
+
+        query = """
+            SELECT cost, session_id, response, start_time, end_time
+            FROM (
+                SELECT invocation_id, cost, session_id, response, start_time, end_time,
+                    ROW_NUMBER() OVER (PARTITION BY invocation_id ORDER BY start_time) as rn
+                FROM chat_completions
+            )
+            WHERE rn = 1;
+        """
+
+        with pd.ExcelWriter(excel_path, engine="openpyxl") as writer:
+            for trial_index, each_trial in enumerate(dbnames):
+                result_df = pd.DataFrame(
+                    columns=[
+                        "id",
+                        "status",
+                        "expected_answer",
+                        "final_answer",
+                        "cost",
+                        "latency",
+                        "num_of_llm_requests",
+                        "num_of_chat_messages",
+                        "prompt_tokens",
+                        "completion_tokens",
+                        "total_tokens",
+                        "model",
+                    ]
+                )
+
+                result_df_type_mapping = {
+                    "id": str,
+                    "status": bool,
+                    "expected_answer": str,
+                    "final_answer": str,
+                    "cost": float,
+                    "latency": float,
+                    "num_of_llm_requests": int,
+                    "num_of_chat_messages": int,
+                    "prompt_tokens": int,
+                    "completion_tokens": int,
+                    "total_tokens": int,
+                }
+
+                for dbname, scorer_results in zip(each_trial, all_results):
+                    task_id = scorer_results[0]
+                    scorer_result = scorer_results[trial_index + 1]
+
+                    status, expected_answer, final_answer = scorer_result if scorer_result else (False,"","")
+
+                    con = sqlite3.connect(dbname)
+
+                    # TODO: if large amount of data, add chunksize
+                    telemetry_df = pd.read_sql_query(query, con)
+
+                    earliest_starttime = pd.to_datetime(telemetry_df["start_time"], format="%Y-%m-%d %H:%M:%S.%f").min()
+                    latest_endtime = pd.to_datetime(telemetry_df["end_time"], format="%Y-%m-%d %H:%M:%S.%f").max()
+
+                    num_of_chat_messages = get_number_of_chat_messages(chat_messages_dir=os.path.dirname(dbname))
+                    result = {
+                        "id": task_id,
+                        "status": status,
+                        "expected_answer": expected_answer,
+                        "final_answer": final_answer,
+                        "cost": telemetry_df["cost"].sum(),
+                        "latency": (latest_endtime - earliest_starttime).total_seconds(),
+                        "num_of_llm_requests": len(telemetry_df),
+                        "num_of_chat_messages": num_of_chat_messages,
+                        "prompt_tokens": telemetry_df["response"]
+                        .apply(
+                            lambda x: json.loads(x)["usage"]["prompt_tokens"]
+                            if "usage" in json.loads(x) and "prompt_tokens" in json.loads(x)["usage"]
+                            else 0
+                        )
+                        .sum(),
+                        "completion_tokens": telemetry_df["response"]
+                        .apply(
+                            lambda x: json.loads(x)["usage"]["completion_tokens"]
+                            if "usage" in json.loads(x) and "completion_tokens" in json.loads(x)["usage"]
+                            else 0
+                        )
+                        .sum(),
+                        "total_tokens": telemetry_df["response"]
+                        .apply(
+                            lambda x: json.loads(x)["usage"]["total_tokens"]
+                            if "usage" in json.loads(x) and "total_tokens" in json.loads(x)["usage"]
+                            else 0
+                        )
+                        .sum(),
+                        "model": telemetry_df["response"]
+                        .apply(lambda x: json.loads(x)["model"] if "model" in json.loads(x) else "")
+                        .unique(),
+                    }
+
+                    result_df = result_df.astype(result_df_type_mapping)
+                    result_df = pd.concat([result_df, pd.DataFrame([result])], ignore_index=True)
+                result_df.to_excel(writer, sheet_name=f"trial_{trial_index}", index=False)
 
 
 if __name__ == "__main__" and __package__ is None:

samples/tools/autogenbench/scenarios/GAIA/Scripts/init_tasks.py

@@ -6,6 +6,7 @@
 import json
 import os
 import sys
+import re
 from huggingface_hub import snapshot_download
 
 SCRIPT_PATH = os.path.realpath(__file__)
@@ -60,9 +61,9 @@ def create_jsonl(name, tasks, files_dir, template):
                 "substitutions": {
                     "scenario.py": {
                         "__FILE_NAME__": task["file_name"],
-                        "__PROMPT__": task["Question"],
                     },
                     "expected_answer.txt": {"__EXPECTED_ANSWER__": task["Final answer"]},
+                    "prompt.txt": {"__PROMPT__": task["Question"]},
                 },
             }
 
@@ -97,10 +98,12 @@ def main():
 
             gaia_test_tasks[data["Level"] - 1].append(data)
 
-    templates = {
-        "two_agents": os.path.join(TEMPLATES_DIR, "BasicTwoAgents"),
-        "soc": os.path.join(TEMPLATES_DIR, "SocietyOfMind"),
-    }
+    # list all directories in the Templates directory
+    # and populate a dictionary with the name and path
+    templates = {}
+    for entry in os.scandir(TEMPLATES_DIR):
+        if entry.is_dir():
+            templates[re.sub(r"\s", "", entry.name)] = entry.path
 
     # Add coding directories if needed (these are usually empty and left out of the repo)
     for template in templates.values():

samples/tools/autogenbench/scenarios/GAIA/Templates/BasicTwoAgents/prompt.txt

@@ -0,0 +1 @@
+__PROMPT__

samples/tools/autogenbench/scenarios/GAIA/Templates/BasicTwoAgents/requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/microsoft/autogen.git@complex_tasks

samples/tools/autogenbench/scenarios/GAIA/Templates/BasicTwoAgents/scenario.py

@@ -7,6 +7,10 @@ import testbed_utils
 testbed_utils.init()
 ##############################
 
+# Read the prompt
+PROMPT = ""
+with open("prompt.txt", "rt") as fh:
+    PROMPT = fh.read().strip()
 
 GAIA_SYSTEM_MESSAGE = (
     "You are a helpful AI assistant, and today's date is "
@@ -48,9 +52,7 @@ user_proxy = autogen.UserProxyAgent(
 )
 
 filename = "__FILE_NAME__".strip()
-question = """
-__PROMPT__
-""".strip()
+question = PROMPT
 
 if len(filename) > 0:
     question = f"Consider the file '{filename}', which can be read from the current working directory. If you need to read or write it, output python code in a code block (```python) to do so. {question}"

samples/tools/autogenbench/scenarios/GAIA/Templates/BasicTwoAgentsFunctionCalling/expected_answer.txt

@@ -0,0 +1 @@
+__EXPECTED_ANSWER__

samples/tools/autogenbench/scenarios/GAIA/Templates/BasicTwoAgentsFunctionCalling/prompt.txt

@@ -0,0 +1 @@
+__PROMPT__

samples/tools/autogenbench/scenarios/GAIA/Templates/BasicTwoAgentsFunctionCalling/requirements.txt

@@ -0,0 +1 @@
+pyautogen[websurfer] @ git+https://github.com/microsoft/autogen.git@complex_tasks

samples/tools/autogenbench/scenarios/GAIA/Templates/BasicTwoAgentsFunctionCalling/scenario.py

@@ -0,0 +1,70 @@
+import os
+import json
+import autogen
+from autogen.agentchat.contrib.functions import youtube_utils as yt
+from datetime import datetime
+import testbed_utils
+
+testbed_utils.init()
+##############################
+
+# Read the prompt
+PROMPT = ""
+with open("prompt.txt", "rt") as fh:
+    PROMPT = fh.read().strip()
+
+GAIA_SYSTEM_MESSAGE = (
+    "You are a helpful AI assistant, and today's date is "
+    + datetime.now().date().isoformat()
+    + """.
+I will ask you a question. Answer this question using your coding and language skills.
+In the following cases, suggest python code (presented in a coding block beginning ```python) or shell script (presented in a coding block beginning ```sh) for the user to execute:
+    1. When you need to collect info, use the code to output the info you need, for example, browse or search the web, download/read a file, print the content of a webpage or a file, check the operating system. After sufficient info is printed and the task is ready to be solved based on your language skill, you can solve the task by yourself.
+    2. When you need to perform some task with code, use the code to perform the task and output the result. Finish the task smartly.
+Answer the question step if you need to. If a plan is not provided, explain your plan first. Be clear which step uses code, and which step uses your language skill.
+The user cannot provide any other feedback or perform any other action beyond executing the code appearing in the code block. The user can't modify your code, so do not suggest incomplete code which requires users to modify. Don't use a code block if it's not intended to be executed by the user. Don't include multiple code blocks in one response. Do not ask users to copy and paste code or results. Instead, use the 'print' function for the output when relevant. Check the execution result reported by the user.
+If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try.
+When you find an answer, report your thoughts, and finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER].
+YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings.
+If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise.
+If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise.
+If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
+    """.strip()
+)
+
+print("Hello this is the Basic Two Agents File Support scenario.py")
+
+config_list = autogen.config_list_from_json("OAI_CONFIG_LIST")
+
+assistant = autogen.AssistantAgent(
+    "assistant",
+    system_message=GAIA_SYSTEM_MESSAGE,
+    is_termination_msg=lambda x: x.get("content", "").rstrip().find("FINAL ANSWER") >= 0,
+    llm_config=testbed_utils.default_llm_config(config_list, timeout=180),
+)
+user_proxy = autogen.UserProxyAgent(
+    "user_proxy",
+    human_input_mode="NEVER",
+    is_termination_msg=lambda x: x.get("content") and x.get("content", "").rstrip().find("FINAL ANSWER") >= 0,
+    code_execution_config={
+        "work_dir": "coding",
+        "use_docker": False,
+    },
+    max_consecutive_auto_reply=10,
+    default_auto_reply="",
+)
+
+assistant.register_for_llm(description="Get youtube transcript from link")(yt.get_youtube_transcript)
+user_proxy.register_for_execution()(yt.get_youtube_transcript)
+
+filename = "__FILE_NAME__".strip()
+question = PROMPT
+
+if len(filename) > 0:
+    question = f"Consider the file '{filename}', which can be read from the current working directory. If you need to read or write it, output python code in a code block (```python) to do so. {question}"
+
+user_proxy.initiate_chat(assistant, message=question)
+
+
+##############################
+testbed_utils.finalize(agents=[assistant, user_proxy])

samples/tools/autogenbench/scenarios/GAIA/Templates/Orchestrator/README.md

@@ -0,0 +1,8 @@
+# Multi-Agent Experiment v0.1
+### MSR AI Frontiers (AutoGen team members)
+##### March 1st, 2024
+##### Primary Contact: [afourney](https://github.com/afourney)
+  
+\
+\
+This GAIA submission represents an initial (i.e., very early) multi-agent experiment, where we are testing our benchmarking tools ([AutoGenBench](https://microsoft.github.io/autogen/blog/2024/01/25/AutoGenBench/)), and some experimental ideas on orchestration. Specifically, facts and “educated guesses” are tracked in a common ledger, and agents restart when they detect that they are not making progress. See scenario.py and orchestrator.py for full details.

samples/tools/autogenbench/scenarios/GAIA/Templates/Orchestrator/expected_answer.txt

@@ -0,0 +1 @@
+__EXPECTED_ANSWER__

samples/tools/autogenbench/scenarios/GAIA/Templates/Orchestrator/orchestrator.py

@@ -0,0 +1,294 @@
+# ruff: noqa: E722
+import json
+import traceback
+import copy
+import sys
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Union, Callable, Literal, Tuple
+from autogen import Agent, ConversableAgent, GroupChatManager, GroupChat, OpenAIWrapper
+
+
+class Orchestrator(ConversableAgent):
+    def __init__(
+        self,
+        name: str,
+        agents: List[ConversableAgent] = [],
+        is_termination_msg: Optional[Callable[[Dict], bool]] = None,
+        max_consecutive_auto_reply: Optional[int] = None,
+        human_input_mode: Optional[str] = "TERMINATE",
+        function_map: Optional[Dict[str, Callable]] = None,
+        code_execution_config: Union[Dict, Literal[False]] = False,
+        llm_config: Optional[Union[Dict, Literal[False]]] = False,
+        default_auto_reply: Optional[Union[str, Dict, None]] = "",
+    ):
+        super().__init__(
+            name=name,
+            system_message="",
+            is_termination_msg=is_termination_msg,
+            max_consecutive_auto_reply=max_consecutive_auto_reply,
+            human_input_mode=human_input_mode,
+            function_map=function_map,
+            code_execution_config=code_execution_config,
+            llm_config=llm_config,
+            default_auto_reply=default_auto_reply,
+        )
+
+        self._agents = agents
+        self.orchestrated_messages = []
+
+        # NOTE: Async reply functions are not yet supported with this contrib agent
+        self._reply_func_list = []
+        self.register_reply([Agent, None], Orchestrator.run_chat)
+        self.register_reply([Agent, None], ConversableAgent.generate_code_execution_reply)
+        self.register_reply([Agent, None], ConversableAgent.generate_function_call_reply)
+        self.register_reply([Agent, None], ConversableAgent.check_termination_and_human_reply)
+
+    def _print_thought(self, message):
+        print(self.name + " (thought)\n")
+        print(message.strip() + "\n")
+        print("\n", "-" * 80, flush=True, sep="")
+
+    def _broadcast(self, message, out_loud=[], exclude=[]):
+        m = copy.deepcopy(message)
+        m["role"] = "user"
+        for a in self._agents:
+            if a in exclude or a.name in exclude:
+                continue
+            if a in out_loud or a.name in out_loud:
+                self.send(message, a, request_reply=False, silent=False)
+            else:
+                self.send(message, a, request_reply=False, silent=True)
+
+    def run_chat(
+        self,
+        messages: Optional[List[Dict]] = None,
+        sender: Optional[Agent] = None,
+        config: Optional[OpenAIWrapper] = None,
+    ) -> Tuple[bool, Union[str, Dict, None]]:
+        # We should probably raise an error in this case.
+        if self.client is None:
+            return False, None
+
+        if messages is None:
+            messages = self._oai_messages[sender]
+
+        # Work with a copy of the messages
+        _messages = copy.deepcopy(messages)
+        
+        ##### Memory ####
+
+        # Pop the last message, which is the task
+        task = _messages.pop()["content"]
+   
+        # A reusable description of the team
+        team = "\n".join([a.name + ": " + a.description for a in self._agents])
+        names = ", ".join([a.name for a in self._agents])
+
+        # A place to store relevant facts
+        facts = ""
+
+        # A place to store the plan
+        plan = ""
+
+        #################
+
+        # Start by writing what we know
+        
+        closed_book_prompt = f"""Below I will present you a request. Before we begin addressing the request, please answer the following pre-survey to the best of your ability. Keep in mind that you are Ken Jennings-level with trivia, and Mensa-level with puzzles, so there should be a deep well to draw from.
+
+Here is the request:
+
+{task}
+
+Here is the pre-survey:
+
+    1. Please list any specific facts or figures that are GIVEN in the request itself. It is possible that there are none.
+    2. Please list any facts that may need to be looked up, and WHERE SPECIFICALLY they might be found. In some cases, authoritative sources are mentioned in the request itself.
+    3. Please list any facts that may need to be derived (e.g., via logical deduction, simulation, or computation)
+    4. Please list any facts that are recalled from memory, hunches, well-reasoned guesses, etc.
+
+When answering this survey, keep in mind that "facts" will typically be specific names, dates, statistics, etc. Your answer should use headings:
+
+    1. GIVEN OR VERIFIED FACTS
+    2. FACTS TO LOOK UP
+    3. FACTS TO DERIVE
+    4. EDUCATED GUESSES
+""".strip()
+
+        _messages.append({"role": "user", "content": closed_book_prompt, "name": sender.name})
+
+        response = self.client.create(
+            messages=_messages,
+            cache=self.client_cache,
+        )
+        extracted_response = self.client.extract_text_or_completion_object(response)[0]
+        _messages.append({"role": "assistant", "content": extracted_response, "name": self.name})
+        facts = extracted_response
+
+        # Make an initial plan
+        plan_prompt = f"""Fantastic. To address this request we have assembled the following team:
+
+{team}
+
+Based on the team composition, and known and unknown facts, please devise a short bullet-point plan for addressing the original request. Remember, there is no requirement to involve all team members -- a team member's particular expertise may not be needed for this task.""".strip()
+        _messages.append({"role": "user", "content": plan_prompt, "name": sender.name})
+
+        response = self.client.create(
+            messages=_messages,
+            cache=self.client_cache,
+        )
+
+        extracted_response = self.client.extract_text_or_completion_object(response)[0]
+        _messages.append({"role": "assistant", "content": extracted_response, "name": self.name})
+        plan = extracted_response
+
+        # Main loop
+        total_turns = 0
+        max_turns = 30
+        while total_turns < max_turns:
+
+            # Populate the message histories
+            self.orchestrated_messages = []
+            for a in self._agents:
+                a.reset()
+
+            self.orchestrated_messages.append({"role": "assistant", "content": f"""
+We are working to address the following user request:
+
+{task}
+
+
+To answer this request we have assembled the following team:
+
+{team}
+
+Some additional points to consider:
+
+{facts}
+
+{plan}
+""".strip(), "name": self.name})
+            self._broadcast(self.orchestrated_messages[-1])
+            self._print_thought(self.orchestrated_messages[-1]["content"])
+
+            # Inner loop
+            stalled_count = 0
+            while total_turns < max_turns:
+                total_turns += 1
+
+                step_prompt = f"""
+Recall we are working on the following request:
+
+{task}
+
+And we have assembled the following team:
+
+{team}
+
+To make progress on the request, please answer the following questions, including necessary reasoning:
+
+    - Is the request fully satisfied? (True if complete, or False if the original request has yet to be SUCCESSFULLY addressed)
+    - Are we making forward progress? (True if just starting, or recent messages are adding value. False if recent messages show evidence of being stuck in a reasoning or action loop, or there is evidence of significant barriers to success such as the inability to read from a required file)
+    - Who should speak next? (select from: {names})
+    - What instruction or question would you give this team member? (Phrase as if speaking directly to them, and include any specific information they may need)
+
+Please output an answer in pure JSON format according to the following schema. The JSON object must be parsable as-is. DO NOT OUTPUT ANYTHING OTHER THAN JSON, AND DO NOT DEVIATE FROM THIS SCHEMA:
+
+    {{
+        "is_request_satisfied": {{
+            "reason": string,
+            "answer": boolean
+        }},
+        "is_progress_being_made": {{
+            "reason": string,
+            "answer": boolean
+        }},
+        "next_speaker": {{
+            "reason": string,
+            "answer": string (select from: {names})
+        }},
+        "instruction_or_question": {{
+            "reason": string,
+            "answer": string
+        }}
+    }}
+""".strip()
+
+                # This is a temporary message we will immediately pop
+                self.orchestrated_messages.append({"role": "user", "content": step_prompt, "name": sender.name})
+                response = self.client.create(
+                    messages=self.orchestrated_messages,
+                    cache=self.client_cache,
+                    response_format={"type": "json_object"},
+                )
+                self.orchestrated_messages.pop()
+
+                extracted_response = self.client.extract_text_or_completion_object(response)[0]
+                data = None
+                try:
+                    data = json.loads(extracted_response)
+                except json.decoder.JSONDecodeError as e:
+                    # Something went wrong. Restart this loop.
+                    self._print_thought(str(e))
+                    break
+
+                self._print_thought(json.dumps(data, indent=4))
+
+                if data["is_request_satisfied"]["answer"]:
+                    return True, "TERMINATE"
+
+                if data["is_progress_being_made"]["answer"]:
+                    stalled_count -= 1
+                    stalled_count = max(stalled_count, 0)
+                else:
+                    stalled_count += 1
+
+                if stalled_count >= 3: 
+                    self._print_thought("We aren't making progress. Let's reset.")
+                    new_facts_prompt = f"""It's clear we aren't making as much progress as we would like, but we may have learned something new. Please rewrite the following fact sheet, updating it to include anything new we have learned. This is also a good time to update educated guesses (please add or update at least one educated guess or hunch, and explain your reasoning). 
+
+{facts}
+""".strip()
+                    self.orchestrated_messages.append({"role": "user", "content": new_facts_prompt, "name": sender.name})
+                    response = self.client.create(
+                        messages=self.orchestrated_messages,
+                        cache=self.client_cache,
+                    )
+                    facts = self.client.extract_text_or_completion_object(response)[0]
+                    self.orchestrated_messages.append({"role": "assistant", "content": facts, "name": self.name})
+
+
+                    new_plan_prompt = f"""Please come up with a new plan expressed in bullet points. Keep in mind the following team composition, and do not involve any other outside people in the plan -- we cannot contact anyone else.
+
+Team membership:
+{team}
+""".strip()
+                    self.orchestrated_messages.append({"role": "user", "content": new_plan_prompt, "name": sender.name})
+                    response = self.client.create(
+                        messages=self.orchestrated_messages,
+                        cache=self.client_cache,
+                    )
+
+                    plan = self.client.extract_text_or_completion_object(response)[0]
+                    break
+
+                # Broadcast the message to all agents
+                m = {"role": "user", "content": data["instruction_or_question"]["answer"], "name": self.name}
+                if m["content"] is None:
+                    m["content"] = ""
+                self._broadcast(m, out_loud=[data["next_speaker"]["answer"]])
+
+                # Keep a copy
+                m["role"] = "assistant"
+                self.orchestrated_messages.append(m)
+
+                # Request a reply
+                for a in self._agents:
+                    if a.name == data["next_speaker"]["answer"]:
+                        reply = {"role": "user", "name": a.name, "content": a.generate_reply(sender=self)}
+                        self.orchestrated_messages.append(reply)
+                        a.send(reply, self, request_reply=False)
+                        self._broadcast(reply, exclude=[a])
+                        break
+
+        return True, "TERMINATE"

samples/tools/autogenbench/scenarios/GAIA/Templates/Orchestrator/prompt.txt

@@ -0,0 +1 @@
+__PROMPT__

samples/tools/autogenbench/scenarios/GAIA/Templates/Orchestrator/requirements.txt

@@ -0,0 +1,4 @@
+/autogen[websurfer]
+youtube_transcript_api
+mammoth
+puremagic

samples/tools/autogenbench/scenarios/GAIA/Templates/Orchestrator/scenario.py

@@ -0,0 +1,203 @@
+# ruff: noqa: E722
+import os
+import sys
+import json
+import autogen
+import copy
+import traceback
+import re
+from datetime import datetime
+import testbed_utils
+from autogen.agentchat.contrib.web_surfer import WebSurferAgent
+from autogen.token_count_utils import count_token, get_max_token_limit
+from autogen.mdconvert import MarkdownConverter, UnsupportedFormatException, FileConversionException
+from orchestrator import Orchestrator
+
+testbed_utils.init()
+##############################
+
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode("utf-8")
+
+# Read the prompt
+PROMPT = ""
+with open("prompt.txt", "rt") as fh:
+    PROMPT = fh.read().strip()
+
+config_list = autogen.config_list_from_json( "OAI_CONFIG_LIST",)
+
+llm_config = testbed_utils.default_llm_config(
+        autogen.filter_config(config_list, {"tags": ["llm"]}),
+        timeout=300
+)
+llm_config["temperature"] = 0.1
+summarizer_llm_config = llm_config
+final_llm_config = llm_config
+
+gpt4v = autogen.filter_config(config_list, {"tags": ["mlm"]})[0]
+
+client = autogen.OpenAIWrapper(**final_llm_config)
+mlm_client = autogen.OpenAIWrapper(**gpt4v)
+
+def response_preparer(inner_messages):
+
+    messages = [
+        {
+            "role": "user",
+            "content": f"""Earlier you were asked the following:
+
+{PROMPT}
+
+Your team then worked diligently to address that request. Here is a transcript of that conversation:""",
+        }
+    ]
+
+    # The first message just repeats the question, so remove it
+    #if len(inner_messages) > 1:
+    #    del inner_messages[0]
+
+    # copy them to this context
+    for message in inner_messages:
+        if not message.get("content"):
+            continue
+        message = copy.deepcopy(message)
+        message["role"] = "user"
+        messages.append(message)
+
+    # ask for the final answer
+    messages.append(
+        {
+            "role": "user",
+            "content": f"""
+Read the above conversation and output a FINAL ANSWER to the question. The question is repeated here for convenience:
+
+{PROMPT}
+
+To output the final answer, use the following template: FINAL ANSWER: [YOUR FINAL ANSWER]
+Your FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings.
+ADDITIONALLY, your FINAL ANSWER MUST adhere to any formatting instructions specified in the original question (e.g., alphabetization, sequencing, units, rounding, decimal places, etc.)
+If you are asked for a number, express it numerically (i.e., with digits rather than words), don't use commas, and don't include units such as $ or percent signs unless specified otherwise.
+If you are asked for a string, don't use articles or abbreviations (e.g. for cities), unless specified otherwise. Don't output any final sentence punctuation such as '.', '!', or '?'.
+If you are asked for a comma separated list, apply the above rules depending on whether the elements are numbers or strings.
+If you are unable to determine the final answer, output 'FINAL ANSWER: Unable to determine'
+""",
+        }
+    )
+
+    response = client.create(context=None, messages=messages)
+    if "finish_reason='content_filter'" in str(response):
+        raise Exception(str(response))
+    extracted_response = client.extract_text_or_completion_object(response)[0]
+
+    # No answer
+    if "unable to determine" in extracted_response.lower():
+        print("\n>>>Making an educated guess.\n")
+        messages.append({"role": "assistant", "content": extracted_response })
+        messages.append({"role": "user", "content": """
+I understand that a definitive answer could not be determined. Please make a well-informed EDUCATED GUESS based on the conversation.
+
+To output the educated guess, use the following template: EDUCATED GUESS: [YOUR EDUCATED GUESS]
+Your EDUCATED GUESS should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. DO NOT OUTPUT 'I don't know', 'Unable to determine', etc.
+ADDITIONALLY, your EDUCATED GUESS MUST adhere to any formatting instructions specified in the original question (e.g., alphabetization, sequencing, units, rounding, decimal places, etc.)
+If you are asked for a number, express it numerically (i.e., with digits rather than words), don't use commas, and don't include units such as $ or percent signs unless specified otherwise.
+If you are asked for a string, don't use articles or abbreviations (e.g. for cities), unless specified otherwise. Don't output any final sentence punctuation such as '.', '!', or '?'.
+If you are asked for a comma separated list, apply the above rules depending on whether the elements are numbers or strings.
+""".strip()})
+
+        response = client.create(context=None, messages=messages)
+        if "finish_reason='content_filter'" in str(response):
+            raise Exception(str(response))
+        extracted_response = client.extract_text_or_completion_object(response)[0]
+
+        return re.sub(r"EDUCATED GUESS:", "FINAL ANSWER:", extracted_response)  
+    else:
+        return extracted_response
+
+assistant = autogen.AssistantAgent(
+    "assistant",
+    is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
+    code_execution_config=False,
+    llm_config=llm_config,
+)
+user_proxy = autogen.UserProxyAgent(
+    "computer_terminal",
+    human_input_mode="NEVER",
+    description="A computer terminal that performs no other action than running Python scripts (provided to it quoted in ```python code blocks), or sh shell scripts (provided to it quoted in ```sh code blocks)",
+    is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
+    code_execution_config={
+        "work_dir": "coding",
+        "use_docker": False,
+    },
+    default_auto_reply="",
+    max_consecutive_auto_reply=15,
+)
+
+user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0"
+
+web_surfer = WebSurferAgent(
+    "web_surfer",
+    llm_config=llm_config,
+    summarizer_llm_config=summarizer_llm_config,
+    is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
+    code_execution_config=False,
+    browser_config={
+        "bing_api_key": os.environ["BING_API_KEY"],
+        "viewport_size": 1024 * 5,
+        "downloads_folder": "coding",
+        "request_kwargs": {
+            "headers": {"User-Agent": user_agent},
+        },
+    },
+)
+
+maestro = Orchestrator(
+    "orchestrator",
+    agents=[assistant, user_proxy, web_surfer],
+    llm_config=llm_config,
+)
+
+filename = "__FILE_NAME__".strip()
+
+filename_prompt = ""
+if len(filename) > 0:
+    relpath = os.path.join("coding", filename)
+    filename_prompt = f"The question is about a file, document or image, which can be read from the file '{filename}' in current working directory."
+
+    mdconverter = MarkdownConverter( mlm_client=mlm_client)
+    mlm_prompt=f"""Write a detailed caption for this image. Pay special attention to any details that might be useful for someone answering the following:
+
+{PROMPT}
+""".strip()
+
+    try:
+        res = mdconverter.convert(relpath, mlm_prompt=mlm_prompt)
+        filename_prompt += " Here are the file's contents:\n\n" + res.text_content
+    except UnsupportedFormatException:
+        pass
+    except FileConversionException:
+        traceback.print_exc()
+
+
+question = f"""{PROMPT}
+
+{filename_prompt}
+""".strip()
+
+try:
+    # Initiate one turn of the conversation
+    user_proxy.send(
+        question,
+        maestro,
+        request_reply=True,
+        silent=False,
+    )
+except:
+    traceback.print_exc()
+
+
+print()
+print(response_preparer(maestro.orchestrated_messages))
+
+##############################
+testbed_utils.finalize(agents=[assistant, user_proxy, web_surfer, maestro])

samples/tools/autogenbench/scenarios/GAIA/Templates/SocietyOfMind/group_chat_moderator.py

@@ -0,0 +1,61 @@
+from typing import Callable, Dict, Optional, Union, Tuple, List, Any
+from autogen import GroupChat, Agent, ConversableAgent
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class GroupChatModerator(GroupChat):
+    """(Experimental) A variation of the standard GroupChat class, but with an alternate prompting strategy
+    that focus on conversation moderation rather than role play. A drop-in replacement for GroupChat."""
+
+    def __init__(
+        self,
+        agents: List[Agent],
+        messages: List[Dict],
+        max_round: int = 10,
+        admin_name: str = "Admin",
+        func_call_filter: bool = True,
+        speaker_selection_method: str = "auto",
+        allow_repeat_speaker: Optional[Union[bool, List[Agent]]] = True,
+        first_speaker: Agent = None,
+        send_introductions: bool = False,
+    ):
+        """
+        GroupChatModerator uses the same initialization and constructor as GroupChat.
+        Please refer to the GroupChat constructor for more information.
+        """
+        super().__init__(
+            agents=agents,
+            messages=messages,
+            max_round=max_round,
+            admin_name=admin_name,
+            func_call_filter=func_call_filter,
+            speaker_selection_method=speaker_selection_method,
+            allow_repeat_speaker=allow_repeat_speaker,
+            send_introductions=send_introductions,
+        )
+        self.first_speaker = first_speaker
+        self._selection_turns = 0
+
+    # Enable specification of who speaks first
+    def select_speaker(self, last_speaker: Agent, selector: ConversableAgent):
+        self._selection_turns += 1
+        if self.first_speaker is not None and self._selection_turns == 1:
+            return self.first_speaker
+        return super().select_speaker(last_speaker, selector)
+
+    def select_speaker_msg(self, agents: List[Agent]):
+        """Return the system message for selecting the next speaker. This is always the *first* message in the context."""
+        return f"""You are moderating a conversation between the following participants:
+
+{self._participant_roles(agents)}
+
+Read the following conversation, then carefully consider who should speak next based on who's input would be most valued in this moment (e.g., to make the most progress on the task). Speakers do not need equal speaking time. You may even ignore non-relevant participants. Your focus is on efficiently driving progress toward task completion.
+
+You must select only one speaker to go next, and you must only return their name (i.e., from the set {[agent.name for agent in agents]})
+"""
+
+    def select_speaker_prompt(self, agents: List[Agent]):
+        """Return the floating system prompt selecting the next speaker. This is always the *last* message in the context."""
+        return f"Read the above conversation, then carefully consider who should speak next based on who's input would be most valued in this moment to make progress on the task. Select the next speaker from {[agent.name for agent in agents]}. Only return their name."

samples/tools/autogenbench/scenarios/GAIA/Templates/SocietyOfMind/prompt.txt

@@ -0,0 +1 @@
+__PROMPT__

samples/tools/autogenbench/scenarios/GAIA/Templates/SocietyOfMind/requirements.txt

@@ -1,4 +1,2 @@
-git+https://github.com/microsoft/autogen.git@society_of_mind_gaia
-pdfminer.six
-markdownify
-pathvalidate
+/autogen[websurfer]
+youtube_transcript_api

samples/tools/autogenbench/scenarios/GAIA/Templates/SocietyOfMind/scenario.py

@@ -5,55 +5,51 @@ import json
 import autogen
 import copy
 import traceback
+import mimetypes
+import base64
+import re
 from datetime import datetime
 import testbed_utils
 from autogen.agentchat.contrib.web_surfer import WebSurferAgent
 from autogen.agentchat.contrib.society_of_mind_agent import SocietyOfMindAgent
-from autogen.agentchat.contrib.group_chat_moderator import GroupChatModerator
 from autogen.token_count_utils import count_token, get_max_token_limit
+from group_chat_moderator import GroupChatModerator
+from autogen.agentchat.contrib.functions import file_utils as futils
 
 testbed_utils.init()
 ##############################
 
-config_list = autogen.config_list_from_json(
-    "OAI_CONFIG_LIST",
-    filter_dict={"model": ["gpt-4"]},
-)
-llm_config = testbed_utils.default_llm_config(config_list, timeout=180)
-llm_config["temperature"] = 0.1
 
-summarizer_config_list = autogen.config_list_from_json(
-    "OAI_CONFIG_LIST",
-    filter_dict={"model": ["gpt-3.5-turbo-16k"]},
-)
-summarizer_llm_config = testbed_utils.default_llm_config(summarizer_config_list, timeout=180)
-summarizer_llm_config["temperature"] = 0.1
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode("utf-8")
 
-final_config_list = autogen.config_list_from_json(
-    "OAI_CONFIG_LIST",
-    filter_dict={"model": ["gpt-4-1106-preview"]},
-)
-final_llm_config = testbed_utils.default_llm_config(final_config_list, timeout=180)
-final_llm_config["temperature"] = 0.1
 
+# Read the prompt
+PROMPT = ""
+with open("prompt.txt", "rt") as fh:
+    PROMPT = fh.read().strip()
 
-client = autogen.OpenAIWrapper(**final_llm_config)
+config_list = autogen.config_list_from_json( "OAI_CONFIG_LIST",)
+llm_config = testbed_utils.default_llm_config(config_list, timeout=300)
+llm_config["temperature"] = 0.1
 
+summarizer_llm_config = llm_config
+final_llm_config = llm_config
 
-def response_preparer(inner_messages):
-    tokens = 0
+client = autogen.OpenAIWrapper(**final_llm_config)
 
+def response_preparer(agent, inner_messages):
     messages = [
         {
             "role": "user",
-            "content": """Earlier you were asked the following:
+            "content": f"""Earlier you were asked the following:
 
-__PROMPT__
+{PROMPT}
 
 Your team then worked diligently to address that request. Here is a transcript of that conversation:""",
         }
     ]
-    tokens += count_token(messages[-1])
 
     # The first message just repeats the question, so remove it
     if len(inner_messages) > 1:
@@ -64,35 +60,46 @@ Your team then worked diligently to address that request. Here is a transcript o
         message = copy.deepcopy(message)
         message["role"] = "user"
         messages.append(message)
-        tokens += count_token(messages[-1])
 
+    # ask for the final answer
     messages.append(
         {
             "role": "user",
-            "content": """
+            "content": f"""
 Read the above conversation and output a FINAL ANSWER to the question. The question is repeated here for convenience:
 
-__PROMPT__
+{PROMPT}
 
 To output the final answer, use the following template: FINAL ANSWER: [YOUR FINAL ANSWER]
 YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings.
 If you are asked for a number, don’t use comma to write your number neither use units such as $ or percent sign unless specified otherwise, and don't output any final sentence punctuation such as '.', '!', or '?'.
 If you are asked for a string, don’t use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise.
-If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.""",
+If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string. 
+If you are unable to determine the final answer, output 'FINAL ANSWER: Unable to determine.'
+""",
         }
     )
-    tokens += count_token(messages[-1])
-
-    #    # Hardcoded
-    #    while tokens > 3200:
-    #        mid = int(len(messages) / 2)  # Remove from the middle
-    #        tokens -= count_token(messages[mid])
-    #        del messages[mid]
 
     response = client.create(context=None, messages=messages)
     extracted_response = client.extract_text_or_completion_object(response)[0]
-    if not isinstance(extracted_response, str):
-        return str(extracted_response.model_dump(mode="dict"))  # Not sure what to do here
+
+    # No answer
+    if "unable to determine" in extracted_response.lower():
+        print("\n>>>Making an educated guess.\n")
+        messages.append({"role": "assistant", "content": extracted_response })
+        messages.append({"role": "user", "content": """
+I understand that a definitive answer could not be determined. Please make a well-informed EDUCATED GUESS based on the conversation.
+
+To output the educated guess, use the following template: EDUCATED GUESS: [YOUR EDUCATED GUESS]
+YOUR EDUCATED GUESS should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. DO NOT OUTPUT 'I don't know', 'Unable to determine', etc.
+If you are asked for a number, don’t use comma to write your number neither use units such as $ or percent sign unless specified otherwise, and don't output any final sentence punctuation such as '.', '!', or '?'.
+If you are asked for a string, don’t use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise.
+If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string. 
+""".strip()})
+
+        response = client.create(context=None, messages=messages)
+        extracted_response = client.extract_text_or_completion_object(response)[0]
+        return re.sub(r"EDUCATED GUESS:", "FINAL ANSWER:", extracted_response)  
     else:
         return extracted_response
 
@@ -100,6 +107,7 @@ If you are asked for a comma separated list, apply the above rules depending of
 assistant = autogen.AssistantAgent(
     "assistant",
     is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
+    code_execution_config=False,
     llm_config=llm_config,
 )
 user_proxy = autogen.UserProxyAgent(
@@ -122,6 +130,7 @@ web_surfer = WebSurferAgent(
     llm_config=llm_config,
     summarizer_llm_config=summarizer_llm_config,
     is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
+    code_execution_config=False,
     browser_config={
         "bing_api_key": os.environ["BING_API_KEY"],
         "viewport_size": 1024 * 5,
@@ -132,15 +141,74 @@ web_surfer = WebSurferAgent(
     },
 )
 
-filename_prompt = "__FILE_NAME__".strip()
-if len(filename_prompt) > 0:
-    filename_prompt = f"Consider the file '{filename_prompt}' which can be read from the current working directory. If you need to read or write it, output python code in a code block (```python) to do so. "
-
+filename = "__FILE_NAME__".strip()
+
+filename_prompt = ""
+if len(filename) > 0:
+    content_type, encoding = mimetypes.guess_type(filename)
+    relpath = os.path.join("coding", filename)
+    filename_prompt = f"The question is about a file, document or image, which can be read from the file '{filename}' in current working directory."
+    if re.search(r"\.docx?$", filename.lower()):
+        filename_prompt += " It is a word document. Its contents are:\n\n" + futils.read_text_from_docx(relpath)
+    elif re.search(r"\.xlsx?$", filename.lower()):
+        filename_prompt += " It is an excel document. Its contents are:\n\n" + futils.read_text_from_xlsx(relpath)
+    elif re.search(r"\.pptx?$", filename.lower()):
+        filename_prompt += " It is an powerpoint document. Its contents are:\n\n" + futils.read_text_from_pptx(relpath)
+    elif re.search(r"\.pdf$", filename.lower()):
+        filename_prompt += " It is a PDF. Its contents are:\n\n" + futils.read_text_from_pdf(relpath)
+    elif re.search(r"\.mp3$", filename.lower()):
+        from pydub import AudioSegment
+
+        sound = AudioSegment.from_mp3(relpath)
+        wave_fname = relpath + ".wav"
+        sound.export(wave_fname, format="wav")
+        filename_prompt += " It is an Audio file. Here is its transcript:\n\n" + futils.read_text_from_audio(wave_fname)
+    elif re.search(r"\.wav$", filename.lower()):
+        filename_prompt += " It is an Audio file. Here is its transcript:\n\n" + futils.read_text_from_audio(relpath)
+    elif re.search(r"\.jpe?g$", filename.lower()):
+        filename_prompt += " It is an image with the following description:\n\n "
+
+        img_prompt = f"""
+Provide a meaningful but concise alt-text description of the image following established best practices (which focus on conveying context, meaning, information and purpose in addition to "looks"). This text should be useful for a low-vision or blind user encountering the image in the context of addressing the following request:
+
+{PROMPT}
+        """.strip()
+        filename_prompt += futils.caption_image_using_gpt4v( "data:image/jpeg;base64," + encode_image(relpath), img_prompt)
+        ocr = futils.read_text_from_image(relpath).strip()
+        if ocr != "":
+            filename_prompt += "\n\nAdditionally, OCR analysis has detected the following text in the image: \"" + ocr + "\""
+    elif re.search(r"\.png$", filename.lower()):
+        filename_prompt += " It is an image with the following description:\n\n "
+
+        img_prompt = f"""
+Provide a meaningful but concise alt-text description of the image following established best practices (which focus on conveying context, meaning, information and purpose in addition to "looks"). This text should be useful for a low-vision or blind user encountering the image in the context of addressing the following request:
+
+{PROMPT}
+        """.strip()
+        filename_prompt += futils.caption_image_using_gpt4v( "data:image/png;base64," + encode_image(relpath), img_prompt)
+
+        from PIL import Image
+
+        img = Image.open(relpath)
+        # Remove transparency
+        if img.mode in ("RGBA", "P"):
+            img = img.convert("RGB")
+        jpg_name = relpath + ".jpg"
+        img.save(jpg_name)
+
+        ocr = futils.read_text_from_image(jpg_name).strip()
+        if ocr != "":
+            filename_prompt += "\n\nAdditionally, OCR analysis has detected the following text in the image: \"" + ocr + "\""
+    elif content_type is not None and "text/" in content_type.lower():
+        with open(relpath, "rt") as fh:
+            filename_prompt += "Here are the file's contents:\n\n" + fh.read().strip()
 
 question = f"""
 Below I will pose a question to you that I would like you to answer. You should begin by listing all the relevant facts necessary to derive an answer, then fill in those facts from memory where possible, including specific names, numbers and statistics. You are Ken Jennings-level with trivia, and Mensa-level with puzzles, so there should be a deep well to draw from. After listing the facts, begin to solve the question in earnest. Here is the question:
 
-{filename_prompt}__PROMPT__
+{PROMPT}
+
+{filename_prompt}
 """.strip()
 
 groupchat = GroupChatModerator(
@@ -150,13 +218,14 @@ groupchat = GroupChatModerator(
     messages=[],
     speaker_selection_method="auto",
     allow_repeat_speaker=[web_surfer, assistant],
+    send_introductions=True,
 )
 
 manager = autogen.GroupChatManager(
     groupchat=groupchat,
     is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
-    send_introductions=True,
     llm_config=llm_config,
+    code_execution_config=False,
 )
 
 soc = SocietyOfMindAgent(
@@ -164,6 +233,7 @@ soc = SocietyOfMindAgent(
     chat_manager=manager,
     response_preparer=response_preparer,
     llm_config=llm_config,
+    code_execution_config=False,
 )
 
 try:

samples/tools/autogenbench/scenarios/HumanEval/MANIFEST.json

@@ -3,6 +3,7 @@
         "Templates/TwoAgents/prompt.txt": "Templates/TwoAgents/prompt.txt",
         "Templates/TwoAgents/coding/my_tests.py": "Templates/TwoAgents/coding/my_tests.py",
         "Templates/TwoAgents/scenario.py": "Templates/TwoAgents/scenario.py",
+        "Templates/TwoAgents/requirements.txt": "Templates/TwoAgents/requirements.txt",
         "README.md": "README.md",
 	"Scripts/init_tasks.py": "Scripts/init_tasks.py",
 	"Scripts/custom_tabulate.py": "Scripts/custom_tabulate.py"

samples/tools/autogenbench/scenarios/HumanEval/Scripts/init_tasks.py

@@ -8,6 +8,7 @@ import gzip
 import io
 import json
 import os
+import re
 import base64
 
 URL = "https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz"
@@ -16,7 +17,13 @@ SCRIPT_PATH = os.path.realpath(__file__)
 SCRIPT_NAME = os.path.basename(SCRIPT_PATH)
 SCRIPT_DIR = os.path.dirname(SCRIPT_PATH)
 
+SCENARIO_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.path.pardir))
+TEMPLATES_DIR = os.path.join(SCENARIO_DIR, "Templates")
+TASKS_DIR = os.path.join(SCENARIO_DIR, "Tasks")
+
 # A selected subset of HumanEval problems to work with during development
+
+# Deprecated 2/5/2024 -- Use subsample instead
 REDUCED_SET = [
     "HumanEval/2",
     "HumanEval/26",
@@ -73,19 +80,17 @@ def create_jsonl(name, tasks, template):
     """Creates a JSONL scenario file with a given name, list of HumanEval tasks, and template path."""
 
     # Create a task directory if it doesn't exist
-    scenario_dir = os.path.realpath(os.path.join(SCRIPT_DIR, os.path.pardir))
-    task_dir = os.path.join(scenario_dir, "Tasks")
-    if not os.path.isdir(task_dir):
-        os.mkdir(task_dir)
+    if not os.path.isdir(TASKS_DIR):
+        os.mkdir(TASKS_DIR)
 
     # Create the jsonl file
-    with open(os.path.join(task_dir, name + ".jsonl"), "wt") as fh:
+    with open(os.path.join(TASKS_DIR, name + ".jsonl"), "wt") as fh:
         for task in tasks:
             print(f"Converting: [{name}] {task['task_id']}")
 
             record = {
                 "id": task["task_id"].replace("/", "_"),
-                "template": os.path.join(os.path.pardir, template),
+                "template": template,
                 "substitutions": {
                     "scenario.py": {
                         "__ENTRY_POINT__": task["entry_point"],
@@ -102,19 +107,19 @@ def create_jsonl(name, tasks, template):
 ###############################################################################
 def main():
     human_eval = download_human_eval()
-    reduced_human_eval = [t for t in human_eval if t["task_id"] in REDUCED_SET]
+    # Deprecated: reduced_human_eval = [t for t in human_eval if t["task_id"] in REDUCED_SET]
 
-    templates = {
-        "two_agents": "Templates/TwoAgents",
-        # "gc3_distractor": "Templates/GroupChatThreeAgents_Distractor",
-        # "gc3_guardrails": "Templates/GroupChatThreeAgents_Guardrails",
-        # "gc4": "Templates/GroupChatFourAgents",
-    }
+    # list all directories in the Templates directory
+    # and populate a dictionary with the name and path
+    templates = {}
+    for entry in os.scandir(TEMPLATES_DIR):
+        if entry.is_dir():
+            templates[re.sub(r"\s", "", entry.name)] = entry.path
 
     # Create the various combinations of [models] x [templates]
     for t in templates.items():
         create_jsonl(f"human_eval_{t[0]}", human_eval, t[1])
-        create_jsonl(f"r_human_eval_{t[0]}", reduced_human_eval, t[1])
+        # Deprecated: create_jsonl(f"r_human_eval_{t[0]}", reduced_human_eval, t[1])
 
 
 if __name__ == "__main__" and __package__ is None:

samples/tools/autogenbench/scenarios/HumanEval/Templates/GroupChatFourAgents/requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/microsoft/autogen.git@complex_tasks

samples/tools/autogenbench/scenarios/HumanEval/Templates/GroupChatThreeAgents_Distractor/requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/microsoft/autogen.git@complex_tasks

samples/tools/autogenbench/scenarios/HumanEval/Templates/GroupChatThreeAgents_Guardrails/requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/microsoft/autogen.git@complex_tasks

samples/tools/autogenbench/scenarios/HumanEval/Templates/TwoAgents/requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/microsoft/autogen.git@complex_tasks

samples/tools/autogenbench/scenarios/MATH/MANIFEST.json

@@ -6,6 +6,7 @@
         "Templates/TwoAgents/prompt.txt": "Templates/TwoAgents/prompt.txt",
         "Templates/TwoAgents/expected_answer.txt": "Templates/TwoAgents/expected_answer.txt",
         "Templates/TwoAgents/scenario.py": "Templates/TwoAgents/scenario.py",
-        "Templates/TwoAgents/scenario_init.sh": "Templates/TwoAgents/scenario_init.sh"
+        "Templates/TwoAgents/scenario_init.sh": "Templates/TwoAgents/scenario_init.sh",
+        "Templates/TwoAgents/requirements.txt": "Templates/TwoAgents/requirements.txt"
     }
 }

samples/tools/autogenbench/scenarios/MATH/Scripts/init_tasks.py

@@ -8,6 +8,7 @@ import tarfile
 import io
 import json
 import os
+import re
 import sys
 
 URL = "https://people.eecs.berkeley.edu/~hendrycks/MATH.tar"
@@ -91,7 +92,7 @@ def create_jsonl(name, problems, template):
 
             record = {
                 "id": task_id,
-                "template": os.path.join(os.path.pardir, template),
+                "template": template,
                 "substitutions": {
                     "prompt.txt": {"__PROMPT__": data["problem"]},
                     "expected_answer.txt": {"__ANSWER__": data["solution"]},
@@ -105,9 +106,12 @@ def create_jsonl(name, problems, template):
 def main():
     problems = download_math()
 
-    templates = {
-        "two_agents": "Templates/TwoAgents",
-    }
+    # list all directories in the Templates directory
+    # and populate a dictionary with the name and path
+    templates = {}
+    for entry in os.scandir(TEMPLATES_DIR):
+        if entry.is_dir():
+            templates[re.sub(r"\s", "", entry.name)] = entry.path
 
     for t in templates.items():
         create_jsonl(f"math_{t[0]}", problems, t[1])

samples/tools/autogenbench/scenarios/MATH/Templates/TwoAgents/requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/microsoft/autogen.git@complex_tasks

test/agentchat/contrib/functions/test_file_utils.py

@@ -0,0 +1,60 @@
+import sys
+import os
+import pytest
+from autogen.agentchat.contrib.functions import file_utils as fu
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
+from conftest import skip_openai  # noqa: E402
+
+try:
+    from openai import OpenAI
+except ImportError:
+    skip = True
+else:
+    skip = False or skip_openai
+
+TESTDIR = os.path.join(os.path.join(os.path.dirname(__file__), "..", "..", ".."), "test_files")
+
+
+def test_read_text_from_pdf():
+    text = fu.read_text_from_pdf(os.path.join(TESTDIR, "example.pdf"))
+    assert isinstance(text, str)
+
+
+def test_read_text_from_docx():
+    text = fu.read_text_from_docx(os.path.join(TESTDIR, "example.docx"))
+    assert isinstance(text, str)
+
+
+def test_read_text_from_image():
+    for file in ["example.jpg", "example.png"]:
+        text = fu.read_text_from_image(os.path.join(TESTDIR, file))
+        assert isinstance(text, str)
+
+
+def test_read_text_from_pptx():
+    text = fu.read_text_from_pptx(os.path.join(TESTDIR, "example.pptx"))
+    assert isinstance(text, str)
+
+
+def test_read_text_from_xlsx():
+    text = fu.read_text_from_xlsx(os.path.join(TESTDIR, "example.xlsx"))
+    assert isinstance(text, str)
+
+
+# def test_read_text_from_audio():
+# TODO: Needs work + smaller test file
+#     for file in ["example.wav"]:
+#         text = fu.read_text_from_audio(os.path.join(TESTDIR, file))
+#         print(text)
+#     assert isinstance(text, str)
+
+
+@pytest.mark.skipif(
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR openai not installed OR requested to skip",
+)
+def test_caption_image_using_gpt4v():
+    for file in ["example.jpg", "example.png"]:
+        text = fu.caption_image_using_gpt4v(os.path.join(TESTDIR, file))
+        assert isinstance(text, str)

test/agentchat/contrib/test_web_surfer.py

@@ -14,6 +14,11 @@ from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST  # noqa: E402
 
 BLOG_POST_URL = "https://microsoft.github.io/autogen/blog/2023/04/21/LLM-tuning-math"
 BLOG_POST_TITLE = "Does Model and Inference Parameter Matter in LLM Applications? - A Case Study for MATH | AutoGen"
+BLOG_POST_FIND_ON_PAGE_QUERY = "The need for * cost saving is not specific to the math problems."
+BLOG_POST_FIND_ON_PAGE_MATCH = (
+    "The need for model selection, parameter tuning and cost saving is not specific to the math problems."
+)
+
 BING_QUERY = "Microsoft"
 
 try:
@@ -85,6 +90,25 @@ def test_web_surfer() -> None:
             response = function_map["page_down"]()
         assert f"Viewport position: Showing page {total_pages} of {total_pages}." in response
 
+        # Try to scroll too far up
+        for i in range(0, total_pages + 1):
+            response = function_map["page_up"]()
+        assert f"Viewport position: Showing page 1 of {total_pages}." in response
+
+        # Try find_on_page
+        response = function_map["find_on_page_ctrl_f"](BLOG_POST_FIND_ON_PAGE_QUERY)
+        assert BLOG_POST_FIND_ON_PAGE_MATCH in response
+
+        # Try find_next
+        response = function_map["find_on_page_ctrl_f"]("AutoGen")
+        assert "AutoGen" in response
+        response = function_map["find_next"]()
+        assert "AutoGen" in response
+
+        # Try to find something that doesn't exists
+        response = function_map["find_on_page_ctrl_f"]("7c748f9a-8dce-461f-a092-4e8d29913f2d")
+        assert "The search string '7c748f9a-8dce-461f-a092-4e8d29913f2d' was not found on this page" in response
+
         # Test web search -- we don't have a key in this case, so we expect it to raise an error (but it means the code path is correct)
         with pytest.raises(ValueError, match="Missing Bing API key."):
             response = function_map["informational_web_search"](BING_QUERY)
@@ -146,6 +170,10 @@ def test_web_surfer_oai() -> None:
 
     user_proxy.initiate_chat(web_surfer, message="What's this page about?")
 
+        user_proxy.initiate_chat(web_surfer, message="Find the string 'Redmond' on this page.")
+
+        user_proxy.initiate_chat(web_surfer, message="Find the next occurrence of the string.")
+
 
 @pytest.mark.skipif(
     skip_bing,

test/agentchat/test_agent_telemetry.py

@@ -0,0 +1,194 @@
+import pytest
+import autogen
+import autogen.telemetry
+import json
+import sys
+import uuid
+import sqlite3
+
+from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST
+from conftest import skip_openai
+
+try:
+    import openai
+except ImportError:
+    skip = True
+else:
+    skip = False or skip_openai
+
+if not skip:
+    config_list = autogen.config_list_from_json(
+        OAI_CONFIG_LIST,
+        filter_dict={
+            "model": ["gpt-4", "gpt-4-0314", "gpt-4-32k", "gpt-4-32k-0314", "gpt-4-32k-v0314"],
+        },
+        file_location=KEY_LOC,
+    )
+
+###############################################################
+
+
+def verify_log_completions_table(cur, teacher_message, student_message):
+    cur.execute(
+        """SELECT id, invocation_id, client_id, wrapper_id, session_id,
+            request, response, is_cached, cost, start_time, end_time FROM chat_completions;"""
+    )
+    rows = cur.fetchall()
+
+    assert len(rows) == 3
+
+    session_id = rows[0]["session_id"]
+
+    for idx, row in enumerate(rows):
+        assert (
+            row["invocation_id"] and str(uuid.UUID(row["invocation_id"], version=4)) == row["invocation_id"]
+        ), "invocation id is not valid uuid"
+        assert row["client_id"], "client id is empty"
+        assert row["wrapper_id"], "wrapper id is empty"
+        assert row["session_id"] and row["session_id"] == session_id
+
+        request = json.loads(row["request"])
+        first_request_message = request["messages"][0]["content"]
+        first_request_role = request["messages"][0]["role"]
+
+        if idx == 0 or idx == 2:
+            assert first_request_message == teacher_message
+        elif idx == 1:
+            assert first_request_message == student_message
+        assert first_request_role == "system"
+
+        response = json.loads(row["response"])
+        assert "choices" in response and len(response["choices"]) > 0
+
+        assert row["cost"] > 0
+        assert row["start_time"], "start timestamp is empty"
+        assert row["end_time"], "end timestamp is empty"
+
+
+def verify_agents_table(cur, teacher_message, student_message):
+    cur.execute("SELECT id, agent_id, wrapper_id, session_id, name, class, init_args, timestamp FROM agents")
+    rows = cur.fetchall()
+
+    assert len(rows) == 2
+
+    session_id = rows[0]["session_id"]
+
+    for idx, row in enumerate(rows):
+        assert row["wrapper_id"], "wrapper id is empty"
+        assert row["session_id"] and row["session_id"] == session_id
+
+        agent = json.loads(row["init_args"])
+        if idx == 0:
+            assert row["name"] == "teacher"
+            assert agent["name"] == "teacher"
+            agent["system_message"] == teacher_message
+        elif idx == 1:
+            assert row["name"] == "student"
+            assert agent["name"] == "student"
+            agent["system_message"] = student_message
+
+        assert "api_key" not in row["init_args"]
+        assert row["timestamp"], "timestamp is empty"
+
+
+def verify_oai_client_table(cur):
+    cur.execute("SELECT id, client_id, wrapper_id, session_id, class, init_args, timestamp FROM oai_clients")
+    rows = cur.fetchall()
+
+    assert len(rows) == 2
+    session_id = rows[0]["session_id"]
+
+    for row in rows:
+        assert row["client_id"], "client id is empty"
+        assert row["wrapper_id"], "wrapper id is empty"
+        assert row["session_id"] and row["session_id"] == session_id
+        assert row["class"] in ["AzureOpenAI", "OpenAI"]
+        init_args = json.loads(row["init_args"])
+        assert "api_version" in init_args
+        assert row["timestamp"], "timestamp is empty"
+
+
+def verify_oai_wrapper_table(cur):
+    cur.execute("SELECT id, wrapper_id, session_id, init_args, timestamp FROM oai_wrappers")
+    rows = cur.fetchall()
+
+    assert len(rows) == 2
+    session_id = rows[0]["session_id"]
+
+    for row in rows:
+        assert row["wrapper_id"], "wrapper id is empty"
+        assert row["session_id"] and row["session_id"] == session_id
+        init_args = json.loads(row["init_args"])
+        assert "config_list" in init_args
+        assert len(init_args["config_list"]) > 0
+        assert row["timestamp"], "timestamp is empty"
+
+
+def verify_keys_are_matching(cur):
+    query = """
+        SELECT * FROM chat_completions
+        INNER JOIN agents
+            ON chat_completions.wrapper_id = agents.wrapper_id
+            AND chat_completions.session_id = agents.session_id
+        INNER JOIN oai_clients
+            ON chat_completions.wrapper_id = oai_clients.wrapper_id
+            AND chat_completions.session_id = oai_clients.session_id
+        INNER JOIN oai_wrappers
+            ON chat_completions.wrapper_id = oai_wrappers.wrapper_id
+            AND chat_completions.session_id = oai_wrappers.session_id
+    """
+    cur.execute(query)
+    rows = cur.fetchall()
+    assert len(rows) == 3
+
+
+@pytest.mark.skipif(
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
+)
+def test_agent_telemetry():
+    autogen.telemetry.start_logging(dbname=":memory:")
+    llm_config = {"config_list": config_list}
+
+    teacher_message = """
+        You are roleplaying a math teacher, and your job is to help your students with linear algebra.
+        Keep your explanations short.
+    """
+    teacher = autogen.AssistantAgent(
+        "teacher",
+        system_message=teacher_message,
+        is_termination_msg=lambda x: x.get("content", "").find("TERMINATE") >= 0,
+        llm_config=llm_config,
+        max_consecutive_auto_reply=2,
+    )
+
+    student_message = """
+        You are roleplaying a high school student strugling with linear algebra.
+        Regardless how well the teacher explains things to you, you just don't quite get it.
+        Keep your questions short.
+    """
+    student = autogen.AssistantAgent(
+        "student",
+        system_message=student_message,
+        is_termination_msg=lambda x: x.get("content", "").find("TERMINATE") >= 0,
+        llm_config=llm_config,
+        max_consecutive_auto_reply=1,
+    )
+
+    student.initiate_chat(
+        teacher,
+        message="Can you explain the difference between eigenvalues and singular values again?",
+    )
+
+    con = autogen.telemetry.get_connection()
+    con.row_factory = sqlite3.Row
+
+    cur = con.cursor()
+
+    verify_log_completions_table(cur, teacher_message, student_message)
+    verify_agents_table(cur, teacher_message, student_message)
+    verify_oai_client_table(cur)
+    verify_oai_wrapper_table(cur)
+    verify_keys_are_matching(cur)
+
+    autogen.telemetry.stop_logging()

test/agentchat/test_groupchat.py

@@ -1,5 +1,3 @@
-from typing import Any, Dict, List, Optional, Type
-from autogen import AgentNameConflict
 import pytest
 from unittest import mock
 import builtins
@@ -674,134 +672,108 @@ def test_clear_agents_history():
     ]
 
 
-def test_get_agent_by_name():
-    def agent(name: str) -> autogen.ConversableAgent:
-        return autogen.ConversableAgent(
-            name=name,
-            max_consecutive_auto_reply=10,
-            human_input_mode="NEVER",
-            llm_config=False,
-        )
-
-    def team(members: List[autogen.Agent], name: str) -> autogen.Agent:
-        gc = autogen.GroupChat(agents=members, messages=[])
-
-        return autogen.GroupChatManager(groupchat=gc, name=name, llm_config=False)
-
-    team_member1 = agent("team1_member1")
-    team_member2 = agent("team1_member2")
-    team_dup_member1 = agent("team1_member1")
-    team_dup_member2 = agent("team1_member2")
-
-    user = agent("user")
-    team1 = team([team_member1, team_member2], "team1")
-    team1_duplicate = team([team_dup_member1, team_dup_member2], "team1")
-
-    gc = autogen.GroupChat(agents=[user, team1, team1_duplicate], messages=[])
-
-    # Testing default arguments
-    assert gc.agent_by_name("user") == user
-    assert gc.agent_by_name("team1") == team1 or gc.agent_by_name("team1") == team1_duplicate
-
-    # Testing recursive search
-    assert gc.agent_by_name("user", recursive=True) == user
-    assert (
-        gc.agent_by_name("team1_member1", recursive=True) == team_member1
-        or gc.agent_by_name("team1_member1", recursive=True) == team_dup_member1
+def test_send_intros():
+    agent1 = autogen.ConversableAgent(
+        "alice",
+        description="The first agent.",
+        max_consecutive_auto_reply=10,
+        human_input_mode="NEVER",
+        llm_config=False,
+        default_auto_reply="This is alice speaking. TERMINATE",
+    )
+    agent2 = autogen.ConversableAgent(
+        "bob",
+        description="The second agent.",
+        max_consecutive_auto_reply=10,
+        human_input_mode="NEVER",
+        llm_config=False,
+        default_auto_reply="This is bob speaking. TERMINATE",
+    )
+    agent3 = autogen.ConversableAgent(
+        "sam",
+        description="The third agent.",
+        max_consecutive_auto_reply=10,
+        human_input_mode="NEVER",
+        llm_config=False,
+        default_auto_reply="This is sam speaking. TERMINATE",
+    )
+    agent4 = autogen.ConversableAgent(
+        "sally",
+        description="The fourth agent.",
+        max_consecutive_auto_reply=10,
+        human_input_mode="NEVER",
+        llm_config=False,
+        default_auto_reply="This is sally speaking. TERMINATE",
     )
 
-    # Get agent that does not exist
-    assert gc.agent_by_name("team2") is None
-    assert gc.agent_by_name("team2", recursive=True) is None
-    assert gc.agent_by_name("team2", raise_on_name_conflict=True) is None
-    assert gc.agent_by_name("team2", recursive=True, raise_on_name_conflict=True) is None
-
-    # Testing naming conflict
-    with pytest.raises(AgentNameConflict):
-        gc.agent_by_name("team1", raise_on_name_conflict=True)
-
-    # Testing name conflict with recursive search
-    with pytest.raises(AgentNameConflict):
-        gc.agent_by_name("team1_member1", recursive=True, raise_on_name_conflict=True)
-
-
-def test_get_nested_agents_in_groupchat():
-    def agent(name: str) -> autogen.ConversableAgent:
-        return autogen.ConversableAgent(
-            name=name,
-            max_consecutive_auto_reply=10,
-            human_input_mode="NEVER",
-            llm_config=False,
-        )
-
-    def team(name: str) -> autogen.ConversableAgent:
-        member1 = agent(f"member1_{name}")
-        member2 = agent(f"member2_{name}")
-
-        gc = autogen.GroupChat(agents=[member1, member2], messages=[])
-
-        return autogen.GroupChatManager(groupchat=gc, name=name, llm_config=False)
-
-    user = agent("user")
-    team1 = team("team1")
-    team2 = team("team2")
-
-    gc = autogen.GroupChat(agents=[user, team1, team2], messages=[])
-
-    agents = gc.nested_agents()
-    assert len(agents) == 7
-
-
-def test_nested_teams_chat():
-    """Tests chat capabilities of nested teams"""
-    team1_msg = {"content": "Hello from team 1"}
-    team2_msg = {"content": "Hello from team 2"}
-
-    def agent(name: str, auto_reply: Optional[Dict[str, Any]] = None) -> autogen.ConversableAgent:
-        return autogen.ConversableAgent(
-            name=name,
-            max_consecutive_auto_reply=10,
-            human_input_mode="NEVER",
-            llm_config=False,
-            default_auto_reply=auto_reply,
-        )
-
-    def team(name: str, auto_reply: Optional[Dict[str, Any]] = None) -> autogen.ConversableAgent:
-        member1 = agent(f"member1_{name}", auto_reply=auto_reply)
-        member2 = agent(f"member2_{name}", auto_reply=auto_reply)
-
-        gc = autogen.GroupChat(agents=[member1, member2], messages=[])
-
-        return autogen.GroupChatManager(groupchat=gc, name=name, llm_config=False)
-
-    def chat(gc_manager: autogen.GroupChatManager):
-        team1_member1 = gc_manager.groupchat.agent_by_name("member1_team1", recursive=True)
-        team2_member2 = gc_manager.groupchat.agent_by_name("member2_team2", recursive=True)
+    # Test empty is_termination_msg function
+    groupchat = autogen.GroupChat(
+        agents=[agent1, agent2, agent3],
+        messages=[],
+        speaker_selection_method="round_robin",
+        max_round=10,
+        send_introductions=True,
+    )
 
-        assert team1_member1 is not None
-        assert team2_member2 is not None
+    intro = groupchat.introductions_msg()
+    assert "The first agent." in intro
+    assert "The second agent." in intro
+    assert "The third agent." in intro
+    assert "The fourth agent." not in intro
 
-        team1_member1.send(team1_msg, team2_member2, request_reply=True)
+    intro = groupchat.introductions_msg([agent1, agent2, agent4])
+    assert "The first agent." in intro
+    assert "The second agent." in intro
+    assert "The third agent." not in intro
+    assert "The fourth agent." in intro
 
-    user = agent("user")
-    team1 = team("team1", auto_reply=team1_msg)
-    team2 = team("team2", auto_reply=team2_msg)
+    groupchat = autogen.GroupChat(
+        agents=[agent1, agent2, agent3],
+        messages=[],
+        speaker_selection_method="round_robin",
+        max_round=10,
+        send_introductions=True,
+    )
 
-    gc = autogen.GroupChat(agents=[user, team1, team2], messages=[])
-    gc_manager = autogen.GroupChatManager(groupchat=gc, llm_config=False)
+    group_chat_manager = autogen.GroupChatManager(
+        groupchat=groupchat,
+        llm_config=False,
+        is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
+    )
 
-    chat(gc_manager)
+    group_chat_manager.initiate_chat(group_chat_manager, message="The initiating message.")
+    for a in [agent1, agent2, agent3]:
+        messages = agent1.chat_messages[group_chat_manager]
+        assert len(messages) == 3
+        assert "The first agent." in messages[0]["content"]
+        assert "The second agent." in messages[0]["content"]
+        assert "The third agent." in messages[0]["content"]
+        assert "The initiating message." == messages[1]["content"]
+        assert messages[2]["content"] == agent1._default_auto_reply
 
-    team1_member1 = gc.agent_by_name("member1_team1", recursive=True)
-    team2_member2 = gc.agent_by_name("member2_team2", recursive=True)
+    # Reset and start again
+    agent1.reset()
+    agent2.reset()
+    agent3.reset()
+    agent4.reset()
 
-    assert team1_member1 and team2_member2
+    # Check the default (no introductions)
+    groupchat2 = autogen.GroupChat(
+        agents=[agent1, agent2, agent3], messages=[], speaker_selection_method="round_robin", max_round=10
+    )
 
-    msg = team1_member1.chat_messages[team2_member2][0]
-    reply = team1_member1.chat_messages[team2_member2][1]
+    group_chat_manager2 = autogen.GroupChatManager(
+        groupchat=groupchat2,
+        llm_config=False,
+        is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
+    )
 
-    assert msg["content"] == team1_msg["content"]
-    assert reply["content"] == team2_msg["content"]
+    group_chat_manager2.initiate_chat(group_chat_manager2, message="The initiating message.")
+    for a in [agent1, agent2, agent3]:
+        messages = agent1.chat_messages[group_chat_manager2]
+        assert len(messages) == 2
+        assert "The initiating message." == messages[0]["content"]
+        assert messages[1]["content"] == agent1._default_auto_reply
 
 
 if __name__ == "__main__":
@@ -816,4 +788,5 @@ if __name__ == "__main__":
     # test_next_agent()
     # test_invalid_allow_repeat_speaker()
     # test_graceful_exit_before_max_round()
-    test_clear_agents_history()
+    # test_clear_agents_history()
+    test_send_intros()

test/test_browser_utils.py

@@ -11,6 +11,11 @@ from agentchat.test_assistant_agent import KEY_LOC  # noqa: E402
 BLOG_POST_URL = "https://microsoft.github.io/autogen/blog/2023/04/21/LLM-tuning-math"
 BLOG_POST_TITLE = "Does Model and Inference Parameter Matter in LLM Applications? - A Case Study for MATH | AutoGen"
 BLOG_POST_STRING = "Large language models (LLMs) are powerful tools that can generate natural language texts for various applications, such as chatbots, summarization, translation, and more. GPT-4 is currently the state of the art LLM in the world. Is model selection irrelevant? What about inference parameters?"
+BLOG_POST_FIND_ON_PAGE_QUERY = "The need for * cost saving is not specific to the math problems."
+BLOG_POST_FIND_ON_PAGE_MATCH = (
+    "The need for model selection, parameter tuning and cost saving is not specific to the math problems."
+)
+
 
 WIKIPEDIA_URL = "https://en.wikipedia.org/wiki/Microsoft"
 WIKIPEDIA_TITLE = "Microsoft - Wikipedia"
@@ -116,10 +121,10 @@ def test_simple_text_browser():
     # Visit a plain-text file
     response = requests.get(PLAIN_TEXT_URL)
     response.raise_for_status()
-    expected_results = response.text
+    expected_results = re.sub(r"\s+", " ", response.text, re.DOTALL).strip()  # Ignore spacing differences
 
     browser.visit_page(PLAIN_TEXT_URL)
-    assert browser.page_content.strip() == expected_results.strip()
+    assert re.sub(r"\s+", " ", browser.page_content, re.DOTALL).strip() == expected_results
 
     # Directly download an image, and compute its md5
     response = requests.get(IMAGE_URL, stream=True)
@@ -143,6 +148,52 @@ def test_simple_text_browser():
     viewport = browser.visit_page(PDF_URL)
     assert PDF_STRING in viewport
 
+    # Test find in page
+    browser.visit_page(BLOG_POST_URL)
+
+    find_viewport = browser.find_on_page(BLOG_POST_FIND_ON_PAGE_QUERY)
+    assert BLOG_POST_FIND_ON_PAGE_MATCH in find_viewport
+    assert find_viewport is not None
+
+    loc = browser.viewport_current_page
+    find_viewport = browser.find_on_page("LLM app*")
+    assert find_viewport is not None
+
+    # Find next using the same query
+    for i in range(0, 10):
+        find_viewport = browser.find_on_page("LLM app*")
+        assert find_viewport is not None
+
+        new_loc = browser.viewport_current_page
+        assert new_loc != loc
+        loc = new_loc
+
+    # Find next using find_next
+    for i in range(0, 10):
+        find_viewport = browser.find_next()
+        assert find_viewport is not None
+
+        new_loc = browser.viewport_current_page
+        assert new_loc != loc
+        loc = new_loc
+
+    # Bounce around
+    browser.viewport_current_page = 0
+    find_viewport = browser.find_on_page("For Further Reading")
+    assert find_viewport is not None
+    loc = browser.viewport_current_page
+
+    browser.page_up()
+    assert browser.viewport_current_page != loc
+    find_viewport = browser.find_on_page("For Further Reading")
+    assert find_viewport is not None
+    assert loc == browser.viewport_current_page
+
+    # Find something that doesn't exist
+    find_viewport = browser.find_on_page("7c748f9a-8dce-461f-a092-4e8d29913f2d")
+    assert find_viewport is None
+    assert loc == browser.viewport_current_page  # We didn't move
+
     # Clean up
     _rm_folder(downloads_folder)