Merge branch 'main' into headless_web_surfer

.devcontainer/Dockerfile

@@ -14,13 +14,13 @@ RUN apt-get update \
    && apt-get -y install --no-install-recommends build-essential npm \
    && apt-get autoremove -y \
    && apt-get clean -y \
-   && wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.4.549/quarto-1.4.549-linux-amd64.deb \
-   && dpkg -i quarto-1.4.549-linux-amd64.deb \
-   && rm -rf /var/lib/apt/lists/* quarto-1.4.549-linux-amd64.deb
+   && wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-amd64.deb \
+   && dpkg -i quarto-1.5.23-linux-amd64.deb \
+   && rm -rf /var/lib/apt/lists/* quarto-1.5.23-linux-amd64.deb
 ENV DEBIAN_FRONTEND=dialog
 
 # For docs
 RUN npm install --global yarn
 RUN pip install pydoc-markdown
 RUN pip install pyyaml
-RUN pip install colored
+RUN pip install colored

.devcontainer/dev/Dockerfile

@@ -33,12 +33,12 @@ RUN cd website
 RUN yarn install --frozen-lockfile --ignore-engines
 
 RUN arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \
-    wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.4.549/quarto-1.4.549-linux-${arch}.tar.gz && \
+    wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-${arch}.tar.gz && \
     mkdir -p /home/autogen/quarto/ && \
-    tar -xzf quarto-1.4.549-linux-${arch}.tar.gz --directory /home/autogen/quarto/ && \
-    rm quarto-1.4.549-linux-${arch}.tar.gz
+    tar -xzf quarto-1.5.23-linux-${arch}.tar.gz --directory /home/autogen/quarto/ && \
+    rm quarto-1.5.23-linux-${arch}.tar.gz
 
-ENV PATH="${PATH}:/home/autogen/quarto/quarto-1.4.549/bin/"
+ENV PATH="${PATH}:/home/autogen/quarto/quarto-1.5.23/bin/"
 
 # Exposes the Yarn port for Docusaurus
 EXPOSE 3000

.github/workflows/contrib-openai.yml

@@ -4,7 +4,7 @@
 name: OpenAI4ContribTests
 
 on:
-  pull_request_target:
+  pull_request:
     branches: ['main']
     paths:
       - 'autogen/**'
@@ -173,7 +173,7 @@ jobs:
           AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
           OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
         run: |
-          coverage run -a -m pytest test/agentchat/contrib/test_teachable_agent.py
+          coverage run -a -m pytest test/agentchat/contrib/capabilities/test_teachable_agent.py
           coverage xml
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3

.github/workflows/contrib-tests.yml

@@ -172,7 +172,7 @@ jobs:
       - name: Coverage
         run: |
           pip install coverage>=5.3
-          coverage run -a -m pytest test/agentchat/contrib/test_teachable_agent.py --skip-openai
+          coverage run -a -m pytest test/agentchat/contrib/capabilities/test_teachable_agent.py --skip-openai
           coverage xml
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3

.github/workflows/deploy-website.yml

@@ -37,19 +37,16 @@ jobs:
       - name: pydoc-markdown install
         run: |
           python -m pip install --upgrade pip
-          pip install pydoc-markdown pyyaml colored
+          pip install pydoc-markdown pyyaml termcolor
       - name: pydoc-markdown run
         run: |
           pydoc-markdown
       - name: quarto install
         working-directory: ${{ runner.temp }}
         run: |
-          wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.4.549/quarto-1.4.549-linux-amd64.tar.gz
-          tar -xzf quarto-1.4.549-linux-amd64.tar.gz
-          echo "$(pwd)/quarto-1.4.549/bin/" >> $GITHUB_PATH
-      - name: quarto run
-        run: |
-          quarto render .
+          wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-amd64.tar.gz
+          tar -xzf quarto-1.5.23-linux-amd64.tar.gz
+          echo "$(pwd)/quarto-1.5.23/bin/" >> $GITHUB_PATH
       - name: Process notebooks
         run: |
           python process_notebooks.py render
@@ -83,19 +80,16 @@ jobs:
       - name: pydoc-markdown install
         run: |
           python -m pip install --upgrade pip
-          pip install pydoc-markdown pyyaml colored
+          pip install pydoc-markdown pyyaml termcolor
       - name: pydoc-markdown run
         run: |
           pydoc-markdown
       - name: quarto install
         working-directory: ${{ runner.temp }}
         run: |
-          wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.4.549/quarto-1.4.549-linux-amd64.tar.gz
-          tar -xzf quarto-1.4.549-linux-amd64.tar.gz
-          echo "$(pwd)/quarto-1.4.549/bin/" >> $GITHUB_PATH
-      - name: quarto run
-        run: |
-          quarto render .
+          wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-amd64.tar.gz
+          tar -xzf quarto-1.5.23-linux-amd64.tar.gz
+          echo "$(pwd)/quarto-1.5.23/bin/" >> $GITHUB_PATH
       - name: Process notebooks
         run: |
           python process_notebooks.py render

.github/workflows/openai.yml

@@ -4,7 +4,7 @@
 name: OpenAI
 
 on:
-  pull_request_target:
+  pull_request:
     branches: ["main"]
     paths:
       - "autogen/**"

README.md

@@ -2,7 +2,7 @@
 [![Build](https://github.com/microsoft/autogen/actions/workflows/python-package.yml/badge.svg)](https://github.com/microsoft/autogen/actions/workflows/python-package.yml)
 ![Python Version](https://img.shields.io/badge/3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue)
 [![Downloads](https://static.pepy.tech/badge/pyautogen/week)](https://pepy.tech/project/pyautogen)
-[![](https://img.shields.io/discord/1153072414184452236?logo=discord&style=flat)](https://discord.gg/pAbnFJrkgZ)
+[![Discord](https://img.shields.io/discord/1153072414184452236?logo=discord&style=flat)](https://discord.gg/pAbnFJrkgZ)
 [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/cloudposse.svg?style=social&label=Follow%20%40pyautogen)](https://twitter.com/pyautogen)
 
 
@@ -12,6 +12,7 @@
     <img src="https://github.com/microsoft/autogen/blob/main/website/static/img/flaml.svg"  width=200>
     <br>
 </p> -->
+:fire: Mar 1: the first AutoGen multi-agent experiment on the challenging [GAIA](https://huggingface.co/spaces/gaia-benchmark/leaderboard) benchmark achieved the No. 1 accuracy in all the three levels.
 
 :fire: Jan 30: AutoGen is highlighted by Peter Lee in Microsoft Research Forum [Keynote](https://t.co/nUBSjPDjqD).
 

autogen/agentchat/chat.py

@@ -3,15 +3,9 @@ import logging
 from collections import defaultdict
 from typing import Dict, List, Any, Set, Tuple
 from dataclasses import dataclass
-from .utils import consolidate_chat_info
 import warnings
-
-try:
-    from termcolor import colored
-except ImportError:
-
-    def colored(x, *args, **kwargs):
-        return x
+from termcolor import colored
+from .utils import consolidate_chat_info
 
 
 logger = logging.getLogger(__name__)

autogen/agentchat/contrib/capabilities/context_handling.py

@@ -25,10 +25,9 @@ class TransformChatHistory:
     2. Second, it limits the number of message to keep
     3. Third, it limits the total number of tokens in the chat history
 
-    Args:
-        max_tokens_per_message (Optional[int]): Maximum number of tokens to keep in each message.
-        max_messages (Optional[int]): Maximum number of messages to keep in the context.
-        max_tokens (Optional[int]): Maximum number of tokens to keep in the context.
+    When adding this capability to an agent, the following are modified:
+    - A hook is added to the hookable method `process_all_messages_before_reply` to transform the received messages for possible truncation.
+    Not modifying the stored message history.
     """
 
     def __init__(
@@ -38,6 +37,12 @@ class TransformChatHistory:
         max_messages: Optional[int] = None,
         max_tokens: Optional[int] = None,
     ):
+        """
+        Args:
+            max_tokens_per_message (Optional[int]): Maximum number of tokens to keep in each message.
+            max_messages (Optional[int]): Maximum number of messages to keep in the context.
+            max_tokens (Optional[int]): Maximum number of tokens to keep in the context.
+        """
         self.max_tokens_per_message = max_tokens_per_message if max_tokens_per_message else sys.maxsize
         self.max_messages = max_messages if max_messages else sys.maxsize
         self.max_tokens = max_tokens if max_tokens else sys.maxsize

autogen/agentchat/contrib/capabilities/teachability.py

@@ -1,18 +1,12 @@
 import os
-from autogen.agentchat.assistant_agent import ConversableAgent
-from autogen.agentchat.contrib.capabilities.agent_capability import AgentCapability
-from autogen.agentchat.contrib.text_analyzer_agent import TextAnalyzerAgent
-from typing import Dict, Optional, Union, List, Tuple, Any
+from typing import Dict, Optional, Union
 import chromadb
 from chromadb.config import Settings
 import pickle
-
-try:
-    from termcolor import colored
-except ImportError:
-
-    def colored(x, *args, **kwargs):
-        return x
+from autogen.agentchat.assistant_agent import ConversableAgent
+from autogen.agentchat.contrib.capabilities.agent_capability import AgentCapability
+from autogen.agentchat.contrib.text_analyzer_agent import TextAnalyzerAgent
+from autogen.agentchat.conversable_agent import colored
 
 
 class Teachability(AgentCapability):
@@ -23,6 +17,13 @@ class Teachability(AgentCapability):
     To make any conversable agent teachable, instantiate both the agent and the Teachability class,
     then pass the agent to teachability.add_to_agent(agent).
     Note that teachable agents in a group chat must be given unique path_to_db_dir values.
+
+    When adding Teachability to an agent, the following are modified:
+    - The agent's system message is appended with a note about the agent's new ability.
+    - A hook is added to the agent's `process_last_received_message` hookable method,
+    and the hook potentially modifies the last of the received messages to include earlier teachings related to the message.
+    Added teachings do not propagate into the stored message history.
+    If new user teachings are detected, they are added to new memos in the vector database.
     """
 
     def __init__(

autogen/agentchat/contrib/llava_agent.py

@@ -1,25 +1,14 @@
 import json
 import logging
-import os
-import pdb
-import re
-from typing import Any, Dict, List, Optional, Tuple, Union
-
+from typing import List, Optional, Tuple
 import replicate
 import requests
-from regex import R
 
 from autogen.agentchat.agent import Agent
 from autogen.agentchat.contrib.img_utils import get_image_data, llava_formatter
 from autogen.agentchat.contrib.multimodal_conversable_agent import MultimodalConversableAgent
 from autogen.code_utils import content_str
-
-try:
-    from termcolor import colored
-except ImportError:
-
-    def colored(x, *args, **kwargs):
-        return x
+from autogen.agentchat.conversable_agent import colored
 
 
 logger = logging.getLogger(__name__)

autogen/agentchat/contrib/multimodal_conversable_agent.py

@@ -1,26 +1,16 @@
 import copy
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 from autogen import OpenAIWrapper
 from autogen.agentchat import Agent, ConversableAgent
 from autogen.agentchat.contrib.img_utils import (
-    convert_base64_to_data_uri,
     gpt4v_formatter,
     message_formatter_pil_to_b64,
-    pil_to_data_uri,
 )
+from autogen.code_utils import content_str
 
 from ..._pydantic import model_dump
 
-try:
-    from termcolor import colored
-except ImportError:
-
-    def colored(x, *args, **kwargs):
-        return x
-
-
-from autogen.code_utils import content_str
 
 DEFAULT_LMM_SYS_MSG = """You are a helpful AI assistant."""
 DEFAULT_MODEL = "gpt-4-vision-preview"

autogen/agentchat/contrib/retrieve_user_proxy_agent.py

@@ -1,4 +1,6 @@
 import re
+from typing import Callable, Dict, Optional, Union, List, Tuple, Any
+from IPython import get_ipython
 
 try:
     import chromadb
@@ -10,16 +12,7 @@ from autogen.retrieve_utils import create_vector_db_from_dir, query_vector_db, T
 from autogen.token_count_utils import count_token
 from autogen.code_utils import extract_code
 from autogen import logger
-
-from typing import Callable, Dict, Optional, Union, List, Tuple, Any
-from IPython import get_ipython
-
-try:
-    from termcolor import colored
-except ImportError:
-
-    def colored(x, *args, **kwargs):
-        return x
+from autogen.agentchat.conversable_agent import colored
 
 
 PROMPT_DEFAULT = """You're a retrieve augmented chatbot. You answer user's questions based on your own knowledge and the

autogen/agentchat/conversable_agent.py

@@ -197,6 +197,21 @@ class ConversableAgent(LLMAgent):
             self._code_execution_config = code_execution_config
 
             if self._code_execution_config.get("executor") is not None:
+                if "use_docker" in self._code_execution_config:
+                    raise ValueError(
+                        "'use_docker' in code_execution_config is not valid when 'executor' is set. Use the appropriate arg in the chosen executor instead."
+                    )
+
+                if "work_dir" in self._code_execution_config:
+                    raise ValueError(
+                        "'work_dir' in code_execution_config is not valid when 'executor' is set. Use the appropriate arg in the chosen executor instead."
+                    )
+
+                if "timeout" in self._code_execution_config:
+                    raise ValueError(
+                        "'timeout' in code_execution_config is not valid when 'executor' is set. Use the appropriate arg in the chosen executor instead."
+                    )
+
                 # Use the new code executor.
                 self._code_executor = CodeExecutorFactory.create(self._code_execution_config)
                 self.register_reply([Agent, None], ConversableAgent._generate_code_execution_reply_using_executor)
@@ -362,7 +377,7 @@ class ConversableAgent(LLMAgent):
     def register_nested_chats(
         self,
         chat_queue: List[Dict[str, Any]],
-        trigger: Union[Type[Agent], str, Agent, Callable[[Agent], bool], List] = [Agent, None],
+        trigger: Union[Type[Agent], str, Agent, Callable[[Agent], bool], List],
         reply_func_from_nested_chats: Union[str, Callable] = "summary_from_nested_chats",
         position: int = 2,
         **kwargs,
@@ -370,7 +385,7 @@ class ConversableAgent(LLMAgent):
         """Register a nested chat reply function.
         Args:
             chat_queue (list): a list of chat objects to be initiated.
-            trigger (Agent class, str, Agent instance, callable, or list): Default to [Agent, None]. Ref to `register_reply` for details.
+            trigger (Agent class, str, Agent instance, callable, or list): refer to `register_reply` for details.
             reply_func_from_nested_chats (Callable, str): the reply function for the nested chat.
                 The function takes a chat_queue for nested chat, recipient agent, a list of messages, a sender agent and a config as input and returns a reply message.
                 Default to "summary_from_nested_chats", which corresponds to a built-in reply function that get summary from the nested chat_queue.
@@ -1126,8 +1141,18 @@ class ConversableAgent(LLMAgent):
         if recipient is None:
             if nr_messages_to_preserve:
                 for key in self._oai_messages:
+                    nr_messages_to_preserve_internal = nr_messages_to_preserve
+                    # if breaking history between function call and function response, save function call message
+                    # additionally, otherwise openai will return error
+                    first_msg_to_save = self._oai_messages[key][-nr_messages_to_preserve_internal]
+                    if "tool_responses" in first_msg_to_save:
+                        nr_messages_to_preserve_internal += 1
+                        print(
+                            f"Preserving one more message for {self.name} to not divide history between tool call and "
+                            f"tool response."
+                        )
                     # Remove messages from history except last `nr_messages_to_preserve` messages.
-                    self._oai_messages[key] = self._oai_messages[key][-nr_messages_to_preserve:]
+                    self._oai_messages[key] = self._oai_messages[key][-nr_messages_to_preserve_internal:]
             else:
                 self._oai_messages.clear()
         else:
@@ -1722,14 +1747,14 @@ class ConversableAgent(LLMAgent):
         if messages is None:
             messages = self._oai_messages[sender]
 
-        # Call the hookable method that gives registered hooks a chance to process all messages.
-        # Message modifications do not affect the incoming messages or self._oai_messages.
-        messages = self.process_all_messages_before_reply(messages)
-
         # Call the hookable method that gives registered hooks a chance to process the last message.
         # Message modifications do not affect the incoming messages or self._oai_messages.
         messages = self.process_last_received_message(messages)
 
+        # Call the hookable method that gives registered hooks a chance to process all messages.
+        # Message modifications do not affect the incoming messages or self._oai_messages.
+        messages = self.process_all_messages_before_reply(messages)
+
         for reply_func_tuple in self._reply_func_list:
             reply_func = reply_func_tuple["reply_func"]
             if "exclude" in kwargs and reply_func in kwargs["exclude"]:

autogen/agentchat/groupchat.py

@@ -596,9 +596,11 @@ class GroupChatManager(ConversableAgent):
             if (
                 groupchat.enable_clear_history
                 and isinstance(reply, dict)
+                and reply["content"]
                 and "CLEAR HISTORY" in reply["content"].upper()
             ):
-                reply["content"] = self.clear_agents_history(reply["content"], groupchat)
+                reply["content"] = self.clear_agents_history(reply, groupchat)
+
             # The speaker sends the message without requesting a reply
             speaker.send(reply, self, request_reply=False)
             message = self.last_message(speaker)
@@ -684,7 +686,7 @@ class GroupChatManager(ConversableAgent):
         for agent in self._groupchat.agents:
             agent._raise_exception_on_async_reply_functions()
 
-    def clear_agents_history(self, reply: str, groupchat: GroupChat) -> str:
+    def clear_agents_history(self, reply: dict, groupchat: GroupChat) -> str:
         """Clears history of messages for all agents or selected one. Can preserve selected number of last messages.
         That function is called when user manually provide "clear history" phrase in his reply.
         When "clear history" is provided, the history of messages for all agents is cleared.
@@ -696,23 +698,27 @@ class GroupChatManager(ConversableAgent):
         Phrase "clear history" and optional arguments are cut out from the reply before it passed to the chat.
 
         Args:
-            reply (str): Admin reply to analyse.
+            reply (dict): reply message dict to analyze.
             groupchat (GroupChat): GroupChat object.
         """
+        reply_content = reply["content"]
         # Split the reply into words
-        words = reply.split()
+        words = reply_content.split()
         # Find the position of "clear" to determine where to start processing
         clear_word_index = next(i for i in reversed(range(len(words))) if words[i].upper() == "CLEAR")
         # Extract potential agent name and steps
         words_to_check = words[clear_word_index + 2 : clear_word_index + 4]
         nr_messages_to_preserve = None
+        nr_messages_to_preserve_provided = False
         agent_to_memory_clear = None
 
         for word in words_to_check:
             if word.isdigit():
                 nr_messages_to_preserve = int(word)
+                nr_messages_to_preserve_provided = True
             elif word[:-1].isdigit():  # for the case when number of messages is followed by dot or other sign
                 nr_messages_to_preserve = int(word[:-1])
+                nr_messages_to_preserve_provided = True
             else:
                 for agent in groupchat.agents:
                     if agent.name == word:
@@ -721,6 +727,12 @@ class GroupChatManager(ConversableAgent):
                     elif agent.name == word[:-1]:  # for the case when agent name is followed by dot or other sign
                         agent_to_memory_clear = agent
                         break
+        # preserve last tool call message if clear history called inside of tool response
+        if "tool_responses" in reply and not nr_messages_to_preserve:
+            nr_messages_to_preserve = 1
+            logger.warning(
+                "The last tool call message will be saved to prevent errors caused by tool response without tool call."
+            )
         # clear history
         if agent_to_memory_clear:
             if nr_messages_to_preserve:
@@ -746,7 +758,7 @@ class GroupChatManager(ConversableAgent):
                 agent.clear_history(nr_messages_to_preserve=nr_messages_to_preserve)
 
         # Reconstruct the reply without the "clear history" command and parameters
-        skip_words_number = 2 + int(bool(agent_to_memory_clear)) + int(bool(nr_messages_to_preserve))
-        reply = " ".join(words[:clear_word_index] + words[clear_word_index + skip_words_number :])
+        skip_words_number = 2 + int(bool(agent_to_memory_clear)) + int(nr_messages_to_preserve_provided)
+        reply_content = " ".join(words[:clear_word_index] + words[clear_word_index + skip_words_number :])
 
-        return reply
+        return reply_content

autogen/coding/jupyter/base.py

@@ -7,9 +7,13 @@ class JupyterConnectionInfo:
     """(Experimental)"""
 
     host: str
+    """`str` - Host of the Jupyter gateway server"""
     use_https: bool
+    """`bool` - Whether to use HTTPS"""
     port: int
+    """`int` - Port of the Jupyter gateway server"""
     token: Optional[str]
+    """`Optional[str]` - Token for authentication. If None, no token is used"""
 
 
 @runtime_checkable
@@ -18,4 +22,5 @@ class JupyterConnectable(Protocol):
 
     @property
     def connection_info(self) -> JupyterConnectionInfo:
+        """Return the connection information for this connectable."""
         pass

autogen/coding/jupyter/jupyter_client.py

@@ -23,9 +23,12 @@ from .base import JupyterConnectionInfo
 
 
 class JupyterClient:
-    """(Experimental) A client for communicating with a Jupyter gateway server."""
-
     def __init__(self, connection_info: JupyterConnectionInfo):
+        """(Experimental) A client for communicating with a Jupyter gateway server.
+
+        Args:
+            connection_info (JupyterConnectionInfo): Connection information
+        """
         self._connection_info = connection_info
         self._session = requests.Session()
         retries = Retry(total=5, backoff_factor=0.1)

autogen/coding/jupyter/jupyter_code_executor.py

@@ -22,27 +22,6 @@ from .jupyter_client import JupyterClient
 
 
 class JupyterCodeExecutor(CodeExecutor):
-    """(Experimental) A code executor class that executes code statefully using an embedded
-    IPython kernel managed by this class.
-
-    **This will execute LLM generated code on the local machine.**
-
-    Each execution is stateful and can access variables created from previous
-    executions in the same session. The kernel must be installed before using
-    this class. The kernel can be installed using the following command:
-    `python -m ipykernel install --user --name {kernel_name}`
-    where `kernel_name` is the name of the kernel to install.
-
-    Args:
-        timeout (int): The timeout for code execution, by default 60.
-        kernel_name (str): The kernel name to use. Make sure it is installed.
-            By default, it is "python3".
-        output_dir (str): The directory to save output files, by default ".".
-        system_message_update (str): The system message update to add to the
-            agent that produces code. By default it is
-            `JupyterCodeExecutor.DEFAULT_SYSTEM_MESSAGE_UPDATE`.
-    """
-
     DEFAULT_SYSTEM_MESSAGE_UPDATE: ClassVar[
         str
     ] = """
@@ -72,24 +51,10 @@ the output will be a path to the image instead of the image itself.
 """
 
     class UserCapability:
-        """(Experimental) An AgentCapability class that gives agent ability use a stateful
-        IPython code executor. This capability can be added to an agent using
-        the `add_to_agent` method which append a system message update to the
-        agent's system message."""
-
         def __init__(self, system_message_update: str):
             self._system_message_update = system_message_update
 
         def add_to_agent(self, agent: LLMAgent) -> None:
-            """Add this capability to an agent by appending a system message
-            update to the agent's system message.
-
-            **Currently we do not check for conflicts with existing content in
-            the agent's system message.**
-
-            Args:
-                agent (LLMAgent): The agent to add the capability to.
-            """
             agent.update_system_message(agent.system_message + self._system_message_update)
 
     def __init__(
@@ -100,6 +65,19 @@ the output will be a path to the image instead of the image itself.
         output_dir: Union[Path, str] = Path("."),
         system_message_update: str = DEFAULT_SYSTEM_MESSAGE_UPDATE,
     ):
+        """(Experimental) A code executor class that executes code statefully using
+        a Jupyter server supplied to this class.
+
+        Each execution is stateful and can access variables created from previous
+        executions in the same session.
+
+        Args:
+            jupyter_server (Union[JupyterConnectable, JupyterConnectionInfo]): The Jupyter server to use.
+            timeout (int): The timeout for code execution, by default 60.
+            kernel_name (str): The kernel name to use. Make sure it is installed.
+                By default, it is "python3".
+            output_dir (str): The directory to save output files, by default ".".
+        """
         if timeout < 1:
             raise ValueError("Timeout must be greater than or equal to 1.")
 
@@ -130,8 +108,6 @@ the output will be a path to the image instead of the image itself.
 
     @property
     def user_capability(self) -> "JupyterCodeExecutor.UserCapability":
-        """(Experimental) Export a user capability for this executor that can be added to
-        an agent using the `add_to_agent` method."""
         return JupyterCodeExecutor.UserCapability(self._system_message_update)
 
     @property
@@ -142,8 +118,7 @@ the output will be a path to the image instead of the image itself.
     def execute_code_blocks(self, code_blocks: List[CodeBlock]) -> IPythonCodeResult:
         """(Experimental) Execute a list of code blocks and return the result.
 
-        This method executes a list of code blocks as cells in an IPython kernel
-        managed by this class.
+        This method executes a list of code blocks as cells in the Jupyter kernel.
         See: https://jupyter-client.readthedocs.io/en/stable/messaging.html
         for the message protocol.
 

autogen/coding/jupyter/local_jupyter_server.py

@@ -39,6 +39,17 @@ class LocalJupyterServer(JupyterConnectable):
         log_max_bytes: int = 1048576,
         log_backup_count: int = 3,
     ):
+        """Runs a Jupyter Kernel Gateway server locally.
+
+        Args:
+            ip (str, optional): IP address to bind to. Defaults to "127.0.0.1".
+            port (Optional[int], optional): Port to use, if None it automatically selects a port. Defaults to None.
+            token (Union[str, GenerateToken], optional): Token to use for Jupyter server. By default will generate a token. Using None will use no token for authentication. Defaults to GenerateToken().
+            log_file (str, optional): File for Jupyter Kernel Gateway logs. Defaults to "jupyter_gateway.log".
+            log_level (str, optional): Level for Jupyter Kernel Gateway logs. Defaults to "INFO".
+            log_max_bytes (int, optional): Max logfile size. Defaults to 1048576.
+            log_backup_count (int, optional): Number of backups for rotating log. Defaults to 3.
+        """
         # Remove as soon as https://github.com/jupyter-server/kernel_gateway/issues/398 is fixed
         if sys.platform == "win32":
             raise ValueError("LocalJupyterServer is not supported on Windows due to kernelgateway bug.")

autogen/coding/local_commandline_code_executor.py

@@ -3,21 +3,14 @@ import re
 import uuid
 import warnings
 from typing import Any, ClassVar, List, Optional
-
 from pydantic import BaseModel, Field, field_validator
 
+from autogen.agentchat.conversable_agent import colored
 from ..agentchat.agent import LLMAgent
 from ..code_utils import execute_code
 from .base import CodeBlock, CodeExtractor, CodeResult
 from .markdown_code_extractor import MarkdownCodeExtractor
 
-try:
-    from termcolor import colored
-except ImportError:
-
-    def colored(x: Any, *args: Any, **kwargs: Any) -> str:  # type: ignore[misc]
-        return x  # type: ignore[no-any-return]
-
 
 __all__ = (
     "LocalCommandlineCodeExecutor",

autogen/version.py

@@ -1 +1 @@
-__version__ = "0.2.16"
+__version__ = "0.2.17"

notebook/agentchat_RetrieveChat.ipynb

@@ -5,12 +5,6 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "<!--\n",
-    "tags: [\"RAG\"]\n",
-    "description: |\n",
-    "    Explore the use of AutoGen's RetrieveChat for tasks like code generation from docstrings, answering complex questions with human feedback, and exploiting features like Update Context, custom prompts, and few-shot learning.\n",
-    "-->\n",
-    "\n",
     "# Using RetrieveChat for Retrieve Augmented Code Generation and Question Answering\n",
     "\n",
     "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n",
@@ -92,7 +86,7 @@
    "source": [
     "````{=mdx}\n",
     ":::tip\n",
-    "Learn more about configuring LLMs for agents [here](/docs/llm_configuration).\n",
+    "Learn more about configuring LLMs for agents [here](/docs/topics/llm_configuration).\n",
     ":::\n",
     "````\n",
     "\n",
@@ -3020,6 +3014,12 @@
   }
  ],
  "metadata": {
+  "front_matter": {
+   "description": "Explore the use of AutoGen's RetrieveChat for tasks like code generation from docstrings, answering complex questions with human feedback, and exploiting features like Update Context, custom prompts, and few-shot learning.",
+   "tags": [
+    "RAG"
+   ]
+  },
   "kernelspec": {
    "display_name": "Python 3 (ipykernel)",
    "language": "python",
@@ -3036,8 +3036,8 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.10.12"
-},
-"test_skip": "Requires interactive usage"
+  },
+  "skip_test": "Requires interactive usage"
  },
  "nbformat": 4,
  "nbformat_minor": 4

notebook/agentchat_auto_feedback_from_code_execution.ipynb

@@ -8,12 +8,6 @@
     }
    },
    "source": [
-    "<!--\n",
-    "tags: [\"code generation\", \"debugging\"]\n",
-    "description: |\n",
-    "    Use conversable language learning model agents to solve tasks and provide automatic feedback through a comprehensive example of writing, executing, and debugging Python code to compare stock price changes.\n",
-    "-->\n",
-    "\n",
     "# Task Solving with Code Generation, Execution and Debugging\n",
     "\n",
     "AutoGen offers conversable LLM agents, which can be used to solve various tasks with human or automatic feedback, including tasks that require using tools via code.\n",
@@ -61,7 +55,7 @@
    "source": [
     "````{=mdx}\n",
     ":::tip\n",
-    "Learn more about configuring LLMs for agents [here](/docs/llm_configuration).\n",
+    "Learn more about configuring LLMs for agents [here](/docs/topics/llm_configuration).\n",
     ":::\n",
     "````"
    ]
@@ -1098,6 +1092,13 @@
   }
  ],
  "metadata": {
+  "front_matter": {
+    "tags": [
+      "code generation",
+      "debugging"
+    ],
+    "description": "Use conversable language learning model agents to solve tasks and provide automatic feedback through a comprehensive example of writing, executing, and debugging Python code to compare stock price changes."
+  },
   "kernelspec": {
    "display_name": "Python 3",
    "language": "python",

notebook/agentchat_cost_token_tracking.ipynb

@@ -102,7 +102,7 @@
     "]\n",
     "```\n",
     "\n",
-    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/llm_configuration.ipynb) for full code examples of the different methods."
+    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods."
    ]
   },
   {

notebook/agentchat_function_call.ipynb

@@ -103,7 +103,7 @@
     "]\n",
     "```\n",
     "\n",
-    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/llm_configuration.ipynb) for full code examples of the different methods."
+    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods."
    ]
   },
   {

notebook/agentchat_function_call_async.ipynb

@@ -5,12 +5,6 @@
    "id": "ae1f50ec",
    "metadata": {},
    "source": [
-    "<!--\n",
-    "tags: [\"code generation\", \"function call\", \"async\"]\n",
-    "description: |\n",
-    "    Learn how to implement both synchronous and asynchronous function calls using AssistantAgent and UserProxyAgent in AutoGen, with examples of their application in individual and group chat settings for task execution with language models.\n",
-    "-->\n",
-    "\n",
     "# Task Solving with Provided Tools as Functions (Asynchronous Function Calls)\n"
    ]
   },
@@ -61,7 +55,7 @@
    "source": [
     "````{=mdx}\n",
     ":::tip\n",
-    "Learn more about configuring LLMs for agents [here](/docs/llm_configuration).\n",
+    "Learn more about configuring LLMs for agents [here](/docs/topics/llm_configuration).\n",
     ":::\n",
     "````\n",
     "\n",
@@ -366,6 +360,10 @@
   }
  ],
  "metadata": {
+  "front_matter": {
+    "tags": ["code generation", "function call", "async"],
+    "description": "Learn how to implement both synchronous and asynchronous function calls using AssistantAgent and UserProxyAgent in AutoGen, with examples of their application in individual and group chat settings for task execution with language models."
+  },
   "kernelspec": {
    "display_name": "flaml_dev",
    "language": "python",

notebook/agentchat_function_call_currency_calculator.ipynb

@@ -106,7 +106,7 @@
     "]\n",
     "```\n",
     "\n",
-    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/llm_configuration.ipynb) for full code examples of the different methods."
+    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods."
    ]
   },
   {

notebook/agentchat_groupchat.ipynb

@@ -5,12 +5,6 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "<!--\n",
-    "tags: [\"orchestration\", \"group chat\"]\n",
-    "description: |\n",
-    "    Explore the utilization of large language models in automated group chat scenarios, where agents perform tasks collectively, demonstrating how they can be configured, interact with each other, and retrieve specific information from external resources.\n",
-    "-->\n",
-    "\n",
     "# Group Chat\n",
     "\n",
     "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n",
@@ -63,7 +57,7 @@
    "source": [
     "````{=mdx}\n",
     ":::tip\n",
-    "Learn more about configuring LLMs for agents [here](/docs/llm_configuration).\n",
+    "Learn more about configuring LLMs for agents [here](/docs/topics/llm_configuration).\n",
     ":::\n",
     "````\n",
     "\n",
@@ -223,6 +217,10 @@
   }
  ],
  "metadata": {
+  "front_matter": {
+    "tags": ["orchestration", "group chat"],
+    "description": "Explore the utilization of large language models in automated group chat scenarios, where agents perform tasks collectively, demonstrating how they can be configured, interact with each other, and retrieve specific information from external resources."
+  },
   "kernelspec": {
    "display_name": "flaml",
    "language": "python",

notebook/agentchat_groupchat_RAG.ipynb

@@ -1,17 +1,5 @@
 {
  "cells": [
-  {
-   "attachments": {},
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "<!--\n",
-    "tags: [\"group chat\", \"orchestration\", \"RAG\"]\n",
-    "description: |\n",
-    "    Implement and manage a multi-agent chat system using AutoGen, where AI assistants retrieve information, generate code, and interact collaboratively to solve complex tasks, especially in areas not covered by their training data.\n",
-    "-->"
-   ]
-  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -78,7 +66,7 @@
    "source": [
     "````{=mdx}\n",
     ":::tip\n",
-    "Learn more about configuring LLMs for agents [here](/docs/llm_configuration).\n",
+    "Learn more about configuring LLMs for agents [here](/docs/topics/llm_configuration).\n",
     ":::\n",
     "````\n",
     "\n",
@@ -1120,6 +1108,10 @@
   }
  ],
  "metadata": {
+ "front_matter": {
+    "tags": ["group chat", "orchestration", "RAG"],
+    "description": "Implement and manage a multi-agent chat system using AutoGen, where AI assistants retrieve information, generate code, and interact collaboratively to solve complex tasks, especially in areas not covered by their training data."
+  },
   "kernelspec": {
    "display_name": "flaml",
    "language": "python",
@@ -1135,7 +1127,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.11.7"
   }
  },
  "nbformat": 4,

notebook/agentchat_groupchat_finite_state_machine.ipynb

@@ -5,13 +5,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "<!--\n",
-    "tags: [\"orchestration\"]\n",
-    "description: |\n",
-    "    Explore the demonstration of the Finite State Machine implementation, which allows the user to input speaker transition contraints.\n",
-    "-->\n",
-    "\n",
-    "# FSM - User can input speaker transition contraints.\n",
+    "# FSM - User can input speaker transition constraints\n",
     "\n",
     "AutoGen offers conversable agents powered by LLM, tool, or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n",
     "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n",
@@ -597,6 +591,12 @@
   }
  ],
  "metadata": {
+  "front_matter": {
+   "description": "Explore the demonstration of the Finite State Machine implementation, which allows the user to input speaker transition constraints.",
+   "tags": [
+    "orchestration"
+   ]
+  },
   "kernelspec": {
    "display_name": "Python 3 (ipykernel)",
    "language": "python",

notebook/agentchat_groupchat_research.ipynb

@@ -93,7 +93,7 @@
     "]\n",
     "```\n",
     "\n",
-    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/llm_configuration.ipynb) for full code examples of the different methods."
+    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods."
    ]
   },
   {

notebook/agentchat_groupchat_vis.ipynb

@@ -110,7 +110,7 @@
     "]\n",
     "```\n",
     "\n",
-    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/llm_configuration.ipynb) for full code examples of the different methods."
+    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](/docs/topics/llm_configuration) for full code examples of the different methods."
    ]
   },
   {

notebook/agentchat_human_feedback.ipynb

@@ -102,7 +102,7 @@
     "]\n",
     "```\n",
     "\n",
-    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/llm_configuration.ipynb) for full code examples of the different methods."
+    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods."
    ]
   },
   {

notebook/agentchat_langchain.ipynb

@@ -139,7 +139,7 @@
     "]\n",
     "```\n",
     "\n",
-    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/llm_configuration.ipynb) for full code examples of the different methods."
+    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods."
    ]
   },
   {

notebook/agentchat_logging.ipynb

@@ -4,12 +4,6 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "<!--\n",
-    "tags: [\"logging\", \"debugging\"]\n",
-    "description: |\n",
-    "    Provide capabilities of runtime logging for debugging and performance analysis.\n",
-    "-->\n",
-    "\n",
     "# Runtime Logging with AutoGen \n",
     "\n",
     "AutoGen offers utilities to log data for debugging and performance analysis. This notebook demonstrates how to use them. \n",
@@ -294,6 +288,13 @@
   }
  ],
  "metadata": {
+  "front_matter": {
+   "description": "Provide capabilities of runtime logging for debugging and performance analysis.",
+   "tags": [
+    "logging",
+    "debugging"
+   ]
+  },
   "kernelspec": {
    "display_name": "autog",
    "language": "python",
@@ -309,7 +310,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.11.7"
   }
  },
  "nbformat": 4,

notebook/agentchat_multi_task_async_chats.ipynb

@@ -9,12 +9,6 @@
     }
    },
    "source": [
-    "<!--\n",
-    "tags: [\"sequential chat\"]\n",
-    "description: |\n",
-    "    Use conversational agents to solve a set of tasks with a sequence of async chats.\n",
-    "-->\n",
-    "\n",
     "# Solving Multiple Tasks in a Sequence of Async Chats\n",
     "\n",
     "This notebook showcases how to use the new chat interface of conversational agents in AutoGen: a_initiate_chats, to conduct a series of tasks. Similar to \"notebook/agentchat_microsoft_fabric.ipynb\", this new interface allows one to pass multiple tasks and their corresponding dedicated agents and execute concurrently. Depending on the prerequisite task(s), the tasks will be solved concurrently, with the summaries from prerequisite task(s) provided to subsequent tasks as context, if the `summary_method` argument is specified.\n",
@@ -49,7 +43,7 @@
    "source": [
     "\\:\\:\\:tip\n",
     "\n",
-    "Learn more about the various ways to configure LLM endpoints [here](/docs/llm_configuration).\n",
+    "Learn more about the various ways to configure LLM endpoints [here](/docs/topics/llm_configuration).\n",
     "\n",
     "\\:\\:\\:"
    ]
@@ -1484,6 +1478,10 @@
   }
  ],
  "metadata": {
+  "front_matter": {
+    "tags": ["sequential chat"],
+    "description": "Use conversational agents to solve a set of tasks with a sequence of async chats."
+  },
   "kernelspec": {
    "display_name": "Python 3",
    "language": "python",
@@ -1499,7 +1497,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.8"
+   "version": "3.11.7"
   },
   "vscode": {
    "interpreter": {

notebook/agentchat_multi_task_chats.ipynb

@@ -9,12 +9,6 @@
     }
    },
    "source": [
-    "<!--\n",
-    "tags: [\"sequential chat\"]\n",
-    "description: |\n",
-    "    Use conversational agents to solve a set of tasks with a sequence of chats.\n",
-    "-->\n",
-    "\n",
     "# Solving Multiple Tasks in a Sequence of Chats\n",
     "\n",
     "This notebook showcases how to use the new chat interface of conversational agents in AutoGen: initiate_chats, to conduct a series of tasks. This new interface allows one to pass multiple tasks and their corresponding dedicated agents. Once initiate_chats is invoked, the tasks will be solved sequentially, with the summaries from previous tasks provided to subsequent tasks as context, if the `summary_method` argument is specified.\n",
@@ -49,7 +43,7 @@
    "source": [
     "\\:\\:\\:tip\n",
     "\n",
-    "Learn more about the various ways to configure LLM endpoints [here](/docs/llm_configuration).\n",
+    "Learn more about the various ways to configure LLM endpoints [here](/docs/topics/llm_configuration).\n",
     "\n",
     "\\:\\:\\:"
    ]
@@ -1536,6 +1530,12 @@
   }
  ],
  "metadata": {
+  "front_matter": {
+   "description": "Use conversational agents to solve a set of tasks with a sequence of chats.",
+   "tags": [
+    "sequential chat"
+   ]
+  },
   "kernelspec": {
    "display_name": "Python 3",
    "language": "python",

notebook/agentchat_nestedchat.ipynb

@@ -9,12 +9,6 @@
     }
    },
    "source": [
-    "<!--\n",
-    "tags: [\"nested chat\"]\n",
-    "description: |\n",
-    "    Solve complex tasks with one or more sequence chats nested as inner monologue.\n",
-    "-->\n",
-    "\n",
     "# Solving Complex Tasks with Nested Chats\n",
     "\n",
     "This notebook shows how you can leverage \"nested chats\" to solve complex task with AutoGen. Nested chats allow AutoGen agents to use other agents as their inner monologue to accomplish tasks. This abstraction is powerful as it allows you to compose agents in rich ways. This notebook shows how you can nest a pretty complex sequence of chats among _inner_ agents inside an _outer_ agent.\n",
@@ -49,7 +43,7 @@
    "source": [
     "\\:\\:\\:tip\n",
     "\n",
-    "Learn more about the various ways to configure LLM endpoints [here](/docs/llm_configuration).\n",
+    "Learn more about the various ways to configure LLM endpoints [here](/docs/topics/llm_configuration).\n",
     "\n",
     "\\:\\:\\:"
    ]
@@ -800,6 +794,7 @@
     "]\n",
     "assistant_1.register_nested_chats(\n",
     "    nested_chat_queue,\n",
+    "    trigger=user,\n",
     ")\n",
     "# user.initiate_chat(assistant, message=tasks[0], max_turns=1)\n",
     "\n",
@@ -813,6 +808,10 @@
   }
  ],
  "metadata": {
+  "front_matter": {
+    "tags": ["nested chat"],
+    "description": "Solve complex tasks with one or more sequence chats nested as inner monologue."
+  },
   "kernelspec": {
    "display_name": "Python 3",
    "language": "python",

notebook/agentchat_oai_assistant_groupchat.ipynb

@@ -62,7 +62,7 @@
     "]\n",
     "```\n",
     "\n",
-    "Currently Azure OpenAI does not support assistant api. You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/llm_configuration.ipynb) for full code examples of the different methods."
+    "Currently Azure OpenAI does not support assistant api. You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods."
    ]
   },
   {

notebook/agentchat_society_of_mind.ipynb

@@ -5,12 +5,6 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "<!--\n",
-    "tags: [\"orchestration\"]\n",
-    "description: |\n",
-    "    Explore the demonstration of the SocietyOfMindAgent in the AutoGen library, which runs a group chat as an internal monologue, but appears to the external world as a single agent, offering a structured way to manage complex interactions among multiple agents and handle issues such as extracting responses from complex dialogues and dealing with context window constraints.\n",
-    "-->\n",
-    "\n",
     "# SocietyOfMindAgent\n",
     "\n",
     "This notebook demonstrates the SocietyOfMindAgent, which runs a group chat as an internal monologue, but appears to the external world as a single agent. This confers three distinct advantages:\n",
@@ -57,7 +51,7 @@
    "source": [
     "````{=mdx}\n",
     ":::tip\n",
-    "Learn more about configuring LLMs for agents [here](/docs/llm_configuration).\n",
+    "Learn more about configuring LLMs for agents [here](/docs/topics/llm_configuration).\n",
     ":::\n",
     "````\n",
     "\n",
@@ -362,6 +356,10 @@
   }
  ],
  "metadata": {
+  "front_matter": {
+    "tags": ["orchestration"],
+    "description": "Explore the demonstration of the SocietyOfMindAgent in the AutoGen library, which runs a group chat as an internal monologue, but appears to the external world as a single agent, offering a structured way to manage complex interactions among multiple agents and handle issues such as extracting responses from complex dialogues and dealing with context window constraints."
+  },
   "kernelspec": {
    "display_name": "Python 3 (ipykernel)",
    "language": "python",

notebook/agentchat_stream.ipynb

@@ -102,7 +102,7 @@
     "]\n",
     "```\n",
     "\n",
-    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/llm_configuration.ipynb) for full code examples of the different methods."
+    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods."
    ]
   },
   {

notebook/agentchat_teachability.ipynb

@@ -161,14 +161,6 @@
     "# Now add the Teachability capability to the agent.\n",
     "teachability.add_to_agent(teachable_agent)\n",
     "\n",
-    "try:\n",
-    "    from termcolor import colored\n",
-    "except ImportError:\n",
-    "\n",
-    "    def colored(x, *args, **kwargs):\n",
-    "        return x\n",
-    "\n",
-    "\n",
     "# Instantiate a UserProxyAgent to represent the user. But in this notebook, all user input will be simulated.\n",
     "user = UserProxyAgent(\n",
     "    name=\"user\",\n",

notebook/agentchat_two_users.ipynb

@@ -82,7 +82,7 @@
     "]\n",
     "```\n",
     "\n",
-    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/llm_configuration.ipynb) for full code examples of the different methods."
+    "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods."
    ]
   },
   {

notebook/agentchats_sequential_chats.ipynb

@@ -1,16 +1,5 @@
 {
  "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "<!--\n",
-    "tags: [\"sequential chats\"]\n",
-    "description: |\n",
-    "    Use AutoGen to solve a set of tasks with a sequence of chats.\n",
-    "-->"
-   ]
-  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -20,11 +9,6 @@
     }
    },
    "source": [
-    "<!--\n",
-    "tags: [\"sequential chat\"]\n",
-    "description: |\n",
-    "    Use AutoGen.initiate_chats to solve a set of tasks with a sequence of chats.\n",
-    "-->\n",
     "# Solving Multiple Tasks in a Sequence of Chats\n",
     "\n",
     "This notebook showcases how to use the new chat interface `autogen.initiate_chats` to solve a set of tasks with a sequence of chats. \n",
@@ -59,7 +43,7 @@
    "source": [
     "\\:\\:\\:tip\n",
     "\n",
-    "Learn more about the various ways to configure LLM endpoints [here](/docs/llm_configuration).\n",
+    "Learn more about the various ways to configure LLM endpoints [here](/docs/topics/llm_configuration).\n",
     "\n",
     "\\:\\:\\:"
    ]
@@ -837,6 +821,12 @@
   }
  ],
  "metadata": {
+  "front_matter": {
+   "description": "Use AutoGen to solve a set of tasks with a sequence of chats.",
+   "tags": [
+    "sequential chats"
+   ]
+  },
   "kernelspec": {
    "display_name": "Python 3",
    "language": "python",

notebook/config_loader_utility_functions.ipynb

@@ -6,7 +6,7 @@
    "source": [
     "# Config loader utility functions\n",
     "\n",
-    "For an introduction to configuring LLMs, refer to the [main configuration docs](https://microsoft.github.io/autogen/docs/llm_configuration). This guide will run through examples of the more advanced utility functions for managing API configurations.\n",
+    "For an introduction to configuring LLMs, refer to the [main configuration docs](https://microsoft.github.io/autogen/docs/topics/llm_configuration). This guide will run through examples of the more advanced utility functions for managing API configurations.\n",
     "\n",
     "Managing API configurations can be tricky, especially when dealing with multiple models and API versions. The provided utility functions assist users in managing these configurations effectively. Ensure your API keys and other sensitive data are stored securely. You might store keys in `.txt` or `.env` files or environment variables for local development. Never expose your API keys publicly. If you insist on storing your key files locally on your repo (you shouldn't), ensure the key file path is added to the `.gitignore` file.\n",
     "\n",

notebook/contributing.md

@@ -2,16 +2,23 @@
 
 ## How to get a notebook displayed on the website
 
-Ensure the first cell is markdown and before absolutely anything else include the following yaml within a comment.
-
-```markdown
-<!--
-tags: ["code generation", "debugging"]
-description: |
-    Use conversable language learning model agents to solve tasks and provide automatic feedback through a comprehensive example of writing, executing, and debugging Python code to compare stock price changes.
--->
+In the notebook metadata set the `tags` and `description` `front_matter` properties. For example:
+
+```json
+{
+    "...": "...",
+    "metadata": {
+        "...": "...",
+        "front_matter": {
+            "tags": ["code generation", "debugging"],
+            "description": "Use conversable language learning model agents to solve tasks and provide automatic feedback through a comprehensive example of writing, executing, and debugging Python code to compare stock price changes."
+        }
+    }
+}
 ```
 
+**Note**: Notebook metadata can be edited by opening the notebook in a text editor (Or "Open With..." -> "Text Editor" in VSCode)
+
 The `tags` field is a list of tags that will be used to categorize the notebook. The `description` field is a brief description of the notebook.
 
 ## Best practices for authoring notebooks
@@ -70,7 +77,7 @@ Then after the code cell where this is used, include the following markdown snip
 ``````
 ````{=mdx}
 :::tip
-Learn more about configuring LLMs for agents [here](/docs/llm_configuration).
+Learn more about configuring LLMs for agents [here](/docs/topics/llm_configuration).
 :::
 ````
 ``````
@@ -102,9 +109,26 @@ If a notebook needs to be skipped then add to the notebook metadata:
 {
     "...": "...",
     "metadata": {
-        "test_skip": "REASON"
+        "skip_test": "REASON"
     }
 }
 ```
 
-Note: Notebook metadata can be edited by opening the notebook in a text editor (Or "Open With..." -> "Text Editor" in VSCode)
+## Metadata fields
+
+All possible metadata fields are as follows:
+```json
+{
+    "...": "...",
+    "metadata": {
+        "...": "...",
+        "front_matter": {
+            "tags": "List[str] - List of tags to categorize the notebook",
+            "description": "str - Brief description of the notebook",
+        },
+        "skip_test": "str - Reason for skipping the test. If present, the notebook will be skipped during testing",
+        "skip_render": "str - Reason for skipping rendering the notebook. If present, the notebook will be left out of the website.",
+        "extra_files_to_copy": "List[str] - List of files to copy to the website. The paths are relative to the notebook directory",
+    }
+}
+```

test/agentchat/contrib/chat_with_teachable_agent.py → test/agentchat/contrib/capabilities/chat_with_teachable_agent.py

@@ -1,24 +1,16 @@
 #!/usr/bin/env python3 -m pytest
 
+import os
+import sys
+from termcolor import colored
 from autogen import UserProxyAgent, config_list_from_json
 from autogen.agentchat.contrib.capabilities.teachability import Teachability
 from autogen import ConversableAgent
 
-import os
-import sys
-
-sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))
 from test_assistant_agent import OAI_CONFIG_LIST, KEY_LOC  # noqa: E402
 
 
-try:
-    from termcolor import colored
-except ImportError:
-
-    def colored(x, *args, **kwargs):
-        return x
-
-
 # Specify the model to use. GPT-3.5 is less reliable than GPT-4 at learning from user input.
 filter_dict = {"model": ["gpt-4-0125-preview"]}
 # filter_dict = {"model": ["gpt-3.5-turbo-1106"]}

test/agentchat/contrib/capabilities/test_context_handling.py

@@ -10,7 +10,7 @@ from autogen import AssistantAgent, UserProxyAgent
 
 # from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST
 
-sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.append(os.path.join(os.path.dirname(__file__), "../../.."))
 from conftest import skip_openai  # noqa: E402
 
 sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))

test/agentchat/contrib/test_teachable_agent.py → test/agentchat/contrib/capabilities/test_teachable_agent.py

@@ -3,28 +3,21 @@
 import pytest
 import os
 import sys
+from termcolor import colored
 from autogen import ConversableAgent, config_list_from_json
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))
+sys.path.append(os.path.join(os.path.dirname(__file__), "../../.."))
 from conftest import skip_openai  # noqa: E402
 
-sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))
 from test_assistant_agent import OAI_CONFIG_LIST, KEY_LOC  # noqa: E402
 
 try:
-    from openai import OpenAI
     from autogen.agentchat.contrib.capabilities.teachability import Teachability
 except ImportError:
     skip = True
 else:
-    skip = False or skip_openai
-
-try:
-    from termcolor import colored
-except ImportError:
-
-    def colored(x, *args, **kwargs):
-        return x
+    skip = skip_openai
 
 
 # Specify the model to use by uncommenting one of the following lines.
@@ -141,7 +134,7 @@ def use_task_advice_pair_phrasing():
 
 @pytest.mark.skipif(
     skip,
-    reason="do not run if dependency is not installed",
+    reason="do not run if dependency is not installed or requested to skip",
 )
 def test_teachability_code_paths():
     """Runs this file's unit tests."""
@@ -172,7 +165,7 @@ def test_teachability_code_paths():
 
 @pytest.mark.skipif(
     skip,
-    reason="do not run if dependency is not installed",
+    reason="do not run if dependency is not installed or requested to skip",
 )
 def test_teachability_accuracy():
     """A very cheap and fast test of teachability accuracy."""

test/agentchat/test_groupchat.py

@@ -779,6 +779,103 @@ def test_clear_agents_history():
         {"content": "How you doing?", "name": "sam", "role": "user"},
     ]
 
+    # testing saving tool_call message when clear history going to remove it leaving only tool_response message
+    agent1.reset()
+    agent2.reset()
+    agent3.reset()
+    # we want to broadcast the message only in the preparation.
+    groupchat = autogen.GroupChat(agents=[agent1, agent2, agent3], messages=[], max_round=1, enable_clear_history=True)
+    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
+    # We want to trigger the broadcast of group chat manager, which requires `request_reply` to be set to True.
+    agent1.send("dummy message", group_chat_manager, request_reply=True)
+    agent1.send(
+        {
+            "content": None,
+            "role": "assistant",
+            "function_call": None,
+            "tool_calls": [
+                {"id": "call_test_id", "function": {"arguments": "", "name": "test_tool"}, "type": "function"}
+            ],
+        },
+        group_chat_manager,
+        request_reply=True,
+    )
+    agent1.send(
+        {
+            "role": "tool",
+            "tool_responses": [{"tool_call_id": "call_emulated", "role": "tool", "content": "example tool response"}],
+            "content": "example tool response",
+        },
+        group_chat_manager,
+        request_reply=True,
+    )
+    # increase max_round to 3
+    groupchat.max_round = 3
+    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
+    with mock.patch.object(builtins, "input", lambda _: "clear history alice 1. How you doing?"):
+        agent1.initiate_chat(group_chat_manager, message="hello", clear_history=False)
+
+    agent1_history = list(agent1._oai_messages.values())[0]
+    assert agent1_history == [
+        {
+            "tool_calls": [
+                {"id": "call_test_id", "function": {"arguments": "", "name": "test_tool"}, "type": "function"},
+            ],
+            "content": None,
+            "role": "assistant",
+        },
+        {
+            "content": "example tool response",
+            "tool_responses": [{"tool_call_id": "call_emulated", "role": "tool", "content": "example tool response"}],
+            "role": "tool",
+        },
+    ]
+
+    # testing clear history called from tool response
+    agent1.reset()
+    agent2.reset()
+    agent3.reset()
+    agent2 = autogen.ConversableAgent(
+        "bob",
+        max_consecutive_auto_reply=10,
+        human_input_mode="NEVER",
+        llm_config=False,
+        default_auto_reply={
+            "role": "tool",
+            "tool_responses": [{"tool_call_id": "call_emulated", "role": "tool", "content": "USER INTERRUPTED"}],
+            "content": "Clear history. How you doing?",
+        },
+    )
+    groupchat = autogen.GroupChat(agents=[agent1, agent2, agent3], messages=[], max_round=1, enable_clear_history=True)
+    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
+    agent1.send("dummy message", group_chat_manager, request_reply=True)
+    agent1.send(
+        {
+            "content": None,
+            "role": "assistant",
+            "function_call": None,
+            "tool_calls": [
+                {"id": "call_test_id", "function": {"arguments": "", "name": "test_tool"}, "type": "function"}
+            ],
+        },
+        group_chat_manager,
+        request_reply=True,
+    )
+    groupchat.max_round = 2
+    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
+
+    agent1.initiate_chat(group_chat_manager, message="hello")
+    agent1_history = list(agent1._oai_messages.values())[0]
+    assert agent1_history == [
+        {
+            "tool_calls": [
+                {"id": "call_test_id", "function": {"arguments": "", "name": "test_tool"}, "type": "function"},
+            ],
+            "content": None,
+            "role": "assistant",
+        },
+    ]
+
 
 def test_get_agent_by_name():
     def agent(name: str) -> autogen.ConversableAgent:

test/agentchat/test_nested.py

@@ -112,6 +112,7 @@ def test_nested():
     ]
     assistant.register_nested_chats(
         nested_chat_queue,
+        trigger=user,
     )
     user.initiate_chats([{"recipient": assistant, "message": tasks[0]}, {"recipient": assistant, "message": tasks[1]}])
 

website/.gitignore

@@ -11,7 +11,8 @@ package-lock.json
 docs/reference
 /docs/notebooks
 
-docs/llm_configuration.mdx
+docs/topics/llm_configuration.mdx
+docs/topics/code-execution/jupyter-code-executor.mdx
 
 # Misc
 .DS_Store

website/blog/2024-03-03-AutoGen-Update/img/.gitattributes

@@ -0,0 +1,3 @@
+gaia.png filter=lfs diff=lfs merge=lfs -text
+dalle_gpt4v.png filter=lfs diff=lfs merge=lfs -text
+teach.png filter=lfs diff=lfs merge=lfs -text

website/blog/2024-03-03-AutoGen-Update/index.mdx

@@ -0,0 +1,178 @@
+---
+title: What's New in AutoGen?
+authors: sonichi
+tags: [news, summary, roadmap]
+---
+
+![autogen is loved](img/love.png)
+
+**TL;DR**
+- **AutoGen has received tremendous interest and recognition.**
+- **AutoGen has many exciting new features and ongoing research.**
+
+Five months have passed since the initial spinoff of AutoGen from [FLAML](https://github.com/microsoft/FLAML). What have we learned since then? What are the milestones achieved? What's next?
+
+## Background
+
+AutoGen was motivated by two big questions:
+- What are future AI applications like?
+- How do we empower every developer to build them?
+
+Last year, I worked with my colleagues and collaborators from Penn State University and University of Washington, on a new multi-agent framework, to enable the next generation of applications powered by large language models.
+We have been building AutoGen, as a programming framework for agentic AI, just like PyTorch for deep learning.
+We developed AutoGen in an open source project [FLAML](https://github.com/microsoft/FLAML): a fast library for AutoML and tuning. After a few studies like [EcoOptiGen](https://arxiv.org/abs/2303.04673v1) and [MathChat](https://arxiv.org/abs/2306.01337), in August, we published a [technical report](https://arxiv.org/abs/2308.08155v1) about the multi-agent framework.
+In October, we moved AutoGen from FLAML to a standalone repo on GitHub, and published an [updated technical report](https://arxiv.org/abs/2308.08155).
+
+## Feedback
+
+Since then, we've got new feedback every day, everywhere. Users have shown really high recognition of the new levels of capability enabled by AutoGen. For example, there are many comments like the following on X (Twitter) or YouTube.
+
+> Autogen gave me the same a-ha moment that I haven't felt since trying out GPT-3
+for the first time.
+
+> I have never been this surprised since ChatGPT.
+
+
+Many users have deep understanding of the value in different dimensions, such as the modularity, flexibility and simplicity.
+
+> The same reason autogen is significant is the same reason OOP is a good idea. Autogen packages up all that complexity into an agent I can create in one line, or modify with another.
+<!--
+I had lots of ideas I wanted to implement, but it needed a framework like this
+and I am just not the guy to make such a robust and intelligent framework.
+-->
+
+Over time, more and more users share their experiences in using or contributing to autogen.
+
+> In our Data Science department Autogen is helping us develop a production ready
+multi-agents framework.
+>> Sam Khalil, VP Data Insights & FounData, Novo Nordisk
+
+> When I built an interactive learning tool for students, I looked for a tool that
+could streamline the logistics but also give enough flexibility so I could use
+customized tools. AutoGen has both. It simplified the work. Thanks to Chi and his
+team for sharing such a wonderful tool with the community.
+>> Yongsheng Lian, Professor at the University of Louisville, Mechanical Engineering
+
+> Exciting news: the latest AutoGen release now features my contribution…
+This experience has been a wonderful blend of learning and contributing,
+demonstrating the dynamic and collaborative spirit of the tech community.
+>> Davor Runje, Cofounder @ airt / President of the board @ CISEx
+
+> With the support of a grant through the Data Intensive Studies Center at Tufts
+University, our group is hoping to solve some of the challenges students face when
+transitioning from undergraduate to graduate-level courses, particularly in Tufts'
+Doctor of Physical Therapy program in the School of Medicine. We're experimenting
+with Autogen to create tailored assessments, individualized study guides, and focused
+tutoring. This approach has led to significantly better results than those we
+achieved using standard chatbots. With the help of Chi and his group at Microsoft,
+our current experiments include using multiple agents in sequential chat, teachable
+agents, and round-robin style debate formats. These methods have proven more
+effective in generating assessments and feedback compared to other large language
+models (LLMs) we've explored. I've also used OpenAI Assistant agents through Autogen
+in my Primary Care class to facilitate student engagement in patient interviews
+through digital simulations. The agent retrieved information from a real patient
+featured in a published case study, allowing students to practice their interview
+skills with realistic information.
+>> Benjamin D Stern, MS, DPT, Assistant Professor, Doctor of Physical Therapy Program,
+Tufts University School of Medicine
+
+> Autogen has been a game changer for how we analyze companies and products! Through
+collaborative discourse between AI Agents we are able to shave days off our research
+and analysis process.
+>> Justin Trugman, Cofounder & Head of Technology at BetterFutureLabs
+
+These are just a small fraction of examples. We have seen big enterprise customers’ interest from pretty much every vertical industry: Accounting, Airlines, Biotech, Consulting, Consumer Packaged Goods, Electronics, Entertainment, Finance, Fintech, Government, Healthcare, Manufacturer, Metals, Pharmacy, Research, Retailer, Social Media, Software, Supply Chain, Technology, Telecom…
+
+AutoGen is used or contributed by companies, organizations, universities from A to Z, in all over the world. We have seen hundreds of example applications. Some organization uses AutoGen as the backbone to build their agent platform. Others use AutoGen for diverse scenarios, including research and investment to novel and creative applications of multiple agents.
+
+## Milestones
+
+AutoGen has a large and active community of developers, researchers and AI practitioners.
+- 22K+ stars on [GitHub](https://aka.ms/autogen-gh), 3K+ forks
+- 14K+ members on [Discord](https://aka.ms/autogen-dc)
+- 100K+ downloads per months
+- 3M+ views on Youtube (400+ community-generated videos)
+- 100+ citations on [Google Scholar](https://scholar.google.com/citations?view_op=view_citation&hl=en&user=IiSNwnAAAAAJ&citation_for_view=IiSNwnAAAAAJ:zCpYd49hD24C)
+
+I am so amazed by their creativity and passion.
+I also appreciate the recognition and awards AutoGen has received, such as:
+- Selected by [TheSequence: My Five Favorite AI Papers of 2023](https://thesequence.substack.com/p/my-five-favorite-ai-papers-of-2023)
+- Top trending repo on GitHub in Oct'23
+- Selected into [Open100: Top 100 Open Source achievements](https://www.benchcouncil.org/evaluation/opencs/annual.html) only 35 days after spinoff
+
+On March 1, the initial AutoGen multi-agent experiment on the challenging [GAIA](https://huggingface.co/spaces/gaia-benchmark/leaderboard) benchmark turned out to achieve the No. 1 accuracy with a big leap, in all the three levels.
+
+![gaia](img/gaia.png)
+
+That shows the big potential of using AutoGen in solving complex tasks.
+And it's just the beginning of the community's effort to answering a few hard open questions.
+
+## Open Questions
+
+In the [AutoGen technical report](https://arxiv.org/abs/2308.08155), we laid out a number of challenging research questions:
+
+1. How to design optimal multi-agent workflows?
+1. How to create highly capable agents?
+1. How to enable scale, safety and human agency?
+
+The community has been working hard to address them in several dimensions:
+
+- Evaluation. Convenient and insightful evaluation is the foundation of making solid progress.
+- Interface. An intuitive, expressive and standardized interface is the prerequisite of fast experimentation and optimization.
+- Optimization. Both the multi-agent interaction design (e.g., decomposition) and the individual agent capability need to be optimized to satisfy specific application needs.
+- Integration. Integration with new technologies is an effective way to enhance agent capability.
+- Learning/Teaching. Agentic learning and teaching are intuitive approaches for agents to optimize their performance, enable human agency and enhance safety.
+
+## New Features & Ongoing Research
+
+### Evaluation
+
+We are working on agent-based evaluation tools and benchmarking tools. For example:
+
+- [AgentEval](/blog/2023/11/20/AgentEval). Our [research](https://arxiv.org/abs/2402.09015) finds that LLM agents built with AutoGen can be used to automatically identify evaluation criteria and assess the performance from task descriptions and execution logs. It is demonstrated as a [notebook example](https://github.com/microsoft/autogen/blob/main/notebook/agenteval_cq_math.ipynb). Feedback and help are welcome for building it into the library.
+- [AutoGenBench](/blog/2024/01/25/AutoGenBench). AutoGenBench is a commandline tool for downloading, configuring, running an agentic benchmark, and reporting results. It is designed to allow repetition, isolation and instrumentation, leveraging the new [runtime logging](/docs/notebooks/agentchat_logging) feature.
+
+These tools have been used for improving the AutoGen library as well as applications. For example, the new state-of-the-art performance achieved by a multi-agent solution to the [GAIA](https://huggingface.co/spaces/gaia-benchmark/leaderboard) benchmark has benefited from these evaluation tools.
+
+### Interface
+
+We are making rapid progress in further improving the interface to make it even easier to build agent applications. For example:
+
+- [AutoBuild](/blog/2023/11/26/Agent-AutoBuild). AutoBuild is an ongoing area of research to automatically create or select a group of agents for a given task and objective. If successful, it will greatly reduce the effort from users or developers when using the multi-agent technology. It also paves the way for agentic decomposition to handle complex tasks. It is available as an experimental feature and demonstrated in two modes: free-form [creation](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_basic.ipynb) and [selection](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_agent_library.ipynb) from a library.
+- [AutoGen Studio](/blog/2023/12/01/AutoGenStudio). AutoGen Studio is a no-code UI for fast experimentation with the multi-agent conversations. It lowers the barrier of entrance to the AutoGen technology. Models, agents, and workflows can all be configured without writing code. And chatting with multiple agents in a playground is immediately available after the configuration. Although only a subset of `pyautogen` features are available in this sample app, it demonstrates a promising experience. It has generated tremendous excitement in the community.
+- Conversation Programming+. The [AutoGen paper](https://arxiv.org/abs/2308.08155) introduced a key concept of *Conversation Programming*, which can be used to program diverse conversation patterns such as 1-1 chat, group chat, hierarchical chat, nested chat etc. While we offered dynamic group chat as an example of high-level orchestration, it made other patterns relatively less discoverable. Therefore, we have added more convenient conversation programming features which enables easier definition of other types of complex workflow, such as [finite state machine based group chat](/blog/2024/02/11/FSM-GroupChat), [sequential chats](/docs/notebooks/agentchats_sequential_chats), and [nested chats](/docs/notebooks/agentchat_nestedchat). Many users have found them useful in implementing specific patterns, which have been always possible but more obvious with the added features. I will write another blog post for a deep dive.
+
+### Learning/Optimization/Teaching
+
+The features in this category allow agents to remember teachings from users or other agents long term, or improve over iterations. For example:
+
+- [AgentOptimizer](/blog/2023/12/23/AgentOptimizer). This [research](https://arxiv.org/abs/2402.11359) finds an approach of training LLM agents without modifying the model. As a case study, this technique optimizes a set of Python functions for agents to use in solving a set of training tasks. It is planned to be available as an experimental feature.
+- [EcoAssistant](/blog/2023/11/09/EcoAssistant). This [research](https://arxiv.org/abs/2310.03046) finds a multi-agent teaching approach when using agents with different capacities powered by different LLMs. For example, a GPT-4 agent can teach a GPT-3.5 agent by demonstration. With this approach, one only needs 1/3 or 1/2 of GPT-4's cost, while getting 10-20\% higher success rate than GPT-4 on coding-based QA. No finetuning is needed. All you need is a GPT-4 endpoint and a GPT-3.5-turbo endpoint. Help is appreciated to offer this technique as a feature in the AutoGen library.
+- [Teachability](/blog/2023/10/26/TeachableAgent). Every LLM agent in AutoGen can be made teachable, i.e., remember facts, preferences, skills etc. from interacting with other agents. For example, a user behind a user proxy agent can teach an assistant agent instructions in solving a difficult math problem. After teaching once, the problem solving rate for the assistant agent can have a dramatic improvement (e.g., 37% -> 95% for gpt-4-0613).
+![teach](img/teach.png)
+This feature works for GPTAssistantAgent (using OpenAI's assistant API) and group chat as well. One interesting use case of teachability + FSM group chat: [teaching resilience](https://www.linkedin.com/pulse/combatting-ai-naivete-teaching-resilience-emotional-leah-bonser-jdhrc).
+
+### Integration
+
+The extensible design of AutoGen makes it easy to integrate with new technologies. For example:
+- [Custom models and clients](/blog/2024/01/26/Custom-Models) can be used as backends of an agent, such as Huggingface models and inference APIs.
+- [OpenAI assistant](/blog/2023/11/13/OAI-assistants) can be used as the backend of an agent (GPTAssistantAgent). It will be nice to reimplement it as a custom client to increase the compatibility with ConversableAgent.
+- [Multimodality](/blog/2023/11/06/LMM-Agent). LMM models like GPT-4V can be used to provide vision to an agent, and accomplish interesting multimodal tasks by conversing with other agents, including advanced image analysis, figure generation, and automatic iterative improvement in image generation.
+
+![multimodal](img/dalle_gpt4v.png)
+
+The above only covers a subset of new features and roadmap. There are many other interesting new features, integration examples or sample apps:
+- new features like stateful code execution, [tool decorators](/docs/Use-Cases/agent_chat#tool-calling), [long context handling](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_capability_long_context_handling.ipynb), [web agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_surfer.ipynb).
+- integration examples like using [guidance](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_guidance.ipynb) to generate structured response.
+- sample apps like [AutoAnny](/blog/2024/02/02/AutoAnny).
+
+## Call for Help
+
+I appreciate the huge support from more than 14K members in the Discord community.
+Despite all the exciting progress, there are tons of open problems, issues and feature requests awaiting to be solved.
+We need more help to tackle the challenging problems and accelerate the development.
+You're all welcome to join our community and define the future of AI agents together.
+
+*Do you find this update helpful? Would you like to join force? Please join our [Discord](https://discord.gg/pAbnFJrkgZ) server for discussion.*
+
+![contributors](img/contributors.png)

website/docs/Ecosystem.md

@@ -1,29 +0,0 @@
-# Ecosystem
-This page lists libraries that have integrations with Autogen for LLM applications using multiple agents in alphabetical order. Including your own integration to this list is highly encouraged. Simply open a pull request with a few lines of text, see the dropdown below for more information.
-
-
-## MemGPT + AutoGen
-
-
-![MemGPT Example](img/ecosystem-memgpt.png)
-
-MemGPT enables LLMs to manage their own memory and overcome limited context windows. You can use MemGPT to create perpetual chatbots that learn about you and modify their own personalities over time. You can connect MemGPT to your own local filesystems and databases, as well as connect MemGPT to your own tools and APIs. The MemGPT + AutoGen integration allows you to equip any AutoGen agent with MemGPT capabilities.
-
-- [MemGPT + AutoGen Documentation with Code Examples](https://memgpt.readme.io/docs/autogen)
-
-
-## Microsoft Fabric + AutoGen
-
-![Fabric Example](img/ecosystem-fabric.png)
-
-[Microsoft Fabric](https://learn.microsoft.com/en-us/fabric/get-started/microsoft-fabric-overview) is an all-in-one analytics solution for enterprises that covers everything from data movement to data science, Real-Time Analytics, and business intelligence. It offers a comprehensive suite of services, including data lake, data engineering, and data integration, all in one place. In this notenook, we give a simple example for using AutoGen in Microsoft Fabric.
-
-- [Microsoft Fabric + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_microsoft_fabric.ipynb)
-
-## Ollama + AutoGen
-
-![Ollama Example](img/ecosystem-ollama.png)
-
-[Ollama](https://ollama.com/) allows the users to run open-source large language models, such as Llama 2, locally. Ollama bundles model weights, configuration, and data into a single package, defined by a Modelfile. It optimizes setup and configuration details, including GPU usage.
-
-- [Ollama + AutoGen instruction](https://ollama.ai/blog/openai-compatibility)

website/docs/Examples.md

@@ -97,7 +97,7 @@ Links to notebook examples:
 ## Enhanced Inferences
 ### Utilities
 - API Unification  - [View Documentation with Code Example](https://microsoft.github.io/autogen/docs/Use-Cases/enhanced_inference/#api-unification)
-- Utility Functions to Help Managing API configurations effectively - [View Notebook](/docs/llm_configuration)
+- Utility Functions to Help Managing API configurations effectively - [View Notebook](/docs/topics/llm_configuration)
 - Cost Calculation - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_client_cost.ipynb)
 
 ### Inference Hyperparameters Tuning

website/docs/FAQ.md → website/docs/FAQ.mdx

@@ -1,24 +1,8 @@
+import TOCInline from '@theme/TOCInline';
+
 # Frequently Asked Questions
 
-- [Install the correct package - `pyautogen`](#install-the-correct-package---pyautogen)
-- [Set your API endpoints](#set-your-api-endpoints)
-  - [Use the constructed configuration list in agents](#use-the-constructed-configuration-list-in-agents)
-  - [Unexpected keyword argument 'base_url'](#unexpected-keyword-argument-base_url)
-  - [How does an agent decide which model to pick out of the list?](#how-does-an-agent-decide-which-model-to-pick-out-of-the-list)
-  - [Can I use non-OpenAI models?](#can-i-use-non-openai-models)
-- [Handle Rate Limit Error and Timeout Error](#handle-rate-limit-error-and-timeout-error)
-- [How to continue a finished conversation](#how-to-continue-a-finished-conversation)
-- [How do we decide what LLM is used for each agent? How many agents can be used? How do we decide how many agents in the group?](#how-do-we-decide-what-llm-is-used-for-each-agent-how-many-agents-can-be-used-how-do-we-decide-how-many-agents-in-the-group)
-- [Why is code not saved as file?](#why-is-code-not-saved-as-file)
-- [Code execution](#code-execution)
-  - [Enable Python 3 docker image](#enable-python-3-docker-image)
-  - [Agents keep thanking each other when using `gpt-3.5-turbo`](#agents-keep-thanking-each-other-when-using-gpt-35-turbo)
-- [ChromaDB fails in codespaces because of old version of sqlite3](#chromadb-fails-in-codespaces-because-of-old-version-of-sqlite3)
-- [How to register a reply function](#how-to-register-a-reply-function)
-- [How to get last message?](#how-to-get-last-message)
-- [How to get each agent message?](#how-to-get-each-agent-message)
-- [When using autogen docker, is it always necessary to reinstall modules?](#when-using-autogen-docker-is-it-always-necessary-to-reinstall-modules)
-- [Agents are throwing due to docker not running, how can I resolve this?](#agents-are-throwing-due-to-docker-not-running-how-can-i-resolve-this)
+<TOCInline toc={toc} />
 
 ## Install the correct package - `pyautogen`
 
@@ -31,15 +15,15 @@ Typical errors that you might face when using the wrong package are `AttributeEr
 
 ## Set your API endpoints
 
-This documentation has been moved [here](/docs/llm_configuration).
+This documentation has been moved [here](/docs/topics/llm_configuration).
 
 ### Use the constructed configuration list in agents
 
-This documentation has been moved [here](/docs/llm_configuration).
+This documentation has been moved [here](/docs/topics/llm_configuration).
 
 ### How does an agent decide which model to pick out of the list?
 
-This documentation has been moved [here](/docs/llm_configuration#how-does-an-agent-decide-which-model-to-pick-out-of-the-list).
+This documentation has been moved [here](/docs/topics/llm_configuration#how-does-an-agent-decide-which-model-to-pick-out-of-the-list).
 
 ### Unexpected keyword argument 'base_url'
 

website/docs/Research.md

@@ -47,3 +47,29 @@ For technical details, please check our technical report and research publicatio
     booktitle={ArXiv preprint arXiv:2310.03046},
 }
 ```
+
+* [Towards better Human-Agent Alignment: Assessing Task Utility in LLM-Powered Applications](https://arxiv.org/abs/2402.09015). Negar Arabzadeh, Julia Kiseleva, Qingyun Wu, Chi Wang, Ahmed Awadallah, Victor Dibia, Adam Fourney, Charles Clarke. ArXiv preprint arXiv:2402.09015 (2024).
+
+```bibtex
+@misc{Kiseleva2024agenteval,
+      title={Towards better Human-Agent Alignment: Assessing Task Utility in LLM-Powered Applications},
+      author={Negar Arabzadeh and Julia Kiseleva and Qingyun Wu and Chi Wang and Ahmed Awadallah and Victor Dibia and Adam Fourney and Charles Clarke},
+      year={2024},
+      eprint={2402.09015},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
+
+* [Training Language Model Agents without Modifying Language Models](https://arxiv.org/abs/2402.11359). Shaokun Zhang, Jieyu Zhang, Jiale Liu, Linxin Song, Chi Wang, Ranjay Krishna, Qingyun Wu. ArXiv preprint arXiv:2402.09015 (2024).
+
+```bibtex
+@misc{zhang2024agentoptimizer,
+      title={Training Language Model Agents without Modifying Language Models},
+      author={Shaokun Zhang and Jieyu Zhang and Jiale Liu and Linxin Song and Chi Wang and Ranjay Krishna and Qingyun Wu},
+      year={2024},
+      eprint={2402.11359},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI}
+}
+```

website/docs/Use-Cases/enhanced_inference.md

@@ -279,7 +279,7 @@ For convenience, we provide a number of utility functions to load config lists.
 - `config_list_from_models`: Creates configurations based on a provided list of models, useful when targeting specific models without manually specifying each configuration.
 - `config_list_from_dotenv`: Constructs a configuration list from a `.env` file, offering a consolidated way to manage multiple API configurations and keys from a single file.
 
-We suggest that you take a look at this [notebook](/docs/llm_configuration) for full code examples of the different methods to configure your model endpoints.
+We suggest that you take a look at this [notebook](/docs/topics/llm_configuration) for full code examples of the different methods to configure your model endpoints.
 
 ### Logic error
 

website/docs/ecosystem/memgpt.md

@@ -0,0 +1,7 @@
+# MemGPT
+
+![MemGPT Example](img/ecosystem-memgpt.png)
+
+MemGPT enables LLMs to manage their own memory and overcome limited context windows. You can use MemGPT to create perpetual chatbots that learn about you and modify their own personalities over time. You can connect MemGPT to your own local filesystems and databases, as well as connect MemGPT to your own tools and APIs. The MemGPT + AutoGen integration allows you to equip any AutoGen agent with MemGPT capabilities.
+
+- [MemGPT + AutoGen Documentation with Code Examples](https://memgpt.readme.io/docs/autogen)

website/docs/ecosystem/microsoft-fabric.md

@@ -0,0 +1,7 @@
+# Microsoft Fabric
+
+![Fabric Example](img/ecosystem-fabric.png)
+
+[Microsoft Fabric](https://learn.microsoft.com/en-us/fabric/get-started/microsoft-fabric-overview) is an all-in-one analytics solution for enterprises that covers everything from data movement to data science, Real-Time Analytics, and business intelligence. It offers a comprehensive suite of services, including data lake, data engineering, and data integration, all in one place. In this notenook, we give a simple example for using AutoGen in Microsoft Fabric.
+
+- [Microsoft Fabric + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_microsoft_fabric.ipynb)

website/docs/ecosystem/ollama.md

@@ -0,0 +1,7 @@
+# Ollama
+
+![Ollama Example](img/ecosystem-ollama.png)
+
+[Ollama](https://ollama.com/) allows the users to run open-source large language models, such as Llama 2, locally. Ollama bundles model weights, configuration, and data into a single package, defined by a Modelfile. It optimizes setup and configuration details, including GPU usage.
+
+- [Ollama + AutoGen instruction](https://ollama.ai/blog/openai-compatibility)

website/docs/installation/Installation.mdx

@@ -81,7 +81,7 @@ pip install pyautogen
 
 ## Code execution with Docker (default)
 
-Even if you install AutoGen locally, we highly recommend using Docker for [code execution](FAQ.md#code-execution).
+Even if you install AutoGen locally, we highly recommend using Docker for [code execution](FAQ.mdx#code-execution).
 
 The default behaviour for code-execution agents is for code execution to be performed in a docker container.
 

website/docs/topics/code-execution/_category_.json

@@ -0,0 +1,5 @@
+{
+    "position": 2,
+    "label": "Code Execution",
+    "collapsible": true
+}

website/docs/topics/code-execution/jupyter-code-executor.ipynb

@@ -0,0 +1,454 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Jupyter Code Executor\n",
+    "\n",
+    "AutoGen is able to execute code in a stateful Jupyter kernel, this is in contrast to the command line code executor where each code block is executed in a new process. This means that you can define variables in one code block and use them in another. One of the interesting properties of this is that when an error is encountered, only the failing code needs to be re-executed, and not the entire script.\n",
+    "\n",
+    "To use the [`JupyterCodeExecutor`](/docs/reference/coding/jupyter/jupyter_code_executor#jupytercodeexecutor) you need a Jupyter server running. This can be local, in Docker or even a remove server. Then, when constructing the [`JupyterCodeExecutor`](/docs/reference/coding/jupyter/jupyter_code_executor#jupytercodeexecutor) you pass it the server it should connect to.\n",
+    "\n",
+    "## Dependencies\n",
+    "\n",
+    "In order to use Jupyter based code execution some extra dependencies are required. These can be installed with the extra `jupyter-executor`:\n",
+    "\n",
+    "```bash\n",
+    "pip install 'pyautogen[jupyter-executor]'\n",
+    "```\n",
+    "\n",
+    "## Jupyter Server\n",
+    "\n",
+    "### Local\n",
+    "\n",
+    "To run a local Jupyter server, the [`LocalJupyterServer`](/docs/reference/coding/jupyter/local_jupyter_server#localjupyterserver) can be used.\n",
+    "\n",
+    "````{=mdx}\n",
+    ":::warning\n",
+    "The [`LocalJupyterServer`](/docs/reference/coding/jupyter/local_jupyter_server#localjupyterserver) does not function on Windows due to a bug. In this case, you can use the [`DockerJupyterServer`](/docs/reference/coding/jupyter/docker_jupyter_server#dockerjupyterserver) instead or use the [`EmbeddedJupyterServer`](/docs/reference/coding/jupyter/embedded_ipython_code_executor). Do note that the intention is to remove the [`EmbeddedJupyterServer`](/docs/reference/coding/jupyter/embedded_ipython_code_executor) when the bug is fixed.\n",
+    ":::\n",
+    "````"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "exit_code=0 output='Hello, World!\\n' output_files=[]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from autogen.coding import CodeBlock\n",
+    "from autogen.coding.jupyter import LocalJupyterServer, JupyterCodeExecutor\n",
+    "\n",
+    "with LocalJupyterServer() as server:\n",
+    "    executor = JupyterCodeExecutor(server)\n",
+    "    print(\n",
+    "        executor.execute_code_blocks(\n",
+    "            code_blocks=[\n",
+    "                CodeBlock(language=\"python\", code=\"print('Hello, World!')\"),\n",
+    "            ]\n",
+    "        )\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Docker\n",
+    "\n",
+    "To run a Docker based Jupyter server, the [`DockerJupyterServer`](/docs/reference/coding/jupyter/docker_jupyter_server#dockerjupyterserver) can be used."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "exit_code=0 output='Hello, World!\\n' output_files=[]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from autogen.coding import CodeBlock\n",
+    "from autogen.coding.jupyter import DockerJupyterServer, JupyterCodeExecutor\n",
+    "\n",
+    "with DockerJupyterServer() as server:\n",
+    "    executor = JupyterCodeExecutor(server)\n",
+    "    print(\n",
+    "        executor.execute_code_blocks(\n",
+    "            code_blocks=[\n",
+    "                CodeBlock(language=\"python\", code=\"print('Hello, World!')\"),\n",
+    "            ]\n",
+    "        )\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "By default the [`DockerJupyterServer`](/docs/reference/coding/jupyter/docker_jupyter_server#dockerjupyterserver) will build and use a bundled Dockerfile, which can be seen below:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "FROM quay.io/jupyter/docker-stacks-foundation\n",
+      "\n",
+      "SHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n",
+      "\n",
+      "USER ${NB_UID}\n",
+      "RUN mamba install --yes jupyter_kernel_gateway ipykernel &&     mamba clean --all -f -y &&     fix-permissions \"${CONDA_DIR}\" &&     fix-permissions \"/home/${NB_USER}\"\n",
+      "\n",
+      "ENV TOKEN=\"UNSET\"\n",
+      "CMD python -m jupyter kernelgateway --KernelGatewayApp.ip=0.0.0.0     --KernelGatewayApp.port=8888     --KernelGatewayApp.auth_token=\"${TOKEN}\"     --JupyterApp.answer_yes=true     --JupyterWebsocketPersonality.list_kernels=true\n",
+      "\n",
+      "EXPOSE 8888\n",
+      "\n",
+      "WORKDIR \"${HOME}\"\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(DockerJupyterServer.DEFAULT_DOCKERFILE)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Custom Docker Image\n",
+    "\n",
+    "A custom image can be used by passing the `custom_image_name` parameter to the [`DockerJupyterServer`](/docs/reference/coding/jupyter/docker_jupyter_server#dockerjupyterserver) constructor. There are some requirements of the image for it to work correctly:\n",
+    "\n",
+    "- The image must have [Jupyer Kernel Gateway](https://jupyter-kernel-gateway.readthedocs.io/en/latest/) installed and running on port 8888 for the [`JupyterCodeExecutor`](/docs/reference/coding/jupyter/jupyter_code_executor) to be able to connect to it.\n",
+    "- Respect the `TOKEN` environment variable, which is used to authenticate the [`JupyterCodeExecutor`](/docs/reference/coding/jupyter/jupyter_code_executor) with the Jupyter server.\n",
+    "- Ensure the `jupyter kernelgateway` is started with:\n",
+    "    - `--JupyterApp.answer_yes=true` - this ensures that the kernel gateway does not prompt for confirmation when shut down.\n",
+    "    - `--JupyterWebsocketPersonality.list_kernels=true` - this ensures that the kernel gateway lists the available kernels.\n",
+    "\n",
+    "\n",
+    "If you wanted to add extra dependencies (for example `matplotlib` and `numpy`) to this image you could customize the Dockerfile like so:\n",
+    "\n",
+    "```Dockerfile\n",
+    "FROM quay.io/jupyter/docker-stacks-foundation\n",
+    "\n",
+    "SHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n",
+    "\n",
+    "USER ${NB_UID}\n",
+    "RUN mamba install --yes jupyter_kernel_gateway ipykernel matplotlib numpy &&\n",
+    "    mamba clean --all -f -y &&\n",
+    "    fix-permissions \"${CONDA_DIR}\" &&\n",
+    "    fix-permissions \"/home/${NB_USER}\"\n",
+    "\n",
+    "ENV TOKEN=\"UNSET\"\n",
+    "CMD python -m jupyter kernelgateway \\\n",
+    "    --KernelGatewayApp.ip=0.0.0.0 \\\n",
+    "    --KernelGatewayApp.port=8888 \\\n",
+    "    --KernelGatewayApp.auth_token=\"${TOKEN}\" \\\n",
+    "    --JupyterApp.answer_yes=true \\\n",
+    "    --JupyterWebsocketPersonality.list_kernels=true\n",
+    "\n",
+    "EXPOSE 8888\n",
+    "\n",
+    "WORKDIR \"${HOME}\"\n",
+    "```\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Remote\n",
+    "\n",
+    "The [`JupyterCodeExecutor`](/docs/reference/coding/jupyter/jupyter_code_executor) can also connect to a remote Jupyter server. This is done by passing connection information rather than an actual server object into the [`JupyterCodeExecutor`](/docs/reference/coding/jupyter/jupyter_code_executor) constructor.\n",
+    "\n",
+    "```python\n",
+    "from autogen.coding import JupyterCodeExecutor, JupyterConnectionInfo\n",
+    "\n",
+    "executor = JupyterCodeExecutor(\n",
+    "    jupyter_server=JupyterConnectionInfo(host='example.com', use_https=True, port=7893, token='mytoken')\n",
+    ")\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Image outputs\n",
+    "\n",
+    "When Jupyter outputs an image, this is saved as a file into the `output_dir` of the [`JupyterCodeExecutor`](/docs/reference/coding/jupyter/jupyter_code_executor), as specified by the constructor. By default this is the current working directory.\n",
+    "\n",
+    "## Assigning to an agent\n",
+    "\n",
+    "A single server can support multiple agents, as each executor will create its own kernel. To assign an executor to an agent it can be done like so:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from autogen import ConversableAgent\n",
+    "from autogen.coding.jupyter import DockerJupyterServer, JupyterCodeExecutor\n",
+    "from pathlib import Path\n",
+    "\n",
+    "server = DockerJupyterServer()\n",
+    "\n",
+    "output_dir = Path(\"coding\")\n",
+    "output_dir.mkdir(exist_ok=True)\n",
+    "\n",
+    "code_executor_agent = ConversableAgent(\n",
+    "    name=\"code_executor_agent\",\n",
+    "    llm_config=False,\n",
+    "    code_execution_config={\n",
+    "        \"executor\": JupyterCodeExecutor(server, output_dir=output_dir),\n",
+    "    },\n",
+    "    human_input_mode=\"NEVER\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "When using code execution it is critical that you update the system prompt of agents you expect to write code to be able to make use of the executor. For example, for the [`JupyterCodeExecutor`](/docs/reference/coding/jupyter/jupyter_code_executor) you might setup a code writing agent like so:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# The code writer agent's system message is to instruct the LLM on how to\n",
+    "# use the Jupyter code executor with IPython kernel.\n",
+    "code_writer_system_message = \"\"\"\n",
+    "You have been given coding capability to solve tasks using Python code in a stateful IPython kernel.\n",
+    "You are responsible for writing the code, and the user is responsible for executing the code.\n",
+    "\n",
+    "When you write Python code, put the code in a markdown code block with the language set to Python.\n",
+    "For example:\n",
+    "```python\n",
+    "x = 3\n",
+    "```\n",
+    "You can use the variable `x` in subsequent code blocks.\n",
+    "```python\n",
+    "print(x)\n",
+    "```\n",
+    "\n",
+    "Write code incrementally and leverage the statefulness of the kernel to avoid repeating code.\n",
+    "Import libraries in a separate code block.\n",
+    "Define a function or a class in a separate code block.\n",
+    "Run code that produces output in a separate code block.\n",
+    "Run code that involves expensive operations like download, upload, and call external APIs in a separate code block.\n",
+    "\n",
+    "When your code produces an output, the output will be returned to you.\n",
+    "Because you have limited conversation memory, if your code creates an image,\n",
+    "the output will be a path to the image instead of the image itself.\"\"\"\n",
+    "\n",
+    "import os\n",
+    "\n",
+    "code_writer_agent = ConversableAgent(\n",
+    "    \"code_writer\",\n",
+    "    system_message=code_writer_system_message,\n",
+    "    llm_config={\"config_list\": [{\"model\": \"gpt-4\", \"api_key\": os.environ[\"OPENAI_API_KEY\"]}]},\n",
+    "    code_execution_config=False,  # Turn off code execution for this agent.\n",
+    "    max_consecutive_auto_reply=2,\n",
+    "    human_input_mode=\"NEVER\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Then we can use these two agents to solve a problem:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[33mcode_executor_agent\u001b[0m (to code_writer):\n",
+      "\n",
+      "Write Python code to calculate the 14th Fibonacci number.\n",
+      "\n",
+      "--------------------------------------------------------------------------------\n",
+      "\u001b[33mcode_writer\u001b[0m (to code_executor_agent):\n",
+      "\n",
+      "Sure. The Fibonacci sequence is a series of numbers where the next number is found by adding up the two numbers before it. We know that the first two Fibonacci numbers are 0 and 1. After that, the series looks like:\n",
+      "\n",
+      "0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, ...\n",
+      "\n",
+      "So, let's define a Python function to calculate the nth Fibonacci number.\n",
+      "\n",
+      "--------------------------------------------------------------------------------\n",
+      "\u001b[33mcode_executor_agent\u001b[0m (to code_writer):\n",
+      "\n",
+      "\n",
+      "\n",
+      "--------------------------------------------------------------------------------\n",
+      "\u001b[33mcode_writer\u001b[0m (to code_executor_agent):\n",
+      "\n",
+      "Here is the Python function to calculate the nth Fibonacci number:\n",
+      "\n",
+      "```python\n",
+      "def fibonacci(n):\n",
+      "    if n <= 1:\n",
+      "        return n\n",
+      "    else:\n",
+      "        a, b = 0, 1\n",
+      "        for _ in range(2, n+1):\n",
+      "            a, b = b, a+b\n",
+      "        return b\n",
+      "```\n",
+      "\n",
+      "Now, let's use this function to calculate the 14th Fibonacci number.\n",
+      "\n",
+      "```python\n",
+      "fibonacci(14)\n",
+      "```\n",
+      "\n",
+      "--------------------------------------------------------------------------------\n",
+      "\u001b[33mcode_executor_agent\u001b[0m (to code_writer):\n",
+      "\n",
+      "exitcode: 0 (execution succeeded)\n",
+      "Code output: \n",
+      "377\n",
+      "\n",
+      "--------------------------------------------------------------------------------\n",
+      "ChatResult(chat_id=None,\n",
+      "           chat_history=[{'content': 'Write Python code to calculate the 14th '\n",
+      "                                     'Fibonacci number.',\n",
+      "                          'role': 'assistant'},\n",
+      "                         {'content': 'Sure. The Fibonacci sequence is a series '\n",
+      "                                     'of numbers where the next number is '\n",
+      "                                     'found by adding up the two numbers '\n",
+      "                                     'before it. We know that the first two '\n",
+      "                                     'Fibonacci numbers are 0 and 1. After '\n",
+      "                                     'that, the series looks like:\\n'\n",
+      "                                     '\\n'\n",
+      "                                     '0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, '\n",
+      "                                     '...\\n'\n",
+      "                                     '\\n'\n",
+      "                                     \"So, let's define a Python function to \"\n",
+      "                                     'calculate the nth Fibonacci number.',\n",
+      "                          'role': 'user'},\n",
+      "                         {'content': '', 'role': 'assistant'},\n",
+      "                         {'content': 'Here is the Python function to calculate '\n",
+      "                                     'the nth Fibonacci number:\\n'\n",
+      "                                     '\\n'\n",
+      "                                     '```python\\n'\n",
+      "                                     'def fibonacci(n):\\n'\n",
+      "                                     '    if n <= 1:\\n'\n",
+      "                                     '        return n\\n'\n",
+      "                                     '    else:\\n'\n",
+      "                                     '        a, b = 0, 1\\n'\n",
+      "                                     '        for _ in range(2, n+1):\\n'\n",
+      "                                     '            a, b = b, a+b\\n'\n",
+      "                                     '        return b\\n'\n",
+      "                                     '```\\n'\n",
+      "                                     '\\n'\n",
+      "                                     \"Now, let's use this function to \"\n",
+      "                                     'calculate the 14th Fibonacci number.\\n'\n",
+      "                                     '\\n'\n",
+      "                                     '```python\\n'\n",
+      "                                     'fibonacci(14)\\n'\n",
+      "                                     '```',\n",
+      "                          'role': 'user'},\n",
+      "                         {'content': 'exitcode: 0 (execution succeeded)\\n'\n",
+      "                                     'Code output: \\n'\n",
+      "                                     '377',\n",
+      "                          'role': 'assistant'}],\n",
+      "           summary='exitcode: 0 (execution succeeded)\\nCode output: \\n377',\n",
+      "           cost=({'gpt-4-0613': {'completion_tokens': 193,\n",
+      "                                 'cost': 0.028499999999999998,\n",
+      "                                 'prompt_tokens': 564,\n",
+      "                                 'total_tokens': 757},\n",
+      "                  'total_cost': 0.028499999999999998},\n",
+      "                 {'gpt-4-0613': {'completion_tokens': 193,\n",
+      "                                 'cost': 0.028499999999999998,\n",
+      "                                 'prompt_tokens': 564,\n",
+      "                                 'total_tokens': 757},\n",
+      "                  'total_cost': 0.028499999999999998}),\n",
+      "           human_input=[])\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pprint\n",
+    "\n",
+    "chat_result = code_executor_agent.initiate_chat(\n",
+    "    code_writer_agent, message=\"Write Python code to calculate the 14th Fibonacci number.\"\n",
+    ")\n",
+    "\n",
+    "pprint.pprint(chat_result)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Finally, stop the server. Or better yet use a context manager for it to be stopped automatically."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "server.stop()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "autogen",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

website/docs/llm_configuration.ipynb → website/docs/topics/llm_configuration.ipynb

@@ -4,10 +4,6 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "---\n",
-    "custom_edit_url: https://github.com/microsoft/autogen/edit/main/website/docs/llm_configuration.ipynb\n",
-    "---\n",
-    "\n",
     "# LLM Configuration\n",
     "\n",
     "In AutoGen, agents use LLMs as key components to understand and react. To configure an agent's access to LLMs, you can specify an `llm_config` argument in its constructor. For example, the following snippet shows a configuration that uses `gpt-4`:"
@@ -165,7 +161,7 @@
     "Being a list allows you to define multiple models that can be used by the agent. This is useful for a few reasons:\n",
     "\n",
     "- If one model times out or fails, the agent can try another model.\n",
-    "- Having a single global list of models and [filtering it](/docs/llm_configuration#config-list-filtering) based on certain keys (e.g. name, tag) in order to pass select models into a certain agent (e.g. use cheaper GPT 3.5 for agents solving easier tasks)\n",
+    "- Having a single global list of models and [filtering it](#config-list-filtering) based on certain keys (e.g. name, tag) in order to pass select models into a certain agent (e.g. use cheaper GPT 3.5 for agents solving easier tasks)\n",
     "- While the core agents, (e.g. conversable or assistant) do not have special logic around selecting configs, some of the specialized agents *may* have logic to select the best model based on the task at hand.\n",
     "\n",
     "### How does an agent decide which model to pick out of the list?\n",

website/docusaurus.config.js

@@ -50,7 +50,7 @@ module.exports = {
           type: "doc",
           docId: "reference/agentchat/conversable_agent",
           position: "left",
-          label: "SDK",
+          label: "API",
         },
         { to: "blog", label: "Blog", position: "left" },
         {
@@ -76,18 +76,9 @@ module.exports = {
         //   label: "Notebooks",
         // },
         {
-          label: "Resources",
-          type: "dropdown",
-          items: [
-            {
-              type: "doc",
-              docId: "Ecosystem",
-            },
-            {
-              type: "doc",
-              docId: "Gallery",
-            },
-          ],
+          type: "doc",
+          position: "left",
+          docId: "Gallery",
         },
         {
           label: "Other Languages",
@@ -192,9 +183,21 @@ module.exports = {
       {
         redirects: [
           {
-            to: "/docs/llm_configuration/",
+            to: "/docs/topics/llm_configuration",
             from: ["/docs/llm_endpoint_configuration/"],
           },
+          {
+            to: "/docs/ecosystem/memgpt/",
+            from: ["/docs/Ecosystem"],
+          },
+          {
+            to: "/docs/Getting-Started",
+            from: ["/docs/"],
+          },
+          {
+            to: "/docs/topics/llm_configuration",
+            from: ["/docs/llm_configuration"],
+          },
         ],
       },
     ]

website/process_notebooks.py

@@ -14,11 +14,10 @@ import time
 import typing
 import concurrent.futures
 import os
-
-from typing import Optional, Tuple, Union
+from typing import Dict, Optional, Tuple, Union
 from dataclasses import dataclass
-
 from multiprocessing import current_process
+from termcolor import colored
 
 try:
     import yaml
@@ -47,14 +46,6 @@ except ImportError:
         print("test won't work without nbclient")
 
 
-try:
-    from termcolor import colored
-except ImportError:
-
-    def colored(x, *args, **kwargs):
-        return x
-
-
 class Result:
     def __init__(self, returncode: int, stdout: str, stderr: str):
         self.returncode = returncode
@@ -65,7 +56,12 @@ class Result:
 def check_quarto_bin(quarto_bin: str = "quarto") -> None:
     """Check if quarto is installed."""
     try:
-        subprocess.check_output([quarto_bin, "--version"])
+        version = subprocess.check_output([quarto_bin, "--version"], text=True).strip()
+        version = tuple(map(int, version.split(".")))
+        if version < (1, 5, 23):
+            print("Quarto version is too old. Please upgrade to 1.5.23 or later.")
+            sys.exit(1)
+
     except FileNotFoundError:
         print("Quarto is not installed. Please install it from https://quarto.org")
         sys.exit(1)
@@ -76,32 +72,9 @@ def notebooks_target_dir(website_directory: Path) -> Path:
     return website_directory / "docs" / "notebooks"
 
 
-def extract_yaml_from_notebook(notebook: Path) -> typing.Optional[typing.Dict]:
-    with open(notebook, "r", encoding="utf-8") as f:
-        content = f.read()
-
-    json_content = json.loads(content)
-    first_cell = json_content["cells"][0]
-
-    # <!-- and --> must exists on lines on their own
-    if first_cell["cell_type"] != "markdown":
-        return None
-
-    lines = first_cell["source"]
-    if "<!--" != lines[0].strip():
-        return None
-
-    # remove trailing whitespace
-    lines = [line.rstrip() for line in lines]
-
-    if "-->" not in lines:
-        return None
-
-    closing_arrow_idx = lines.index("-->")
-
-    front_matter_lines = lines[1:closing_arrow_idx]
-    front_matter = yaml.safe_load("\n".join(front_matter_lines))
-    return front_matter
+def load_metadata(notebook: Path) -> typing.Dict:
+    content = json.load(notebook.open())
+    return content["metadata"]
 
 
 def skip_reason_or_none_if_ok(notebook: Path) -> typing.Optional[str]:
@@ -125,29 +98,20 @@ def skip_reason_or_none_if_ok(notebook: Path) -> typing.Optional[str]:
     first_cell = json_content["cells"][0]
 
     # <!-- and --> must exists on lines on their own
-    if first_cell["cell_type"] != "markdown":
-        return "first cell is not markdown"
-
-    lines = first_cell["source"]
-    if "<!--" != lines[0].strip():
-        return "first line does not contain only '<!--'"
-
-    # remove trailing whitespace
-    lines = [line.rstrip() for line in lines]
+    if first_cell["cell_type"] == "markdown" and first_cell["source"][0].strip() == "<!--":
+        raise ValueError(
+            f"Error in {str(notebook.resolve())} - Front matter should be defined in the notebook metadata now."
+        )
 
-    if "-->" not in lines:
-        return "no closing --> found, or it is not on a line on its own"
+    metadata = load_metadata(notebook)
 
-    try:
-        front_matter = extract_yaml_from_notebook(notebook)
-    except yaml.YAMLError as e:
-        return colored(f"Failed to parse front matter in {notebook.name}: {e}", "red")
+    if "skip_render" in metadata:
+        return metadata["skip_render"]
 
-    # Should not be none at this point as we have already done the same checks as in extract_yaml_from_notebook
-    assert front_matter is not None, f"Front matter is None for {notebook.name}"
+    if "front_matter" not in metadata:
+        return "front matter missing from notebook metadata ⚠️"
 
-    if "skip" in front_matter and front_matter["skip"] is True:
-        return "skip is set to true"
+    front_matter = metadata["front_matter"]
 
     if "tags" not in front_matter:
         return "tags is not in front matter"
@@ -166,21 +130,54 @@ def skip_reason_or_none_if_ok(notebook: Path) -> typing.Optional[str]:
     return None
 
 
+def extract_title(notebook: Path) -> Optional[str]:
+    """Extract the title of the notebook."""
+    with open(notebook, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Load the json and get the first cell
+    json_content = json.loads(content)
+    first_cell = json_content["cells"][0]
+
+    # find the # title
+    for line in first_cell["source"]:
+        if line.startswith("# "):
+            title = line[2:].strip()
+            # Strip off the { if it exists
+            if "{" in title:
+                title = title[: title.find("{")].strip()
+            return title
+
+    return None
+
+
 def process_notebook(src_notebook: Path, website_dir: Path, notebook_dir: Path, quarto_bin: str, dry_run: bool) -> str:
     """Process a single notebook."""
 
     in_notebook_dir = "notebook" in src_notebook.parts
 
+    metadata = load_metadata(src_notebook)
+
+    title = extract_title(src_notebook)
+    if title is None:
+        return fmt_error(src_notebook, "Title not found in notebook")
+
+    front_matter = {}
+    if "front_matter" in metadata:
+        front_matter = metadata["front_matter"]
+
+    front_matter["title"] = title
+
     if in_notebook_dir:
-        relative_notebook = src_notebook.relative_to(notebook_dir)
+        relative_notebook = src_notebook.resolve().relative_to(notebook_dir.resolve())
         dest_dir = notebooks_target_dir(website_directory=website_dir)
-        target_mdx_file = dest_dir / relative_notebook.with_suffix(".mdx")
+        target_file = dest_dir / relative_notebook.with_suffix(".mdx")
         intermediate_notebook = dest_dir / relative_notebook
 
         # If the intermediate_notebook already exists, check if it is newer than the source file
-        if target_mdx_file.exists():
-            if target_mdx_file.stat().st_mtime > src_notebook.stat().st_mtime:
-                return colored(f"Skipping {src_notebook.name}, as target file is newer", "blue")
+        if target_file.exists():
+            if target_file.stat().st_mtime > src_notebook.stat().st_mtime:
+                return fmt_skip(src_notebook, f"target file ({target_file.name}) is newer ☑️")
 
         if dry_run:
             return colored(f"Would process {src_notebook.name}", "green")
@@ -191,11 +188,8 @@ def process_notebook(src_notebook: Path, website_dir: Path, notebook_dir: Path,
 
         # Check if another file has to be copied too
         # Solely added for the purpose of agent_library_example.json
-        front_matter = extract_yaml_from_notebook(src_notebook)
-        # Should not be none at this point as we have already done the same checks as in extract_yaml_from_notebook
-        assert front_matter is not None, f"Front matter is None for {src_notebook.name}"
-        if "extra_files_to_copy" in front_matter:
-            for file in front_matter["extra_files_to_copy"]:
+        if "extra_files_to_copy" in metadata:
+            for file in metadata["extra_files_to_copy"]:
                 shutil.copy(src_notebook.parent / file, dest_dir / file)
 
         # Capture output
@@ -203,28 +197,19 @@ def process_notebook(src_notebook: Path, website_dir: Path, notebook_dir: Path,
             [quarto_bin, "render", intermediate_notebook], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
         )
         if result.returncode != 0:
-            return (
-                colored(f"Failed to render {intermediate_notebook}", "red")
-                + f"\n{result.stderr}"
-                + f"\n{result.stdout}"
+            return fmt_error(
+                src_notebook, f"Failed to render {src_notebook}\n\nstderr:\n{result.stderr}\nstdout:\n{result.stdout}"
             )
 
         # Unlink intermediate files
         intermediate_notebook.unlink()
-
-        if "extra_files_to_copy" in front_matter:
-            for file in front_matter["extra_files_to_copy"]:
-                (dest_dir / file).unlink()
-
-        # Post process the file
-        post_process_mdx(target_mdx_file)
     else:
-        target_mdx_file = src_notebook.with_suffix(".mdx")
+        target_file = src_notebook.with_suffix(".mdx")
 
         # If the intermediate_notebook already exists, check if it is newer than the source file
-        if target_mdx_file.exists():
-            if target_mdx_file.stat().st_mtime > src_notebook.stat().st_mtime:
-                return colored(f"Skipping {src_notebook.name}, as target file is newer", "blue")
+        if target_file.exists():
+            if target_file.stat().st_mtime > src_notebook.stat().st_mtime:
+                return fmt_skip(src_notebook, f"target file ({target_file.name}) is newer ☑️")
 
         if dry_run:
             return colored(f"Would process {src_notebook.name}", "green")
@@ -233,9 +218,13 @@ def process_notebook(src_notebook: Path, website_dir: Path, notebook_dir: Path,
             [quarto_bin, "render", src_notebook], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
         )
         if result.returncode != 0:
-            return colored(f"Failed to render {src_notebook}", "red") + f"\n{result.stderr}" + f"\n{result.stdout}"
+            return fmt_error(
+                src_notebook, f"Failed to render {src_notebook}\n\nstderr:\n{result.stderr}\nstdout:\n{result.stdout}"
+            )
 
-    return colored(f"Processed {src_notebook.name}", "green")
+    post_process_mdx(target_file, src_notebook, front_matter)
+
+    return fmt_ok(src_notebook)
 
 
 # Notebook execution based on nbmake: https://github.com/treebeardtech/nbmakes
@@ -258,21 +247,14 @@ NB_VERSION = 4
 def test_notebook(notebook_path: Path, timeout: int = 300) -> Tuple[Path, Optional[Union[NotebookError, NotebookSkip]]]:
     nb = nbformat.read(str(notebook_path), NB_VERSION)
 
-    allow_errors = False
-    if "execution" in nb.metadata:
-        if "timeout" in nb.metadata.execution:
-            timeout = nb.metadata.execution.timeout
-        if "allow_errors" in nb.metadata.execution:
-            allow_errors = nb.metadata.execution.allow_errors
-
-    if "test_skip" in nb.metadata:
-        return notebook_path, NotebookSkip(reason=nb.metadata.test_skip)
+    if "skip_test" in nb.metadata:
+        return notebook_path, NotebookSkip(reason=nb.metadata.skip_test)
 
     try:
         c = NotebookClient(
             nb,
             timeout=timeout,
-            allow_errors=allow_errors,
+            allow_errors=False,
             record_timing=True,
         )
         os.environ["PYDEVD_DISABLE_FILE_VALIDATION"] = "1"
@@ -327,24 +309,32 @@ def get_error_info(nb: NotebookNode) -> Optional[NotebookError]:
 
 
 # rendered_notebook is the final mdx file
-def post_process_mdx(rendered_mdx: Path) -> None:
-    notebook_name = f"{rendered_mdx.stem}.ipynb"
+def post_process_mdx(rendered_mdx: Path, source_notebooks: Path, front_matter: Dict) -> None:
     with open(rendered_mdx, "r", encoding="utf-8") as f:
         content = f.read()
 
-    # Check for existence of "export const quartoRawHtml", this indicates there was a front matter line in the file
-    if "export const quartoRawHtml" not in content:
-        raise ValueError(f"File {rendered_mdx} does not contain 'export const quartoRawHtml'")
-
-    # Extract the text between <!-- and -->
-    front_matter = content.split("<!--")[1].split("-->")[0]
-    # Strip empty lines before and after
-    front_matter = "\n".join([line for line in front_matter.split("\n") if line.strip() != ""])
+    # If there is front matter in the mdx file, we need to remove it
+    if content.startswith("---"):
+        front_matter_end = content.find("---", 3)
+        front_matter = yaml.safe_load(content[4:front_matter_end])
+        content = content[front_matter_end + 3 :]
+
+    # Each intermediate path needs to be resolved for this to work reliably
+    repo_root = Path(__file__).parent.resolve().parent.resolve()
+    repo_relative_notebook = source_notebooks.resolve().relative_to(repo_root)
+    front_matter["source_notebook"] = f"/{repo_relative_notebook}"
+    front_matter["custom_edit_url"] = f"https://github.com/microsoft/autogen/edit/main/{repo_relative_notebook}"
+
+    # Is there a title on the content? Only search up until the first code cell
+    first_code_cell = content.find("```")
+    if first_code_cell != -1:
+        title_search_content = content[:first_code_cell]
+    else:
+        title_search_content = content
 
-    # add file path
-    front_matter += f"\nsource_notebook: /notebook/{notebook_name}"
-    # Custom edit url
-    front_matter += f"\ncustom_edit_url: https://github.com/microsoft/autogen/edit/main/notebook/{notebook_name}"
+    title_exists = title_search_content.find("\n# ") != -1
+    if not title_exists:
+        content = f"# {front_matter['title']}\n{content}"
 
     # inject in content directly after the markdown title the word done
     # Find the end of the line with the title
@@ -352,10 +342,11 @@ def post_process_mdx(rendered_mdx: Path) -> None:
 
     # Extract page title
     title = content[content.find("#") + 1 : content.find("\n", content.find("#"))].strip()
+    # If there is a { in the title we trim off the { and everything after it
+    if "{" in title:
+        title = title[: title.find("{")].strip()
 
-    front_matter += f"\ntitle: {title}"
-
-    github_link = f"https://github.com/microsoft/autogen/blob/main/notebook/{notebook_name}"
+    github_link = f"https://github.com/microsoft/autogen/blob/main/{repo_relative_notebook}"
     content = (
         content[:title_end]
         + "\n[![Open on GitHub](https://img.shields.io/badge/Open%20on%20GitHub-grey?logo=github)]("
@@ -366,20 +357,24 @@ def post_process_mdx(rendered_mdx: Path) -> None:
 
     # If no colab link is present, insert one
     if "colab-badge.svg" not in content:
+        colab_link = f"https://colab.research.google.com/github/microsoft/autogen/blob/main/{repo_relative_notebook}"
         content = (
             content[:title_end]
-            + "\n[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/"
-            + notebook_name
+            + "\n[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)]("
+            + colab_link
             + ")"
             + content[title_end:]
         )
 
+    # Dump front_matter to ysaml
+    front_matter = yaml.dump(front_matter, default_flow_style=False)
+
     # Rewrite the content as
     # ---
     # front_matter
     # ---
     # content
-    new_content = f"---\n{front_matter}\n---\n{content}"
+    new_content = f"---\n{front_matter}---\n{content}"
     with open(rendered_mdx, "w", encoding="utf-8") as f:
         f.write(new_content)
 
@@ -395,6 +390,23 @@ def collect_notebooks(notebook_directory: Path, website_directory: Path) -> typi
     return notebooks
 
 
+def fmt_skip(notebook: Path, reason: str) -> str:
+    return f"{colored('[Skip]', 'yellow')} {colored(notebook.name, 'blue')}: {reason}"
+
+
+def fmt_ok(notebook: Path) -> str:
+    return f"{colored('[OK]', 'green')} {colored(notebook.name, 'blue')} ✅"
+
+
+def fmt_error(notebook: Path, error: Union[NotebookError, str]) -> str:
+    if isinstance(error, str):
+        return f"{colored('[Error]', 'red')} {colored(notebook.name, 'blue')}: {error}"
+    elif isinstance(error, NotebookError):
+        return f"{colored('[Error]', 'red')} {colored(notebook.name, 'blue')}: {error.error_name} - {error.error_value}"
+    else:
+        raise ValueError("error must be a string or a NotebookError")
+
+
 def start_thread_to_terminate_when_parent_process_dies(ppid: int):
     pid = os.getpid()
 
@@ -424,7 +436,6 @@ def main() -> None:
     parser.add_argument(
         "--website-directory", type=path, help="Root directory of docusarus website", default=script_dir
     )
-    parser.add_argument("--workers", help="Number of workers to use", type=int, default=-1)
 
     render_parser = subparsers.add_parser("render")
     render_parser.add_argument("--quarto-bin", help="Path to quarto binary", default="quarto")
@@ -435,10 +446,9 @@ def main() -> None:
     test_parser.add_argument("--timeout", help="Timeout for each notebook", type=int, default=60)
     test_parser.add_argument("--exit-on-first-fail", "-e", help="Exit after first test fail", action="store_true")
     test_parser.add_argument("notebooks", type=path, nargs="*", default=None)
+    test_parser.add_argument("--workers", help="Number of workers to use", type=int, default=-1)
 
     args = parser.parse_args()
-    if args.workers == -1:
-        args.workers = None
 
     if args.subcommand is None:
         print("No subcommand specified")
@@ -453,13 +463,13 @@ def main() -> None:
     for notebook in collected_notebooks:
         reason = skip_reason_or_none_if_ok(notebook)
         if reason:
-            print(f"{colored('[Skip]', 'yellow')} {colored(notebook.name, 'blue')}: {reason}")
+            print(fmt_skip(notebook, reason))
         else:
             filtered_notebooks.append(notebook)
 
-    print(f"Processing {len(filtered_notebooks)} notebook{'s' if len(filtered_notebooks) != 1 else ''}...")
-
     if args.subcommand == "test":
+        if args.workers == -1:
+            args.workers = None
         failure = False
         with concurrent.futures.ProcessPoolExecutor(
             max_workers=args.workers,
@@ -471,26 +481,21 @@ def main() -> None:
                 notebook, optional_error_or_skip = future.result()
                 if isinstance(optional_error_or_skip, NotebookError):
                     if optional_error_or_skip.error_name == "timeout":
-                        print(
-                            f"{colored('[Error]', 'red')} {colored(notebook.name, 'blue')}: {optional_error_or_skip.error_name}"
-                        )
+                        print(fmt_error(notebook, optional_error_or_skip.error_name))
 
                     else:
                         print("-" * 80)
-                        print(
-                            f"{colored('[Error]', 'red')} {colored(notebook.name, 'blue')}: {optional_error_or_skip.error_name} - {optional_error_or_skip.error_value}"
-                        )
+
+                        print(fmt_error(notebook, optional_error_or_skip))
                         print(optional_error_or_skip.traceback)
                         print("-" * 80)
                     if args.exit_on_first_fail:
                         sys.exit(1)
                     failure = True
                 elif isinstance(optional_error_or_skip, NotebookSkip):
-                    print(
-                        f"{colored('[Skip]', 'yellow')} {colored(notebook.name, 'blue')}: {optional_error_or_skip.reason}"
-                    )
+                    print(fmt_skip(notebook, optional_error_or_skip.reason))
                 else:
-                    print(f"{colored('[OK]', 'green')} {colored(notebook.name, 'blue')}")
+                    print(fmt_ok(notebook))
 
         if failure:
             sys.exit(1)
@@ -501,15 +506,12 @@ def main() -> None:
         if not notebooks_target_dir(args.website_directory).exists():
             notebooks_target_dir(args.website_directory).mkdir(parents=True)
 
-        with concurrent.futures.ProcessPoolExecutor(max_workers=args.workers) as executor:
-            futures = [
-                executor.submit(
-                    process_notebook, f, args.website_directory, args.notebook_directory, args.quarto_bin, args.dry_run
+        for notebook in filtered_notebooks:
+            print(
+                process_notebook(
+                    notebook, args.website_directory, args.notebook_directory, args.quarto_bin, args.dry_run
                 )
-                for f in filtered_notebooks
-            ]
-            for future in concurrent.futures.as_completed(futures):
-                print(future.result())
+            )
     else:
         print("Unknown subcommand")
         sys.exit(1)

website/sidebars.js

@@ -22,10 +22,11 @@
         id: "installation/Installation"
       },
     },
-    'llm_configuration',
+    {'Topics': [{type: 'autogenerated', dirName: 'topics'}]},
     {'Use Cases': [{type: 'autogenerated', dirName: 'Use-Cases'}]},
     'Contribute',
     'Research',
+    {'Ecosystem': [{type: 'autogenerated', dirName: 'ecosystem'}]},
     'Migration-Guide'
   ],
   // pydoc-markdown auto-generated markdowns from docstrings