Websurfer Refactor (#4165)

* first commit

* use_ocr flag

* initial refactor

* test

* small fixes

* adding animations, in progress

* red boundary animations

* add hover tool

* fix timeout time - reduces a lot of errors

* put prompts in separate file

* prompt organize

* add cursor animation

* format and checks pass

* added check for reset

python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py

@@ -15,7 +15,6 @@ from typing import (
     Optional,
     Sequence,
     Tuple,
-    Union,
     cast,
 )
 
@@ -38,15 +37,17 @@ from autogen_core.components.models import (
     SystemMessage,
     UserMessage,
 )
-from playwright._impl._errors import Error as PlaywrightError
-from playwright._impl._errors import TimeoutError
+from PIL import Image
 from playwright.async_api import BrowserContext, Download, Page, Playwright, async_playwright
 
 from ._events import WebSurferEvent
+from ._playwright_controller import PlaywrightController
+from ._prompts import WEB_SURFER_OCR_PROMPT, WEB_SURFER_QA_PROMPT, WEB_SURFER_QA_SYSTEM_MESSAGE, WEB_SURFER_TOOL_PROMPT
 from ._set_of_mark import add_set_of_mark
 from ._tool_definitions import (
     TOOL_CLICK,
     TOOL_HISTORY_BACK,
+    TOOL_HOVER,
     TOOL_PAGE_DOWN,
     TOOL_PAGE_UP,
     TOOL_READ_PAGE_AND_ANSWER,
@@ -58,13 +59,7 @@ from ._tool_definitions import (
     TOOL_VISIT_URL,
     TOOL_WEB_SEARCH,
 )
-from ._types import (
-    InteractiveRegion,
-    UserContent,
-    VisualViewport,
-    interactiveregion_from_dict,
-    visualviewport_from_dict,
-)
+from ._types import InteractiveRegion, UserContent
 from ._utils import message_content_to_str
 
 # Viewport dimensions
@@ -98,6 +93,11 @@ class MultimodalWebSurfer(BaseChatAgent):
         downloads_folder: str | None = None,
         debug_dir: str | None = os.getcwd(),
         to_save_screenshots: bool = False,
+        animate_actions: bool = False,
+        use_ocr: bool = True,
+        to_resize_viewport: bool = True,
+        playwright: Playwright | None = None,
+        context: BrowserContext | None = None,
     ):
         """
         Initialize the MultimodalWebSurfer.
@@ -113,6 +113,11 @@ class MultimodalWebSurfer(BaseChatAgent):
             downloads_folder (str | None): The folder to save downloads. Defaults to None.
             debug_dir (str | None): The directory to save debug information. Defaults to the current working directory.
             to_save_screenshots (bool): Whether to save screenshots. Defaults to False.
+            animate_actions (bool): Whether to animate actions. Defaults to False.
+            use_ocr (bool): Whether to use OCR to extract text from screenshots, otherwise extract text from page. Defaults to True.
+            to_resize_viewport (bool): Whether to resize the viewport. Defaults to True.
+            playwright (Playwright | None): The playwright instance to use. Defaults to None and creates a new one.
+            context (BrowserContext | None): The browser context to use. Defaults to None and creates a new one.
         """
         super().__init__(name, description)
         self._model_client = model_client
@@ -124,21 +129,18 @@ class MultimodalWebSurfer(BaseChatAgent):
         self.downloads_folder = downloads_folder
         self.debug_dir = debug_dir
         self.to_save_screenshots = to_save_screenshots
+        self.use_ocr = use_ocr
+        self.to_resize_viewport = to_resize_viewport
+        self.animate_actions = animate_actions
 
-        self._chat_history: List[LLMMessage] = []
-
-        # Call init to set these
-        self._playwright: Playwright | None = None
-        self._context: BrowserContext | None = None
+        # Call init to set these in case not set
+        self._playwright: Playwright | None = playwright
+        self._context: BrowserContext | None = context
         self._page: Page | None = None
         self._last_download: Download | None = None
         self._prior_metadata_hash: str | None = None
         self.logger = logging.getLogger(EVENT_LOGGER_NAME + f".{self.name}.MultimodalWebSurfer")
-
-        # Read page_script
-        self._page_script: str = ""
-        with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js"), "rt") as fh:
-            self._page_script = fh.read()
+        self._chat_history: List[LLMMessage] = []
 
         # Define the download handler
         def _download_handler(download: Download) -> None:
@@ -146,6 +148,27 @@ class MultimodalWebSurfer(BaseChatAgent):
 
         self._download_handler = _download_handler
 
+        # Define the Playwright controller that handles the browser interactions
+        self._playwright_controller = PlaywrightController(
+            animate_actions=self.animate_actions,
+            downloads_folder=self.downloads_folder,
+            viewport_width=VIEWPORT_WIDTH,
+            viewport_height=VIEWPORT_HEIGHT,
+            _download_handler=self._download_handler,
+            to_resize_viewport=self.to_resize_viewport,
+        )
+        self.default_tools = [
+            TOOL_VISIT_URL,
+            TOOL_HISTORY_BACK,
+            TOOL_CLICK,
+            TOOL_TYPE,
+            TOOL_READ_PAGE_AND_ANSWER,
+            TOOL_SUMMARIZE_PAGE,
+            TOOL_SLEEP,
+            TOOL_HOVER,
+        ]
+        self.did_lazy_init = False
+
     @property
     def produced_message_types(self) -> List[type[ChatMessage]]:
         return [MultiModalMessage]
@@ -169,9 +192,18 @@ class MultimodalWebSurfer(BaseChatAgent):
             )
 
     async def on_reset(self, cancellation_token: CancellationToken) -> None:
+        if not self.did_lazy_init:
+            return
         assert self._page is not None
+
         self._chat_history.clear()
-        await self._visit_page(self.start_page)
+        reset_prior_metadata, reset_last_download = await self._playwright_controller.visit_page(
+            self._page, self.start_page
+        )
+        if reset_last_download and self._last_download is not None:
+            self._last_download = None
+        if reset_prior_metadata and self._prior_metadata_hash is not None:
+            self._prior_metadata_hash = None
         if self.to_save_screenshots:
             current_timestamp = "_" + int(time.time()).__str__()
             screenshot_png_name = "screenshot" + current_timestamp + ".png"
@@ -202,18 +234,20 @@ class MultimodalWebSurfer(BaseChatAgent):
         launch_args: Dict[str, Any] = {"headless": self.headless}
         if self.browser_channel is not None:
             launch_args["channel"] = self.browser_channel
-        self._playwright = await async_playwright().start()
+        if self._playwright is None:
+            self._playwright = await async_playwright().start()
 
         # Create the context -- are we launching persistent?
-        if self.browser_data_dir is None:
-            browser = await self._playwright.chromium.launch(**launch_args)
-            self._context = await browser.new_context(
-                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0"
-            )
-        else:
-            self._context = await self._playwright.chromium.launch_persistent_context(
-                self.browser_data_dir, **launch_args
-            )
+        if self._context is None:
+            if self.browser_data_dir is None:
+                browser = await self._playwright.chromium.launch(**launch_args)
+                self._context = await browser.new_context(
+                    user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0"
+                )
+            else:
+                self._context = await self._playwright.chromium.launch_persistent_context(
+                    self.browser_data_dir, **launch_args
+                )
 
         # Create the page
         self._context.set_default_timeout(60000)  # One minute
@@ -221,7 +255,8 @@ class MultimodalWebSurfer(BaseChatAgent):
         assert self._page is not None
         # self._page.route(lambda x: True, self._route_handler)
         self._page.on("download", self._download_handler)
-        await self._page.set_viewport_size({"width": VIEWPORT_WIDTH, "height": VIEWPORT_HEIGHT})
+        if self.to_resize_viewport:
+            await self._page.set_viewport_size({"width": VIEWPORT_WIDTH, "height": VIEWPORT_HEIGHT})
         await self._page.add_init_script(
             path=os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js")
         )
@@ -231,10 +266,6 @@ class MultimodalWebSurfer(BaseChatAgent):
         # Prepare the debug directory -- which stores the screenshots generated throughout the process
         await self._set_debug_dir(self.debug_dir)
 
-    async def _sleep(self, duration: Union[int, float]) -> None:
-        assert self._page is not None
-        await self._page.wait_for_timeout(duration * 1000)
-
     async def _set_debug_dir(self, debug_dir: str | None) -> None:
         assert self._page is not None
         self.debug_dir = debug_dir
@@ -275,7 +306,7 @@ class MultimodalWebSurfer(BaseChatAgent):
                 aria_name = re.sub(r"[\n\r]+", " ", rects[r].get("aria_name", "")).strip()
 
                 # What are the actions?
-                actions = ['"click"']
+                actions = ['"click", "hover"']
                 if rects[r]["role"] in ["textbox", "searchbox", "search"]:
                     actions = ['"input_text"']
                 actions_str = "[" + ",".join(actions) + "]"
@@ -289,7 +320,6 @@ class MultimodalWebSurfer(BaseChatAgent):
         message: List[FunctionCall],
         rects: Dict[str, InteractiveRegion],
         tool_names: str,
-        use_ocr: bool = True,
         cancellation_token: Optional[CancellationToken] = None,
     ) -> Tuple[bool, UserContent]:
         name = message[0].name
@@ -311,30 +341,43 @@ class MultimodalWebSurfer(BaseChatAgent):
             action_description = f"I typed '{url}' into the browser address bar."
             # Check if the argument starts with a known protocol
             if url.startswith(("https://", "http://", "file://", "about:")):
-                await self._visit_page(url)
+                reset_prior_metadata, reset_last_download = await self._playwright_controller.visit_page(
+                    self._page, url
+                )
             # If the argument contains a space, treat it as a search query
             elif " " in url:
-                await self._visit_page(f"https://www.bing.com/search?q={quote_plus(url)}&FORM=QBLH")
+                reset_prior_metadata, reset_last_download = await self._playwright_controller.visit_page(
+                    self._page, f"https://www.bing.com/search?q={quote_plus(url)}&FORM=QBLH"
+                )
             # Otherwise, prefix with https://
             else:
-                await self._visit_page("https://" + url)
-
+                reset_prior_metadata, reset_last_download = await self._playwright_controller.visit_page(
+                    self._page, "https://" + url
+                )
+            if reset_last_download and self._last_download is not None:
+                self._last_download = None
+            if reset_prior_metadata and self._prior_metadata_hash is not None:
+                self._prior_metadata_hash = None
         elif name == "history_back":
             action_description = "I clicked the browser back button."
-            await self._back()
+            await self._playwright_controller.back(self._page)
 
         elif name == "web_search":
             query = args.get("query")
             action_description = f"I typed '{query}' into the browser search bar."
-            await self._visit_page(f"https://www.bing.com/search?q={quote_plus(query)}&FORM=QBLH")
-
+            reset_prior_metadata, reset_last_download = await self._playwright_controller.visit_page(
+                self._page, f"https://www.bing.com/search?q={quote_plus(query)}&FORM=QBLH"
+            )
+            if reset_last_download and self._last_download is not None:
+                self._last_download = None
+            if reset_prior_metadata and self._prior_metadata_hash is not None:
+                self._prior_metadata_hash = None
         elif name == "page_up":
             action_description = "I scrolled up one page in the browser."
-            await self._page_up()
-
+            await self._playwright_controller.page_up(self._page)
         elif name == "page_down":
             action_description = "I scrolled down one page in the browser."
-            await self._page_down()
+            await self._playwright_controller.page_down(self._page)
 
         elif name == "click":
             target_id = str(args.get("target_id"))
@@ -343,8 +386,17 @@ class MultimodalWebSurfer(BaseChatAgent):
                 action_description = f"I clicked '{target_name}'."
             else:
                 action_description = "I clicked the control."
-            await self._click_id(target_id)
-
+            new_page_tentative = await self._playwright_controller.click_id(self._page, target_id)
+            if new_page_tentative is not None:
+                self._page = new_page_tentative
+                self._prior_metadata_hash = None
+                self.logger.info(
+                    WebSurferEvent(
+                        source=self.name,
+                        url=self._page.url,
+                        message="New tab or window.",
+                    )
+                )
         elif name == "input_text":
             input_field_id = str(args.get("input_field_id"))
             text_value = str(args.get("text_value"))
@@ -353,7 +405,7 @@ class MultimodalWebSurfer(BaseChatAgent):
                 action_description = f"I typed '{text_value}' into '{input_field_name}'."
             else:
                 action_description = f"I input '{text_value}'."
-            await self._fill_id(input_field_id, text_value)
+            await self._playwright_controller.fill_id(self._page, input_field_id, text_value)
 
         elif name == "scroll_element_up":
             target_id = str(args.get("target_id"))
@@ -364,7 +416,7 @@ class MultimodalWebSurfer(BaseChatAgent):
             else:
                 action_description = "I scrolled the control up."
 
-            await self._scroll_id(target_id, "up")
+            await self._playwright_controller.scroll_id(self._page, target_id, "up")
 
         elif name == "scroll_element_down":
             target_id = str(args.get("target_id"))
@@ -375,17 +427,36 @@ class MultimodalWebSurfer(BaseChatAgent):
             else:
                 action_description = "I scrolled the control down."
 
-            await self._scroll_id(target_id, "down")
+            await self._playwright_controller.scroll_id(self._page, target_id, "down")
+
+        elif name == "answer_question":
+            question = str(args.get("question"))
+            action_description = f"I answered the following question '{question}' based on the web page."
+            # Do Q&A on the DOM. No need to take further action. Browser state does not change.
+            return False, await self._summarize_page(question=question, cancellation_token=cancellation_token)
+        elif name == "summarize_page":
+            # Summarize the DOM. No need to take further action. Browser state does not change.
+            action_description = "I summarized the current web page"
+            return False, await self._summarize_page(cancellation_token=cancellation_token)
+
+        elif name == "hover":
+            target_id = str(args.get("target_id"))
+            target_name = self._target_name(target_id, rects)
+            if target_name:
+                action_description = f"I hovered over '{target_name}'."
+            else:
+                action_description = "I hovered over the control."
+            await self._playwright_controller.hover_id(self._page, target_id)
 
         elif name == "sleep":
             action_description = "I am waiting a short period of time before taking further action."
-            await self._sleep(3)  # There's a 2s sleep below too
+            await self._playwright_controller.sleep(self._page, 3)  # There's a 2s sleep below too
 
         else:
             raise ValueError(f"Unknown tool '{name}'. Please choose from:\n\n{tool_names}")
 
         await self._page.wait_for_load_state()
-        await self._sleep(3)
+        await self._playwright_controller.sleep(self._page, 3)  # There's a 2s sleep below too
 
         # Handle downloads
         if self._last_download is not None and self.downloads_folder is not None:
@@ -399,7 +470,7 @@ class MultimodalWebSurfer(BaseChatAgent):
             await self._page.wait_for_load_state()
 
         # Handle metadata
-        page_metadata = json.dumps(await self._get_page_metadata(), indent=4)
+        page_metadata = json.dumps(await self._playwright_controller.get_page_metadata(self._page), indent=4)
         metadata_hash = hashlib.md5(page_metadata.encode("utf-8")).hexdigest()
         if metadata_hash != self._prior_metadata_hash:
             page_metadata = (
@@ -410,7 +481,7 @@ class MultimodalWebSurfer(BaseChatAgent):
         self._prior_metadata_hash = metadata_hash
 
         # Describe the viewport of the new page in words
-        viewport = await self._get_visual_viewport()
+        viewport = await self._playwright_controller.get_visual_viewport(self._page)
         percent_visible = int(viewport["height"] * 100 / viewport["scrollHeight"])
         percent_scrolled = int(viewport["pageTop"] * 100 / viewport["scrollHeight"])
         if percent_scrolled < 1:  # Allow some rounding error
@@ -435,7 +506,9 @@ class MultimodalWebSurfer(BaseChatAgent):
             )
 
         ocr_text = (
-            await self._get_ocr_text(new_screenshot, cancellation_token=cancellation_token) if use_ocr is True else ""
+            await self._get_ocr_text(new_screenshot, cancellation_token=cancellation_token)
+            if self.use_ocr is True
+            else await self._playwright_controller.get_webpage_text(self._page)
         )
 
         # Return the complete observation
@@ -451,7 +524,7 @@ class MultimodalWebSurfer(BaseChatAgent):
         """Generates the actual reply. First calls the LLM to figure out which tool to use, then executes the tool."""
 
         # Lazy init
-        if self._playwright is None:
+        if not self.did_lazy_init:
             await self._lazy_init()
 
         assert self._page is not None
@@ -471,8 +544,8 @@ class MultimodalWebSurfer(BaseChatAgent):
                     history.append(SystemMessage(content=content))
 
         # Ask the page for interactive elements, then prepare the state-of-mark screenshot
-        rects = await self._get_interactive_rects()
-        viewport = await self._get_visual_viewport()
+        rects = await self._playwright_controller.get_interactive_rects(self._page)
+        viewport = await self._playwright_controller.get_visual_viewport(self._page)
         screenshot = await self._page.screenshot()
         som_screenshot, visible_rects, rects_above, rects_below = add_set_of_mark(screenshot, rects)
 
@@ -488,18 +561,8 @@ class MultimodalWebSurfer(BaseChatAgent):
                 )
             )
         # What tools are available?
-        tools = [
-            TOOL_VISIT_URL,
-            TOOL_HISTORY_BACK,
-            TOOL_CLICK,
-            TOOL_TYPE,
-            TOOL_SUMMARIZE_PAGE,
-            TOOL_READ_PAGE_AND_ANSWER,
-            TOOL_SLEEP,
-        ]
+        tools = self.default_tools.copy()
 
-        # Can we reach Bing to search?
-        # if self._navigation_allow_list("https://www.bing.com/"):
         tools.append(TOOL_WEB_SEARCH)
 
         # We can scroll up
@@ -511,7 +574,7 @@ class MultimodalWebSurfer(BaseChatAgent):
             tools.append(TOOL_PAGE_DOWN)
 
         # Focus hint
-        focused = await self._get_focused_rect_id()
+        focused = await self._playwright_controller.get_focused_rect_id(self._page)
         focused_hint = ""
         if focused:
             name = self._target_name(focused, rects)
@@ -549,18 +612,13 @@ class MultimodalWebSurfer(BaseChatAgent):
 
         tool_names = "\n".join([t["name"] for t in tools])
 
-        text_prompt = f"""
-Consider the following screenshot of a web browser, which is open to the page '{self._page.url}'. In this screenshot, interactive elements are outlined in bounding boxes of different colors. Each bounding box has a numeric ID label in the same color. Additional information about each visible label is listed below:
-
-{visible_targets}{other_targets_str}{focused_hint}You are to respond to the user's most recent request by selecting an appropriate tool the following set, or by answering the question directly if possible:
-
-{tool_names}
-
-When deciding between tools, consider if the request can be best addressed by:
-    - the contents of the current viewport (in which case actions like clicking links, clicking buttons, or inputting text might be most appropriate)
-    - contents found elsewhere on the full webpage (in which case actions like scrolling, summarization, or full-page Q&A might be most appropriate)
-    - on some other website entirely (in which case actions like performing a new web search might be the best option)
-""".strip()
+        text_prompt = WEB_SURFER_TOOL_PROMPT.format(
+            url=self._page.url,
+            visible_targets=visible_targets,
+            other_targets_str=other_targets_str,
+            focused_hint=focused_hint,
+            tool_names=tool_names,
+        ).strip()
 
         # Scale the screenshot for the MLM, and close the original
         scaled_screenshot = som_screenshot.resize((MLM_WIDTH, MLM_HEIGHT))
@@ -574,7 +632,6 @@ When deciding between tools, consider if the request can be best addressed by:
             history, tools=tools, extra_create_args={"tool_choice": "auto"}, cancellation_token=cancellation_token
         )  # , "parallel_tool_calls": False})
         message = response.content
-
         self._last_download = None
 
         if isinstance(message, str):
@@ -587,181 +644,6 @@ When deciding between tools, consider if the request can be best addressed by:
             # Not sure what happened here
             raise AssertionError(f"Unknown response format '{message}'")
 
-    async def _get_interactive_rects(self) -> Dict[str, InteractiveRegion]:
-        assert self._page is not None
-
-        # Read the regions from the DOM
-        try:
-            await self._page.evaluate(self._page_script)
-        except Exception:
-            pass
-        result = cast(
-            Dict[str, Dict[str, Any]], await self._page.evaluate("MultimodalWebSurfer.getInteractiveRects();")
-        )
-
-        # Convert the results into appropriate types
-        assert isinstance(result, dict)
-        typed_results: Dict[str, InteractiveRegion] = {}
-        for k in result:
-            assert isinstance(k, str)
-            typed_results[k] = interactiveregion_from_dict(result[k])
-
-        return typed_results
-
-    async def _get_visual_viewport(self) -> VisualViewport:
-        assert self._page is not None
-        try:
-            await self._page.evaluate(self._page_script)
-        except Exception:
-            pass
-        return visualviewport_from_dict(await self._page.evaluate("MultimodalWebSurfer.getVisualViewport();"))
-
-    async def _get_focused_rect_id(self) -> str:
-        assert self._page is not None
-        try:
-            await self._page.evaluate(self._page_script)
-        except Exception:
-            pass
-        result = await self._page.evaluate("MultimodalWebSurfer.getFocusedElementId();")
-        return str(result)
-
-    async def _get_page_metadata(self) -> Dict[str, Any]:
-        assert self._page is not None
-        try:
-            await self._page.evaluate(self._page_script)
-        except Exception:
-            pass
-        result = await self._page.evaluate("MultimodalWebSurfer.getPageMetadata();")
-        assert isinstance(result, dict)
-        return cast(Dict[str, Any], result)
-
-    async def _on_new_page(self, page: Page) -> None:
-        self._page = page
-        assert self._page is not None
-        # self._page.route(lambda x: True, self._route_handler)
-        self._page.on("download", self._download_handler)
-        await self._page.set_viewport_size({"width": VIEWPORT_WIDTH, "height": VIEWPORT_HEIGHT})
-        await self._sleep(0.2)
-        self._prior_metadata_hash = None
-        await self._page.add_init_script(
-            path=os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js")
-        )
-        await self._page.wait_for_load_state()
-
-    async def _back(self) -> None:
-        assert self._page is not None
-        await self._page.go_back()
-
-    async def _visit_page(self, url: str) -> None:
-        assert self._page is not None
-        try:
-            # Regular webpage
-            await self._page.goto(url)
-            await self._page.wait_for_load_state()
-            self._prior_metadata_hash = None
-        except Exception as e_outer:
-            # Downloaded file
-            if self.downloads_folder and "net::ERR_ABORTED" in str(e_outer):
-                async with self._page.expect_download() as download_info:
-                    try:
-                        await self._page.goto(url)
-                    except Exception as e_inner:
-                        if "net::ERR_ABORTED" in str(e_inner):
-                            pass
-                        else:
-                            raise e_inner
-                    download = await download_info.value
-                    fname = os.path.join(self.downloads_folder, download.suggested_filename)
-                    await download.save_as(fname)
-                    message = f"<body style=\"margin: 20px;\"><h1>Successfully downloaded '{download.suggested_filename}' to local path:<br><br>{fname}</h1></body>"
-                    await self._page.goto(
-                        "data:text/html;base64," + base64.b64encode(message.encode("utf-8")).decode("utf-8")
-                    )
-                    self._last_download = None  # Since we already handled it
-            else:
-                raise e_outer
-
-    async def _page_down(self) -> None:
-        assert self._page is not None
-        await self._page.evaluate(f"window.scrollBy(0, {VIEWPORT_HEIGHT-50});")
-
-    async def _page_up(self) -> None:
-        assert self._page is not None
-        await self._page.evaluate(f"window.scrollBy(0, -{VIEWPORT_HEIGHT-50});")
-
-    async def _click_id(self, identifier: str) -> None:
-        assert self._page is not None
-        target = self._page.locator(f"[__elementId='{identifier}']")
-
-        # See if it exists
-        try:
-            await target.wait_for(timeout=100)
-        except TimeoutError:
-            raise ValueError("No such element.") from None
-
-        # Click it
-        await target.scroll_into_view_if_needed()
-        box = cast(Dict[str, Union[int, float]], await target.bounding_box())
-        try:
-            # Give it a chance to open a new page
-            # TODO: Having trouble with these types
-            async with self._page.expect_event("popup", timeout=1000) as page_info:  # type: ignore
-                await self._page.mouse.click(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2, delay=10)
-                # If we got this far without error, than a popup or new tab opened. Handle it.
-
-                new_page = await page_info.value  # type: ignore
-
-                assert isinstance(new_page, Page)
-                await self._on_new_page(new_page)
-
-                self.logger.info(
-                    WebSurferEvent(
-                        source=self.name,
-                        url=self._page.url,
-                        message="New tab or window.",
-                    )
-                )
-
-        except TimeoutError:
-            pass
-
-    async def _fill_id(self, identifier: str, value: str) -> None:
-        assert self._page is not None
-        target = self._page.locator(f"[__elementId='{identifier}']")
-
-        # See if it exists
-        try:
-            await target.wait_for(timeout=100)
-        except TimeoutError:
-            raise ValueError("No such element.") from None
-
-        # Fill it
-        await target.scroll_into_view_if_needed()
-        await target.focus()
-        try:
-            await target.fill(value)
-        except PlaywrightError:
-            await target.press_sequentially(value)
-        await target.press("Enter")
-
-    async def _scroll_id(self, identifier: str, direction: str) -> None:
-        assert self._page is not None
-        await self._page.evaluate(
-            f"""
-        (function() {{
-            let elm = document.querySelector("[__elementId='{identifier}']");
-            if (elm) {{
-                if ("{direction}" == "up") {{
-                    elm.scrollTop = Math.max(0, elm.scrollTop - elm.clientHeight);
-                }}
-                else {{
-                    elm.scrollTop = Math.min(elm.scrollHeight - elm.clientHeight, elm.scrollTop + elm.clientHeight);
-                }}
-            }}
-        }})();
-    """
-        )
-
     async def _get_ocr_text(
         self, image: bytes | io.BufferedIOBase | PIL.Image.Image, cancellation_token: Optional[CancellationToken] = None
     ) -> str:
@@ -783,7 +665,7 @@ When deciding between tools, consider if the request can be best addressed by:
         messages.append(
             UserMessage(
                 content=[
-                    "Please transcribe all visible text on this page, including both main content and the labels of UI elements.",
+                    WEB_SURFER_OCR_PROMPT,
                     AGImage.from_pil(scaled_screenshot),
                 ],
                 source=self.name,
@@ -793,3 +675,68 @@ When deciding between tools, consider if the request can be best addressed by:
         scaled_screenshot.close()
         assert isinstance(response.content, str)
         return response.content
+
+    async def _summarize_page(
+        self,
+        question: str | None = None,
+        cancellation_token: Optional[CancellationToken] = None,
+    ) -> str:
+        assert self._page is not None
+
+        page_markdown: str = await self._playwright_controller.get_page_markdown(self._page)
+
+        title: str = self._page.url
+        try:
+            title = await self._page.title()
+        except Exception:
+            pass
+
+        # Take a screenshot and scale it
+        screenshot = Image.open(io.BytesIO(await self._page.screenshot()))
+        scaled_screenshot = screenshot.resize((MLM_WIDTH, MLM_HEIGHT))
+        screenshot.close()
+        ag_image = AGImage.from_pil(scaled_screenshot)
+
+        # Prepare the system prompt
+        messages: List[LLMMessage] = []
+        messages.append(SystemMessage(content=WEB_SURFER_QA_SYSTEM_MESSAGE))
+        prompt = WEB_SURFER_QA_PROMPT(title, question)
+        # Grow the buffer (which is added to the prompt) until we overflow the context window or run out of lines
+        buffer = ""
+        # for line in re.split(r"([\r\n]+)", page_markdown):
+        for line in page_markdown.splitlines():
+            message = UserMessage(
+                # content=[
+                prompt + buffer + line,
+                #    ag_image,
+                # ],
+                source=self.name,
+            )
+
+            remaining = self._model_client.remaining_tokens(messages + [message])
+            if remaining > SCREENSHOT_TOKENS:
+                buffer += line
+            else:
+                break
+
+        # Nothing to do
+        buffer = buffer.strip()
+        if len(buffer) == 0:
+            return "Nothing to summarize."
+
+        # Append the message
+        messages.append(
+            UserMessage(
+                content=[
+                    prompt + buffer,
+                    ag_image,
+                ],
+                source=self.name,
+            )
+        )
+
+        # Generate the response
+        response = await self._model_client.create(messages, cancellation_token=cancellation_token)
+        scaled_screenshot.close()
+        assert isinstance(response.content, str)
+        return response.content

python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_playwright_controller.py

@@ -0,0 +1,380 @@
+import asyncio
+import base64
+import os
+import random
+from typing import Any, Callable, Dict, Optional, Tuple, Union, cast
+
+from playwright._impl._errors import Error as PlaywrightError
+from playwright._impl._errors import TimeoutError
+from playwright.async_api import Download, Page
+
+from ._types import (
+    InteractiveRegion,
+    VisualViewport,
+    interactiveregion_from_dict,
+    visualviewport_from_dict,
+)
+
+
+class PlaywrightController:
+    def __init__(
+        self,
+        animate_actions: bool = False,
+        downloads_folder: Optional[str] = None,
+        viewport_width: int = 1440,
+        viewport_height: int = 900,
+        _download_handler: Optional[Callable[[Download], None]] = None,
+        to_resize_viewport: bool = True,
+    ) -> None:
+        """
+        A controller for Playwright to interact with web pages.
+        animate_actions: If True, actions will be animated.
+        downloads_folder: The folder to save downloads to.
+        viewport_width: The width of the viewport.
+        viewport_height: The height of the viewport.
+        _download_handler: A handler for downloads.
+        to_resize_viewport: If True, the viewport will be resized.
+        """
+        self.animate_actions = animate_actions
+        self.downloads_folder = downloads_folder
+        self.viewport_width = viewport_width
+        self.viewport_height = viewport_height
+        self._download_handler = _download_handler
+        self.to_resize_viewport = to_resize_viewport
+        self._page_script: str = ""
+        self.last_cursor_position: Tuple[float, float] = (0.0, 0.0)
+
+        # Read page_script
+        with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js"), "rt") as fh:
+            self._page_script = fh.read()
+
+    async def sleep(self, page: Page, duration: Union[int, float]) -> None:
+        assert page is not None
+        await page.wait_for_timeout(duration * 1000)
+
+    async def get_interactive_rects(self, page: Page) -> Dict[str, InteractiveRegion]:
+        assert page is not None
+        # Read the regions from the DOM
+        try:
+            await page.evaluate(self._page_script)
+        except Exception:
+            pass
+        result = cast(Dict[str, Dict[str, Any]], await page.evaluate("MultimodalWebSurfer.getInteractiveRects();"))
+
+        # Convert the results into appropriate types
+        assert isinstance(result, dict)
+        typed_results: Dict[str, InteractiveRegion] = {}
+        for k in result:
+            assert isinstance(k, str)
+            typed_results[k] = interactiveregion_from_dict(result[k])
+
+        return typed_results
+
+    async def get_visual_viewport(self, page: Page) -> VisualViewport:
+        assert page is not None
+        try:
+            await page.evaluate(self._page_script)
+        except Exception:
+            pass
+        return visualviewport_from_dict(await page.evaluate("MultimodalWebSurfer.getVisualViewport();"))
+
+    async def get_focused_rect_id(self, page: Page) -> str:
+        assert page is not None
+        try:
+            await page.evaluate(self._page_script)
+        except Exception:
+            pass
+        result = await page.evaluate("MultimodalWebSurfer.getFocusedElementId();")
+        return str(result)
+
+    async def get_page_metadata(self, page: Page) -> Dict[str, Any]:
+        assert page is not None
+        try:
+            await page.evaluate(self._page_script)
+        except Exception:
+            pass
+        result = await page.evaluate("MultimodalWebSurfer.getPageMetadata();")
+        assert isinstance(result, dict)
+        return cast(Dict[str, Any], result)
+
+    async def on_new_page(self, page: Page) -> None:
+        assert page is not None
+        page.on("download", self._download_handler)  # type: ignore
+        if self.to_resize_viewport and self.viewport_width and self.viewport_height:
+            await page.set_viewport_size({"width": self.viewport_width, "height": self.viewport_height})
+        await self.sleep(page, 0.2)
+        await page.add_init_script(path=os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js"))
+        await page.wait_for_load_state()
+
+    async def back(self, page: Page) -> None:
+        assert page is not None
+        await page.go_back()
+
+    async def visit_page(self, page: Page, url: str) -> Tuple[bool, bool]:
+        assert page is not None
+        reset_prior_metadata_hash = False
+        reset_last_download = False
+        try:
+            # Regular webpage
+            await page.goto(url)
+            await page.wait_for_load_state()
+            reset_prior_metadata_hash = True
+        except Exception as e_outer:
+            # Downloaded file
+            if self.downloads_folder and "net::ERR_ABORTED" in str(e_outer):
+                async with page.expect_download() as download_info:
+                    try:
+                        await page.goto(url)
+                    except Exception as e_inner:
+                        if "net::ERR_ABORTED" in str(e_inner):
+                            pass
+                        else:
+                            raise e_inner
+                    download = await download_info.value
+                    fname = os.path.join(self.downloads_folder, download.suggested_filename)
+                    await download.save_as(fname)
+                    message = f"<body style=\"margin: 20px;\"><h1>Successfully downloaded '{download.suggested_filename}' to local path:<br><br>{fname}</h1></body>"
+                    await page.goto(
+                        "data:text/html;base64," + base64.b64encode(message.encode("utf-8")).decode("utf-8")
+                    )
+                    reset_last_download = True
+            else:
+                raise e_outer
+        return reset_prior_metadata_hash, reset_last_download
+
+    async def page_down(self, page: Page) -> None:
+        assert page is not None
+        await page.evaluate(f"window.scrollBy(0, {self.viewport_height-50});")
+
+    async def page_up(self, page: Page) -> None:
+        assert page is not None
+        await page.evaluate(f"window.scrollBy(0, -{self.viewport_height-50});")
+
+    async def gradual_cursor_animation(
+        self, page: Page, start_x: float, start_y: float, end_x: float, end_y: float
+    ) -> None:
+        # animation helper
+        steps = 20
+        for step in range(steps):
+            x = start_x + (end_x - start_x) * (step / steps)
+            y = start_y + (end_y - start_y) * (step / steps)
+            # await page.mouse.move(x, y, steps=1)
+            await page.evaluate(f"""
+                (function() {{
+                    let cursor = document.getElementById('red-cursor');
+                    cursor.style.left = '{x}px';
+                    cursor.style.top = '{y}px';
+                }})();
+            """)
+            await asyncio.sleep(0.05)
+
+        self.last_cursor_position = (end_x, end_y)
+
+    async def add_cursor_box(self, page: Page, identifier: str) -> None:
+        # animation helper
+        await page.evaluate(f"""
+            (function() {{
+                let elm = document.querySelector("[__elementId='{identifier}']");
+                if (elm) {{
+                    elm.style.transition = 'border 0.3s ease-in-out';
+                    elm.style.border = '2px solid red';
+                }}
+            }})();
+        """)
+        await asyncio.sleep(0.3)
+
+        # Create a red cursor
+        await page.evaluate("""
+            (function() {
+                let cursor = document.createElement('div');
+                cursor.id = 'red-cursor';
+                cursor.style.width = '10px';
+                cursor.style.height = '10px';
+                cursor.style.backgroundColor = 'red';
+                cursor.style.position = 'absolute';
+                cursor.style.borderRadius = '50%';
+                cursor.style.zIndex = '10000';
+                document.body.appendChild(cursor);
+            })();
+        """)
+
+    async def remove_cursor_box(self, page: Page, identifier: str) -> None:
+        # Remove the highlight and cursor
+        await page.evaluate(f"""
+            (function() {{
+                let elm = document.querySelector("[__elementId='{identifier}']");
+                if (elm) {{
+                    elm.style.border = '';
+                }}
+                let cursor = document.getElementById('red-cursor');
+                if (cursor) {{
+                    cursor.remove();
+                }}
+            }})();
+        """)
+
+    async def click_id(self, page: Page, identifier: str) -> Page | None:
+        """
+        Returns new page if a new page is opened, otherwise None.
+        """
+        new_page: Page | None = None
+        assert page is not None
+        target = page.locator(f"[__elementId='{identifier}']")
+
+        # See if it exists
+        try:
+            await target.wait_for(timeout=5000)
+        except TimeoutError:
+            raise ValueError("No such element.") from None
+
+        # Click it
+        await target.scroll_into_view_if_needed()
+        await asyncio.sleep(0.3)
+
+        box = cast(Dict[str, Union[int, float]], await target.bounding_box())
+
+        if self.animate_actions:
+            await self.add_cursor_box(page, identifier)
+            # Move cursor to the box slowly
+            start_x, start_y = self.last_cursor_position
+            end_x, end_y = box["x"] + box["width"] / 2, box["y"] + box["height"] / 2
+            await self.gradual_cursor_animation(page, start_x, start_y, end_x, end_y)
+            await asyncio.sleep(0.1)
+
+            try:
+                # Give it a chance to open a new page
+                async with page.expect_event("popup", timeout=1000) as page_info:  # type: ignore
+                    await page.mouse.click(end_x, end_y, delay=10)
+                    new_page = await page_info.value  # type: ignore
+                    assert isinstance(new_page, Page)
+                    await self.on_new_page(new_page)
+            except TimeoutError:
+                pass
+            await self.remove_cursor_box(page, identifier)
+
+        else:
+            try:
+                # Give it a chance to open a new page
+                async with page.expect_event("popup", timeout=1000) as page_info:  # type: ignore
+                    await page.mouse.click(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2, delay=10)
+                    new_page = await page_info.value  # type: ignore
+                    assert isinstance(new_page, Page)
+                    await self.on_new_page(new_page)
+            except TimeoutError:
+                pass
+        return new_page  # type: ignore
+
+    async def hover_id(self, page: Page, identifier: str) -> None:
+        """
+        Hovers the mouse over the target with the given id.
+        """
+        assert page is not None
+        target = page.locator(f"[__elementId='{identifier}']")
+
+        # See if it exists
+        try:
+            await target.wait_for(timeout=5000)
+        except TimeoutError:
+            raise ValueError("No such element.") from None
+
+        # Hover over it
+        await target.scroll_into_view_if_needed()
+        await asyncio.sleep(0.3)
+
+        box = cast(Dict[str, Union[int, float]], await target.bounding_box())
+
+        if self.animate_actions:
+            await self.add_cursor_box(page, identifier)
+            # Move cursor to the box slowly
+            start_x, start_y = self.last_cursor_position
+            end_x, end_y = box["x"] + box["width"] / 2, box["y"] + box["height"] / 2
+            await self.gradual_cursor_animation(page, start_x, start_y, end_x, end_y)
+            await asyncio.sleep(0.1)
+            await page.mouse.move(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2)
+
+            await self.remove_cursor_box(page, identifier)
+        else:
+            await page.mouse.move(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2)
+
+    async def fill_id(self, page: Page, identifier: str, value: str) -> None:
+        assert page is not None
+        target = page.locator(f"[__elementId='{identifier}']")
+
+        # See if it exists
+        try:
+            await target.wait_for(timeout=5000)
+        except TimeoutError:
+            raise ValueError("No such element.") from None
+
+        # Fill it
+        await target.scroll_into_view_if_needed()
+        box = cast(Dict[str, Union[int, float]], await target.bounding_box())
+
+        if self.animate_actions:
+            await self.add_cursor_box(page, identifier)
+            # Move cursor to the box slowly
+            start_x, start_y = self.last_cursor_position
+            end_x, end_y = box["x"] + box["width"] / 2, box["y"] + box["height"] / 2
+            await self.gradual_cursor_animation(page, start_x, start_y, end_x, end_y)
+            await asyncio.sleep(0.1)
+
+        # Focus on the element
+        await target.focus()
+        if self.animate_actions:
+            # fill char by char to mimic human speed for short text and type fast for long text
+            if len(value) < 100:
+                delay_typing_speed = 50 + 100 * random.random()
+            else:
+                delay_typing_speed = 10
+            await target.press_sequentially(value, delay=delay_typing_speed)
+        else:
+            try:
+                await target.fill(value)
+            except PlaywrightError:
+                await target.press_sequentially(value)
+        await target.press("Enter")
+
+        if self.animate_actions:
+            await self.remove_cursor_box(page, identifier)
+
+    async def scroll_id(self, page: Page, identifier: str, direction: str) -> None:
+        assert page is not None
+        await page.evaluate(
+            f"""
+        (function() {{
+            let elm = document.querySelector("[__elementId='{identifier}']");
+            if (elm) {{
+                if ("{direction}" == "up") {{
+                    elm.scrollTop = Math.max(0, elm.scrollTop - elm.clientHeight);
+                }}
+                else {{
+                    elm.scrollTop = Math.min(elm.scrollHeight - elm.clientHeight, elm.scrollTop + elm.clientHeight);
+                }}
+            }}
+        }})();
+    """
+        )
+
+    async def get_webpage_text(self, page: Page, n_lines: int = 100) -> str:
+        """
+        page: playwright page object
+        n_lines: number of lines to return from the page innertext
+        return: text in the first n_lines of the page
+        """
+        assert page is not None
+        try:
+            text_in_viewport = await page.evaluate("""() => {
+                return document.body.innerText;
+            }""")
+            text_in_viewport = "\n".join(text_in_viewport.split("\n")[:n_lines])
+            # remove empty lines
+            text_in_viewport = "\n".join([line for line in text_in_viewport.split("\n") if line.strip()])
+            assert isinstance(text_in_viewport, str)
+            return text_in_viewport
+        except Exception:
+            return ""
+
+    async def get_page_markdown(self, page: Page) -> str:
+        # TODO: replace with mdconvert
+        assert page is not None
+        return await self.get_webpage_text(page, n_lines=1000)

python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py

@@ -0,0 +1,32 @@
+WEB_SURFER_TOOL_PROMPT = """
+Consider the following screenshot of a web browser, which is open to the page '{url}'. In this screenshot, interactive elements are outlined in bounding boxes of different colors. Each bounding box has a numeric ID label in the same color. Additional information about each visible label is listed below:
+
+{visible_targets}{other_targets_str}{focused_hint}
+
+You are to respond to the most recent request by selecting an appropriate tool from the following set, or by answering the question directly if possible without tools:
+
+{tool_names}
+
+When deciding between tools, consider if the request can be best addressed by:
+    - the contents of the current viewport (in which case actions like clicking links, clicking buttons, inputting text might be most appropriate, or hovering over element)
+    - contents found elsewhere on the full webpage (in which case actions like scrolling, summarization, or full-page Q&A might be most appropriate)
+    - on some other website entirely (in which case actions like performing a new web search might be the best option)
+"""
+
+WEB_SURFER_OCR_PROMPT = """
+Please transcribe all visible text on this page, including both main content and the labels of UI elements.
+"""
+
+WEB_SURFER_QA_SYSTEM_MESSAGE = """
+You are a helpful assistant that can summarize long documents to answer question.
+"""
+
+
+def WEB_SURFER_QA_PROMPT(title: str, question: str | None = None) -> str:
+    base_prompt = f"We are visiting the webpage '{title}'. Its full-text content are pasted below, along with a screenshot of the page's current viewport."
+    if question is not None:
+        return (
+            f"{base_prompt} Please summarize the webpage into one or two paragraphs with respect to '{question}':\n\n"
+        )
+    else:
+        return f"{base_prompt} Please summarize the webpage into one or two paragraphs:\n\n"

python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.py

@@ -1,6 +1,5 @@
 from typing import Any, Dict
 
-# TODO Why does pylance fail if I import from autogen_core.components.tools instead?
 from autogen_core.components.tools._base import ParametersSchema, ToolSchema
 
 
@@ -16,6 +15,10 @@ def _load_tool(tooldef: Dict[str, Any]) -> ToolSchema:
     )
 
 
+REASONING_TOOL_PROMPT = (
+    "A short description of the action to be performed and reason for doing so, do not mention the user."
+)
+
 TOOL_VISIT_URL: ToolSchema = _load_tool(
     {
         "type": "function",
@@ -27,7 +30,7 @@ TOOL_VISIT_URL: ToolSchema = _load_tool(
                 "properties": {
                     "reasoning": {
                         "type": "string",
-                        "description": "A short explanation of the reasoning for calling this tool and taking this action.",
+                        "description": REASONING_TOOL_PROMPT,
                     },
                     "url": {
                         "type": "string",
@@ -51,7 +54,7 @@ TOOL_WEB_SEARCH: ToolSchema = _load_tool(
                 "properties": {
                     "reasoning": {
                         "type": "string",
-                        "description": "A short explanation of the reasoning for calling this tool and taking this action.",
+                        "description": REASONING_TOOL_PROMPT,
                     },
                     "query": {
                         "type": "string",
@@ -75,7 +78,7 @@ TOOL_HISTORY_BACK: ToolSchema = _load_tool(
                 "properties": {
                     "reasoning": {
                         "type": "string",
-                        "description": "A short explanation of the reasoning for calling this tool and taking this action.",
+                        "description": REASONING_TOOL_PROMPT,
                     },
                 },
                 "required": ["reasoning"],
@@ -95,7 +98,7 @@ TOOL_PAGE_UP: ToolSchema = _load_tool(
                 "properties": {
                     "reasoning": {
                         "type": "string",
-                        "description": "A short explanation of the reasoning for calling this tool and taking this action.",
+                        "description": REASONING_TOOL_PROMPT,
                     },
                 },
                 "required": ["reasoning"],
@@ -115,7 +118,7 @@ TOOL_PAGE_DOWN: ToolSchema = _load_tool(
                 "properties": {
                     "reasoning": {
                         "type": "string",
-                        "description": "A short explanation of the reasoning for calling this tool and taking this action.",
+                        "description": REASONING_TOOL_PROMPT,
                     },
                 },
                 "required": ["reasoning"],
@@ -135,7 +138,7 @@ TOOL_CLICK: ToolSchema = _load_tool(
                 "properties": {
                     "reasoning": {
                         "type": "string",
-                        "description": "A short explanation of the reasoning for calling this tool and taking this action.",
+                        "description": REASONING_TOOL_PROMPT,
                     },
                     "target_id": {
                         "type": "integer",
@@ -159,7 +162,7 @@ TOOL_TYPE: ToolSchema = _load_tool(
                 "properties": {
                     "reasoning": {
                         "type": "string",
-                        "description": "A short explanation of the reasoning for calling this tool and taking this action.",
+                        "description": REASONING_TOOL_PROMPT,
                     },
                     "input_field_id": {
                         "type": "integer",
@@ -187,7 +190,7 @@ TOOL_SCROLL_ELEMENT_DOWN: ToolSchema = _load_tool(
                 "properties": {
                     "reasoning": {
                         "type": "string",
-                        "description": "A short explanation of the reasoning for calling this tool and taking this action.",
+                        "description": REASONING_TOOL_PROMPT,
                     },
                     "target_id": {
                         "type": "integer",
@@ -211,7 +214,7 @@ TOOL_SCROLL_ELEMENT_UP: ToolSchema = _load_tool(
                 "properties": {
                     "reasoning": {
                         "type": "string",
-                        "description": "A short explanation of the reasoning for calling this tool and taking this action.",
+                        "description": REASONING_TOOL_PROMPT,
                     },
                     "target_id": {
                         "type": "integer",
@@ -224,6 +227,31 @@ TOOL_SCROLL_ELEMENT_UP: ToolSchema = _load_tool(
     }
 )
 
+TOOL_HOVER: ToolSchema = _load_tool(
+    {
+        "type": "function",
+        "function": {
+            "name": "hover",
+            "description": "Hovers the mouse over the target with the given id.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "reasoning": {
+                        "type": "string",
+                        "description": REASONING_TOOL_PROMPT,
+                    },
+                    "target_id": {
+                        "type": "integer",
+                        "description": "The numeric id of the target to hover over.",
+                    },
+                },
+                "required": ["reasoning", "target_id"],
+            },
+        },
+    }
+)
+
+
 TOOL_READ_PAGE_AND_ANSWER: ToolSchema = _load_tool(
     {
         "type": "function",
@@ -235,7 +263,7 @@ TOOL_READ_PAGE_AND_ANSWER: ToolSchema = _load_tool(
                 "properties": {
                     "reasoning": {
                         "type": "string",
-                        "description": "A short explanation of the reasoning for calling this tool and taking this action.",
+                        "description": REASONING_TOOL_PROMPT,
                     },
                     "question": {
                         "type": "string",
@@ -259,7 +287,7 @@ TOOL_SUMMARIZE_PAGE: ToolSchema = _load_tool(
                 "properties": {
                     "reasoning": {
                         "type": "string",
-                        "description": "A short explanation of the reasoning for calling this tool and taking this action.",
+                        "description": REASONING_TOOL_PROMPT,
                     },
                 },
                 "required": ["reasoning"],
@@ -279,7 +307,7 @@ TOOL_SLEEP: ToolSchema = _load_tool(
                 "properties": {
                     "reasoning": {
                         "type": "string",
-                        "description": "A short explanation of the reasoning for calling this tool and taking this action.",
+                        "description": REASONING_TOOL_PROMPT,
                     },
                 },
                 "required": ["reasoning"],