Complex tasks file support (#1603)

* Add initial example

* Add decorator for yt utils

* Rename submodule; Add ability to specify package version

* Add support for specifying pip package name

* Fix bugs in requires

* Add more file utils; improve nb

* Add a new template that uses two agents + function calling

* Improve decorator name

* Add decorator for secrets

autogen/agentchat/contrib/functions/file_utils.py

@@ -1,7 +1,7 @@
-from .functions_utils import requires
+from .functions_utils import requires_python_packages, requires_secret
 
 
-@requires("pdfminer.six", "requests")
+@requires_python_packages("pdfminer.six", "requests")
 def read_text_from_pdf(file_path: str) -> str:
     """
     Reads text from a PDF file and returns it as a string.
@@ -39,7 +39,7 @@ def read_text_from_pdf(file_path: str) -> str:
     return text
 
 
-@requires("python-docx")
+@requires_python_packages("python-docx")
 def read_text_from_docx(file_path: str) -> str:
     """
     Reads text from a DOCX file and returns it as a string.
@@ -59,7 +59,7 @@ def read_text_from_docx(file_path: str) -> str:
     return text
 
 
-@requires("pillow", "requests", "easyocr")
+@requires_python_packages("pillow", "requests", "easyocr")
 def read_text_from_image(file_path: str) -> str:
     """
     Reads text from an image file or URL and returns it as a string.
@@ -92,7 +92,7 @@ def read_text_from_image(file_path: str) -> str:
     return text
 
 
-@requires("python-pptx")
+@requires_python_packages("python-pptx")
 def read_text_from_pptx(file_path: str) -> str:
     """
     Reads text from a PowerPoint file and returns it as a string.
@@ -116,7 +116,7 @@ def read_text_from_pptx(file_path: str) -> str:
     return text
 
 
-@requires("pandas")
+@requires_python_packages("pandas")
 def read_text_from_xlsx(file_path: str) -> str:
     """
     Reads text from an Excel file and returns it as a string.
@@ -135,7 +135,7 @@ def read_text_from_xlsx(file_path: str) -> str:
     return text
 
 
-@requires("SpeechRecognition", "requests")
+@requires_python_packages("SpeechRecognition", "requests")
 def read_text_from_audio(file_path: str) -> str:
     """
     Reads text from an audio file or a URL and returns it as a string.
@@ -166,3 +166,52 @@ def read_text_from_audio(file_path: str) -> str:
     text = recognizer.recognize_google(audio)
 
     return text
+
+
+@requires_secret("OPENAI_API_KEY")
+@requires_python_packages("openai")
+def caption_image_using_gpt4v(file_path_or_url: str) -> str:
+    """
+    Generates a caption for an image using the GPT-4 Vision model from OpenAI.
+
+    Args:
+        file_path_or_url (str): The path to the image file or the URL.
+
+    Returns:
+        str: The caption generated for the image.
+    """
+    import os
+    import openai
+    from openai import OpenAI
+
+    caption = ""
+
+    openai.api_key = os.environ["OPENAI_API_KEY"]
+    client = OpenAI()
+
+    # check if the file_path_or_url is a URL
+    if file_path_or_url.startswith("http://") or file_path_or_url.startswith("https://"):
+        image_url = file_path_or_url
+
+        response = client.chat.completions.create(
+            model="gpt-4-vision-preview",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "What’s in this image?"},
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": image_url,
+                            },
+                        },
+                    ],
+                }
+            ],
+            max_tokens=300,
+        )
+        caption = response.choices[0]
+    else:
+        caption = "Please provide a valid image URL"
+    return caption

autogen/agentchat/contrib/functions/functions_utils.py

@@ -1,27 +1,28 @@
+import os
 import subprocess
 import sys
 import functools
 import pkg_resources
 
 
-def requires(*packages, **pip_packages):
+def requires_python_packages(*packages, **pip_packages):
     """
-    Decorator that ensures the required packages are installed before executing the decorated function.
+    Decorator that ensures the required Python packages are installed before executing the decorated function.
 
     Args:
-        *packages: Variable length argument list of package names that should be installed.
-        **pip_packages: Keyword arguments specifying package names and versions in the format `package_name=version`.
+        *packages: Variable length argument list of Python package names that should be installed.
+        **pip_packages: Keyword arguments specifying Python package names and versions in the format `package_name=version`.
 
     Examples:
-        @requires('numpy', 'pandas')
+        @requires_python_packages('numpy', 'pandas')
         def my_function():
             # Code that depends on numpy and pandas
 
-        @requires(matplotlib='3.2.1', seaborn='0.11.1')
+        @requires_python_packages(matplotlib='3.2.1', seaborn='0.11.1')
         def another_function():
             # Code that depends on matplotlib version 3.2.1 and seaborn version 0.11.1
 
-        @requires('numpy', 'pandas', matplotlib='3.2.1', PIL='8.1.0')
+        @requires_python_packages('numpy', 'pandas', matplotlib='3.2.1', PIL='8.1.0')
         def yet_another_function():
             # Code that depends on numpy, pandas, matplotlib version 3.2.1, and PIL version 8.1.0
     """
@@ -67,3 +68,33 @@ def requires(*packages, **pip_packages):
         return wrapper
 
     return decorator
+
+
+def requires_secret(*env):
+    """
+    Decorator that ensures the required environment variables are set before executing the decorated function.
+
+    Args:
+        *env: Variable length argument list of environment variable names that should be set.
+
+    Examples:
+        @requires_secret('OPENAI_API_KEY')
+        def my_function():
+            # Code that depends on the OPENAI_API_KEY environment variable
+
+        @requires_secret('AWS', 'AWS_ACCESS')
+        def another_function():
+            # Code that depends on the AWS and AWS_ACCESS environment variables
+    """
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            for name in env:
+                if name not in os.environ:
+                    raise EnvironmentError(f"Environment variable {name} is not set")
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator

autogen/agentchat/contrib/functions/youtube_utils.py

@@ -1,7 +1,7 @@
-from .functions_utils import requires
+from .functions_utils import requires_python_packages
 
 
-@requires("youtube_transcript_api==0.6.0")
+@requires_python_packages("youtube_transcript_api==0.6.0")
 def get_youtube_transcript(youtube_link: str) -> str:
     """
     Gets the transcript of a YouTube video.