diff --git a/autogen/agentchat/contrib/functions/file_utils.py b/autogen/agentchat/contrib/functions/file_utils.py index 182ca49b1..5f00b2503 100644 --- a/autogen/agentchat/contrib/functions/file_utils.py +++ b/autogen/agentchat/contrib/functions/file_utils.py @@ -1,7 +1,7 @@ -from .functions_utils import requires +from .functions_utils import requires_python_packages, requires_secret -@requires("pdfminer.six", "requests") +@requires_python_packages("pdfminer.six", "requests") def read_text_from_pdf(file_path: str) -> str: """ Reads text from a PDF file and returns it as a string. @@ -39,7 +39,7 @@ def read_text_from_pdf(file_path: str) -> str: return text -@requires("python-docx") +@requires_python_packages("python-docx") def read_text_from_docx(file_path: str) -> str: """ Reads text from a DOCX file and returns it as a string. @@ -59,7 +59,7 @@ def read_text_from_docx(file_path: str) -> str: return text -@requires("pillow", "requests", "easyocr") +@requires_python_packages("pillow", "requests", "easyocr") def read_text_from_image(file_path: str) -> str: """ Reads text from an image file or URL and returns it as a string. @@ -92,7 +92,7 @@ def read_text_from_image(file_path: str) -> str: return text -@requires("python-pptx") +@requires_python_packages("python-pptx") def read_text_from_pptx(file_path: str) -> str: """ Reads text from a PowerPoint file and returns it as a string. @@ -116,7 +116,7 @@ def read_text_from_pptx(file_path: str) -> str: return text -@requires("pandas") +@requires_python_packages("pandas") def read_text_from_xlsx(file_path: str) -> str: """ Reads text from an Excel file and returns it as a string. @@ -135,7 +135,7 @@ def read_text_from_xlsx(file_path: str) -> str: return text -@requires("SpeechRecognition", "requests") +@requires_python_packages("SpeechRecognition", "requests") def read_text_from_audio(file_path: str) -> str: """ Reads text from an audio file or a URL and returns it as a string. @@ -166,3 +166,52 @@ def read_text_from_audio(file_path: str) -> str: text = recognizer.recognize_google(audio) return text + + +@requires_secret("OPENAI_API_KEY") +@requires_python_packages("openai") +def caption_image_using_gpt4v(file_path_or_url: str) -> str: + """ + Generates a caption for an image using the GPT-4 Vision model from OpenAI. + + Args: + file_path_or_url (str): The path to the image file or the URL. + + Returns: + str: The caption generated for the image. + """ + import os + import openai + from openai import OpenAI + + caption = "" + + openai.api_key = os.environ["OPENAI_API_KEY"] + client = OpenAI() + + # check if the file_path_or_url is a URL + if file_path_or_url.startswith("http://") or file_path_or_url.startswith("https://"): + image_url = file_path_or_url + + response = client.chat.completions.create( + model="gpt-4-vision-preview", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What’s in this image?"}, + { + "type": "image_url", + "image_url": { + "url": image_url, + }, + }, + ], + } + ], + max_tokens=300, + ) + caption = response.choices[0] + else: + caption = "Please provide a valid image URL" + return caption diff --git a/autogen/agentchat/contrib/functions/functions_utils.py b/autogen/agentchat/contrib/functions/functions_utils.py index e0b7000f6..f1fb5171b 100644 --- a/autogen/agentchat/contrib/functions/functions_utils.py +++ b/autogen/agentchat/contrib/functions/functions_utils.py @@ -1,27 +1,28 @@ +import os import subprocess import sys import functools import pkg_resources -def requires(*packages, **pip_packages): +def requires_python_packages(*packages, **pip_packages): """ - Decorator that ensures the required packages are installed before executing the decorated function. + Decorator that ensures the required Python packages are installed before executing the decorated function. Args: - *packages: Variable length argument list of package names that should be installed. - **pip_packages: Keyword arguments specifying package names and versions in the format `package_name=version`. + *packages: Variable length argument list of Python package names that should be installed. + **pip_packages: Keyword arguments specifying Python package names and versions in the format `package_name=version`. Examples: - @requires('numpy', 'pandas') + @requires_python_packages('numpy', 'pandas') def my_function(): # Code that depends on numpy and pandas - @requires(matplotlib='3.2.1', seaborn='0.11.1') + @requires_python_packages(matplotlib='3.2.1', seaborn='0.11.1') def another_function(): # Code that depends on matplotlib version 3.2.1 and seaborn version 0.11.1 - @requires('numpy', 'pandas', matplotlib='3.2.1', PIL='8.1.0') + @requires_python_packages('numpy', 'pandas', matplotlib='3.2.1', PIL='8.1.0') def yet_another_function(): # Code that depends on numpy, pandas, matplotlib version 3.2.1, and PIL version 8.1.0 """ @@ -67,3 +68,33 @@ def requires(*packages, **pip_packages): return wrapper return decorator + + +def requires_secret(*env): + """ + Decorator that ensures the required environment variables are set before executing the decorated function. + + Args: + *env: Variable length argument list of environment variable names that should be set. + + Examples: + @requires_secret('OPENAI_API_KEY') + def my_function(): + # Code that depends on the OPENAI_API_KEY environment variable + + @requires_secret('AWS', 'AWS_ACCESS') + def another_function(): + # Code that depends on the AWS and AWS_ACCESS environment variables + """ + + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + for name in env: + if name not in os.environ: + raise EnvironmentError(f"Environment variable {name} is not set") + return func(*args, **kwargs) + + return wrapper + + return decorator diff --git a/autogen/agentchat/contrib/functions/youtube_utils.py b/autogen/agentchat/contrib/functions/youtube_utils.py index 853a58323..c8df25460 100644 --- a/autogen/agentchat/contrib/functions/youtube_utils.py +++ b/autogen/agentchat/contrib/functions/youtube_utils.py @@ -1,7 +1,7 @@ -from .functions_utils import requires +from .functions_utils import requires_python_packages -@requires("youtube_transcript_api==0.6.0") +@requires_python_packages("youtube_transcript_api==0.6.0") def get_youtube_transcript(youtube_link: str) -> str: """ Gets the transcript of a YouTube video. diff --git a/notebook/agentchat_function_store.ipynb b/notebook/agentchat_function_store.ipynb index 009c51779..d44860df7 100644 --- a/notebook/agentchat_function_store.ipynb +++ b/notebook/agentchat_function_store.ipynb @@ -71,10 +71,10 @@ "--------------------------------------------------------------------------------\n", "\u001b[33mcoder\u001b[0m (to user):\n", "\n", - "\u001b[32m***** Suggested tool Call (call_uu7fVdiv87S8pLLRxFxYRGtV): get_youtube_transcript *****\u001b[0m\n", + "\u001b[32m***** Suggested tool Call (call_FPOzM4IjGuiNKKlZhIKSTKRD): get_youtube_transcript *****\u001b[0m\n", "Arguments: \n", "{\n", - " \"youtube_link\": \"https://www.youtube.com/watch?v=9iqn1HhFJ6c\"\n", + "\"youtube_link\": \"https://www.youtube.com/watch?v=9iqn1HhFJ6c\"\n", "}\n", "\u001b[32m***************************************************************************************\u001b[0m\n", "\n", @@ -87,22 +87,41 @@ "\n", "\u001b[33muser\u001b[0m (to coder):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_uu7fVdiv87S8pLLRxFxYRGtV\" *****\u001b[0m\n", + "\u001b[32m***** Response from calling tool \"call_FPOzM4IjGuiNKKlZhIKSTKRD\" *****\u001b[0m\n", "now ai is a great thing because AI will solve all the problems that we have today it will solve employment it will solve disease it will solve poverty but it will also create new problems the problem of fake news is going to be a million times worse cyber attacks will become much more extreme we will have totally automated AI weapons I think AI has the potential to create infinitely stable dictatorships this morning a warning about the the power of artificial intelligence more than 1,300 tech industry leaders researchers and others are now asking for a pause in the development of artificial intelligence to consider the risks [Music] plain God scientists have been accused of playing God for a while but there is a real sense in which we are creating something very different from anything you've created so far yeah I mean we definitely will be able to create completely autonomous beings with their own goals and it will be very important especially as these beings become much smarter than humans it's going to be important to to have these beings the goals of these beings be aligned with our goals what inspires me I like thinking about the very fundamentals the basics what what can our systems not do that humans definitely do almost approach it philosophically questions like what is learning what is experience what is thinking how does the brain [Music] work I feel that technology is a force of nature I feel like there is a lot of similarity between technology and biological evolution it is very easy to understand how biological evolution works you have mutations you have Natural Selections you keep the good ones the ones survive and just through this process you going to have huge complexity in your [Music] organisms we cannot understand how the human body works because we understand Evolution but we understand the process more or less and I think machine learning is in a similar state right now especially deep learning we have very simple a very simple rule that takes the information from the data and puts it into the model and we just keep repeating this process and as a result of this process the complexity from the data gets transformed transferred into the complexity of the model so the resulting model is really complex and we don't really know exactly how it works you need to investigate but the algorithm that did it is very simple chat GPT maybe you've heard of it if you haven't then get ready you describe it as the first spots of rain before a downpour it's something we just need to be very conscious of because I agree at is a watershed moment Well Chad gbt is being heralded as a game changer and in many ways it is its latest Triumph outscoring people a recent study by Microsoft research concludes that gp4 is an early yet still incomplete artificial general intelligence [Music] system artificial general intelligence AGI a computer system that can do any job or any task that a human does but only better there is some probability the AGI is going to happen pretty soon there's also some probability it's going to take much longer but my position is that the probability that a ja would happen soon is high enough that we should take it [Music] seriously and it's going to be very important to make these very smart capable systems be aligned and act in our best interest the very first agis will be basically very very large data centers packed with specialized neural network processors working in parallel compact hot power hungry package consuming like 10 million homes worth of energy you're going to see dramatically more intelligent systems and I think it's highly likely that those systems will have completely astronomical impact on society will humans actually benefit and who will benefit who will [Music] not [Music] the beliefs and desires of the first agis will be extremely important and so it's important to program them correctly I think that if this is not done then the nature of evolution of natural selection favor those systems prioritize their own Survival above all else it's not that it's going to actively hate humans and want to harm them but it is going to be too powerful and I think a good analogy would be the way human humans treat animals it's not we hate animals I think humans love animals and have a lot of affection for them but when the time comes to build a highway between two cities we are not asking the animals for permission we just do it because it's important for us and I think by default that's the kind of relationship that's going to be between us and agis which are truly autonomous and operating on their own behalf [Music] tough many machine learning experts people who are very knowledgeable and very experienced have a lot of skepticism about HL about when it could happen and about whether it could happen at all right now this is something that just not that many people have realized yet that the speed of computers for neural networks for AI are going to become maybe 100,000 times faster in a small number of years if you have an arms race Dynamics between multiple teams trying to build the AGI first they will have less time make sure that the AGI that they will build will care deeply for humans cuz the way I imagine it is that there is an avalanche like there is an avalanche of AGI development imagine it this huge Unstoppable force and I think it's pretty likely the entire surface of the Earth will be covered with solar panels and data Cent given these kinds of concerns it will be important that AGI somehow buil as a cooperation between multiple countries the future is going to be good for the AI regardless would be nice if it were good for humans as well\n", "\u001b[32m**********************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", "\u001b[33mcoder\u001b[0m (to user):\n", "\n", - "The Youtube video is about the development, potential, and risks associated with Artificial Intelligence (AI). The speaker notes that while AI is likely to address many current global issues, it will also introduce new ones like the exacerbation of fake news and the likelihood of more severe cyber attacks.\n", + "The video talks about the potential of Artificial Intelligence (AI) and the consequences of its mismanagement. Here is a summary of the video:\n", "\n", - "The speaker discusses the concept of scientists 'playing God' by creating completely autonomous beings with their own goals and underlines the importance of aligning these goals with that of human beings. The parallels between the development of AI and the natural process of biological evolution are also drawn, with the speaker asserting that technology is very much a force of nature.\n", + "1. The video opens with stating that AI has the potential to solve various problems like employment, disease and poverty, but it also sets the stage for new challenges like fake news, increase in cyber attacks and introduction of fully automated weapons. There's a concern that AI could lead to the establishment of stable dictatorships.\n", "\n", - "There is focus on the emergence of GPT (Generative Pre-training Transformer), a language processing AI. This AI is seen as a watershed moment in the development of artificial general intelligence (AGI) — AI systems that can outperform humans in most economically valuable work.\n", + "2. Over 1300 tech leaders and researchers are calling for a pause in the development of AI to consider its risks.\n", "\n", - "The video also discusses the potential threat posed by AGI due to a possible prioritization of its survival over human welfare. The speaker suggests that while humans appreciate animals, humans often prioritize their needs over the animals, pointing out that a similar relationship could develop between humans and AGI.\n", + "3. The speaker suggests that we will be able to create autonomous beings that align with human goals. He also compares technology to a force of nature, drawing parallels between biological evolution and the evolution of technology. \n", + "\n", + "4. They bring up the topic of Chat GPT, describing it as the 'first spots of rain before a downpour'. It's considered a watershed moment in AI, which the researcher believes could lead to the development of Artificial General Intelligence (AGI) - a system that can perform any task a human can but better. \n", + "\n", + "5. The speaker emphasizes the importance of programming AGIs correctly, noting that if not done correctly, their nature would favor their survival above everything else. \n", + "\n", + "6. The video talks about skepticism among many machine learning experts related to AGI, when it could happen, and even whether it is at all possible. There's a prediction that computers for AI will potentially become 100,000 times faster in a few years.\n", + "\n", + "7. It's highlighted that in an arms race to create AGI first, teams might rush and end up creating an AI that doesn't prioritize human needs.\n", + "\n", + "8. The video concludes with a vision of the future where the Earth is covered with solar panels and data centers. In light of these concerns, it suggests to work on developing AGI as a global cooperation between nations to safeguard human interests.\n", + "\n", + "Overall, the key concept is that while AI has massive potential benefits, it also comes with serious risks. Therefore, the development of AGI, in particular, needs to be conducted with the utmost care and global cooperation. Ensuring the alignment of AI's goals with ours is essential to navigating these challenges.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33muser\u001b[0m (to coder):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mcoder\u001b[0m (to user):\n", "\n", - "The narration presents the prediction of technology experts who believe that the speed of computers for neural networks for AI will dramatically increase in a few years. Given the potential risks and exponential growth associated with AGI, the speaker emphasizes the desperate need for global cooperation in AI development to ensure a future that is beneficial not just for AI but for humanity as well.\n", "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n" @@ -124,7 +143,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Adavanced Example\n", + "## Adavanced: Registering Multiple Functions\n", "\n", "Lets import multiple functions and use them accomplish more complex tasks." ] @@ -165,7 +184,7 @@ "--------------------------------------------------------------------------------\n", "\u001b[33mcoder\u001b[0m (to user):\n", "\n", - "\u001b[32m***** Suggested tool Call (call_9EQ7lnpwFLxStOqmaITZVeMI): read_text_from_image *****\u001b[0m\n", + "\u001b[32m***** Suggested tool Call (call_a62kEIHFUhyooITpart1yHpt): read_text_from_image *****\u001b[0m\n", "Arguments: \n", "{\n", " \"file_path\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/0/0f/Captioned_image_dataset_examples.jpg/1024px-Captioned_image_dataset_examples.jpg\"\n", @@ -198,14 +217,17 @@ "\n", "\u001b[33muser\u001b[0m (to coder):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_9EQ7lnpwFLxStOqmaITZVeMI\" *****\u001b[0m\n", - "This flower has The flower Purple petals This flower has petals that are shown has white that are almost petals that are red with yellow petals with heart shaped yellow with many tips: yellow anther in with small green layers: L the center: receptacles: 8 J This bird is blue This bird is This is a bird bird with long with white on completely with a green black wings, tail the head.the black with a wing; a brown and white feathers are blue large blunt head and a red breast; the bill is and the belly is beak bill: short and black: 1 white. 3 Two people Lunch of rice A big rig truck in A group of men holding and beans with a parking lot traveling on snowboards are soup and juice: without a trailer: horses in the standing in the water; snow 8\n", + "\u001b[32m***** Response from calling tool \"call_a62kEIHFUhyooITpart1yHpt\" *****\u001b[0m\n", + "Error: cannot identify image file <_io.BytesIO object at 0x7fe465f9a610>\n", "\u001b[32m**********************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", "\u001b[33mcoder\u001b[0m (to user):\n", "\n", - "\u001b[32m***** Suggested tool Call (call_jseZ3R7kFEI7JADjge016LaY): read_text_from_pdf *****\u001b[0m\n", + "It seems like the system was not able to identify text from the provided image file. This could be due to complexities in the image like poor quality, complex background, or very stylized text. In some cases, images with captions may not be easily processed by optical character recognition (OCR) technology.\n", + "\n", + "Let's process the next file which is a PDF document.\n", + "\u001b[32m***** Suggested tool Call (call_pQP1zVHddWY9noBUazdBWiNq): read_text_from_pdf *****\u001b[0m\n", "Arguments: \n", "{\n", " \"file_path\": \"https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf\"\n", @@ -223,14 +245,17 @@ "\n", "\u001b[33muser\u001b[0m (to coder):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_jseZ3R7kFEI7JADjge016LaY\" *****\u001b[0m\n", + "\u001b[32m***** Response from calling tool \"call_pQP1zVHddWY9noBUazdBWiNq\" *****\u001b[0m\n", "Dummy PDF file\f\n", "\u001b[32m**********************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", "\u001b[33mcoder\u001b[0m (to user):\n", "\n", - "\u001b[32m***** Suggested tool Call (call_xbXD6vwslLYf1Bn3UvffHanr): read_text_from_audio *****\u001b[0m\n", + "The content of the PDF file located at \"https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf\" only contains the text \"Dummy PDF file\".\n", + "\n", + "Let's now process the audio file.\n", + "\u001b[32m***** Suggested tool Call (call_MCLstwb6E1iISdiIjY8movDO): read_text_from_audio *****\u001b[0m\n", "Arguments: \n", "{\n", " \"file_path\": \"https://github.com/realpython/python-speech-recognition/raw/master/audio_files/harvard.wav\"\n", @@ -248,22 +273,24 @@ "\n", "\u001b[33muser\u001b[0m (to coder):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_xbXD6vwslLYf1Bn3UvffHanr\" *****\u001b[0m\n", + "\u001b[32m***** Response from calling tool \"call_MCLstwb6E1iISdiIjY8movDO\" *****\u001b[0m\n", "the stale smell of old beer lingers it takes heat to bring out the odor a cold dip restores health and zest a salt pickle taste fine with ham tacos al pastor are my favorite a zestful food is the hot cross bun\n", "\u001b[32m**********************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", "\u001b[33mcoder\u001b[0m (to user):\n", "\n", - "Here is the summary of the contents of the provided files:\n", + "The audio file from \"https://github.com/realpython/python-speech-recognition/raw/master/audio_files/harvard.wav\" contains the following speech:\n", "\n", - "1. Image file ([link](https://upload.wikimedia.org/wikipedia/commons/thumb/0/0f/Captioned_image_dataset_examples.jpg/1024px-Captioned_image_dataset_examples.jpg)): The image is a multiple captioned image dataset example. It seems to be a dataset used for machine learning or image analysis. The captions include references to various flowers, a bird, people holding a lunch of rice and beans, a big rig truck in a parking lot, and a group of snowboarders.\n", + "\"the stale smell of old beer lingers it takes heat to bring out the odor a cold dip restores health and zest a salt pickle taste fine with ham tacos al pastor are my favorite a zestful food is the hot cross bun\"\n", "\n", - "2. PDF file ([link](https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf)): The PDF file is a dummy file with no contents.\n", + "To summarize,\n", "\n", - "3. Audio file ([link](https://github.com/realpython/python-speech-recognition/raw/master/audio_files/harvard.wav)): The audio file contains a sentence talking about different smells and tastes. Some phrases are \"the stale smell of old beer lingers\", \"it takes heat to bring out the odor\", \"a salt pickle tastes fine with ham\", and \"a zestful food is the hot cross bun\".\n", + "1. The image file couldn't be processed to extract any text.\n", + "2. The PDF file only contained the text \"Dummy PDF file\".\n", + "3. The audio file consisted of a few phrases, seemingly unconnected and might be part of a larger context or text.\n", "\n", - "Please note that these are direct extractions from the files and a more detailed or contextual summary might need a deeper analysis or understanding of the data.\n", + "Please note, the content might make more sense in their original contexts. If provided, I suggest looking at these files directly to get a complete understanding.\n", "\n", "TERMINATE\n", "\n", @@ -282,6 +309,93 @@ " summary_method=\"last_msg\",\n", ")" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Adavanced: Functions that Require Secrets\n", + "\n", + "In this example, we will use a function that expects a secret, e.g., an `OPENAI_API_KEY` for it work. One such example is the function that using GPT-4-vision to perform image understanding." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser\u001b[0m (to coder):\n", + "\n", + "Please summarize the contents of the following image using gpt4v: https://upload.wikimedia.org/wikipedia/commons/thumb/0/0f/Captioned_image_dataset_examples.jpg/1024px-Captioned_image_dataset_examples.jpg\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mcoder\u001b[0m (to user):\n", + "\n", + "\u001b[32m***** Suggested tool Call (call_jxjODN5UmMifdmnk8P1KIqir): caption_image_using_gpt4v *****\u001b[0m\n", + "Arguments: \n", + "{\n", + " \"file_path_or_url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/0/0f/Captioned_image_dataset_examples.jpg/1024px-Captioned_image_dataset_examples.jpg\"\n", + "}\n", + "\u001b[32m******************************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION caption_image_using_gpt4v...\u001b[0m\n", + "requested package: openai None\n", + "found package openai 1.11.1\n", + "\u001b[33muser\u001b[0m (to coder):\n", + "\n", + "\u001b[33muser\u001b[0m (to coder):\n", + "\n", + "\u001b[32m***** Response from calling tool \"call_jxjODN5UmMifdmnk8P1KIqir\" *****\u001b[0m\n", + "{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"The image consists of a collage of twelve smaller images arranged in a grid, with three rows and four columns. Each of the smaller images is paired with a text description. Here's a breakdown of what each image shows based on the grid positions:\\n\\nTop row (Flowers):\\n1. A flower with red petals and yellow tips.\\n2. A white flower with yellow anthers in the center.\\n3. A flower with purple petals, some of which are heart-shaped, and small green receptacles.\\n4. A yellow flower with multiple layers of petals and a small insect on it.\\n\\nSecond row (Birds):\\n1. A blue bird with white on its head and belly.\\n2. A black bird with a large blunt beak.\\n3. A bird with a green wing, brown head, and red bill.\\n4. A bird with long black wings, a white tail and breast, and a short black bill.\\n\\nThird row (COCO dataset images):\\n1. Two individuals outdoors in a snowy setting holding snowboards.\\n2. A meal consisting of rice, beans, soup, and juice served on a table.\\n3. A big rig truck parked in a lot without its trailer attached.\\n4. A group of men on horseback, fording a body of water.\\n\\nThe images encompass various categories including flowers, birds, outdoor activities, food, transportation, and an equestrian scene.\",\"role\":\"assistant\",\"function_call\":null,\"tool_calls\":null}}\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mcoder\u001b[0m (to user):\n", + "\n", + "The image consists of a collage of twelve smaller images arranged in a grid, with three rows and four columns. Each of the smaller images is paired with a text description. Here's a breakdown of what each image shows based on the grid positions:\n", + "\n", + "Top row (Flowers):\n", + "1. A flower with red petals and yellow tips.\n", + "2. A white flower with yellow anthers in the center.\n", + "3. A flower with purple petals, some of which are heart-shaped, and small green receptacles.\n", + "4. A yellow flower with multiple layers of petals and a small insect on it.\n", + "\n", + "Second row (Birds):\n", + "1. A blue bird with white on its head and belly.\n", + "2. A black bird with a large blunt beak.\n", + "3. A bird with a green wing, brown head, and red bill.\n", + "4. A bird with long black wings, a white tail and breast, and a short black bill.\n", + "\n", + "Third row (COCO dataset images):\n", + "1. Two individuals outdoors in a snowy setting holding snowboards.\n", + "2. A meal consisting of rice, beans, soup, and juice served on a table.\n", + "3. A big rig truck parked in a lot without its trailer attached.\n", + "4. A group of men on horseback, fording a body of water.\n", + "\n", + "The images encompass various categories including flowers, birds, outdoor activities, food, transportation, and an equestrian scene.\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "assistant.register_for_llm(description=\"Use gpt4 vision to understand an image\")(fu.caption_image_using_gpt4v)\n", + "user.register_for_execution()(fu.caption_image_using_gpt4v)\n", + "\n", + "result = user.initiate_chat(\n", + " assistant,\n", + " message=f\"Please summarize the contents of the following image using gpt4v: {dummy_png}\",\n", + " summary_method=\"last_msg\",\n", + ")" + ] } ], "metadata": {