rysweet-4150-xlang-ci-test (#4596)

adds aspire-based integration test that validates:
* registration
* subscriptions
* event delivery
* python -> .NET server -> python subscriber
* .NET -> .NET server  -> python subscriber
* python -> .NET server -> .NET subscriber

.github/workflows/codeql.yml

@@ -61,6 +61,26 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v4
 
+    - name: Set up Python 
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+    - name: Install jupyter and ipykernel
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install jupyter
+        python -m pip install ipykernel
+    - name: list available kernels
+      run: |
+        python -m jupyter kernelspec list
+    - uses: astral-sh/setup-uv@v3
+      with:
+        enable-cache: true
+    - run: uv sync --locked --all-extras
+      working-directory: ./python
+    - name: Prepare python venv
+      run: |
+        source ${{ github.workspace }}/python/.venv/bin/activate
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v3

.github/workflows/dotnet-build.yml

@@ -76,12 +76,18 @@ jobs:
     - name: list available kernels
       run: |
         python -m jupyter kernelspec list
-    - name: Setup .NET
+    - uses: astral-sh/setup-uv@v3
+      with:
+        enable-cache: true
+    - run: uv sync --locked --all-extras
+      working-directory: ./python
+    - name: Prepare python venv
+      run: |
+        source ${{ github.workspace }}/python/.venv/bin/activate
+    - name: Setup .NET 8.0
       uses: actions/setup-dotnet@v4
       with:
         dotnet-version: '8.0.x'
-    - name: Install .NET Aspire workload
-      run: dotnet workload install aspire
     - name: Restore dependencies
       run: |
         # dotnet nuget add source --name dotnet-tool https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json --configfile NuGet.config
@@ -96,7 +102,57 @@ jobs:
         echo "Build AutoGen"
         dotnet build --no-restore --configuration Release -bl /p:SignAssembly=true
     - name: Unit Test
-      run: dotnet test --no-build -bl --configuration Release
+      run: dotnet test --no-build -bl --configuration Release --filter type=!integration
+  
+  integration-test:
+    strategy:
+      fail-fast: true 
+      matrix:
+        os: [ ubuntu-latest]
+        version: [ net8.0 ]
+    needs: build
+    defaults:
+      run:
+        working-directory: dotnet
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+          lfs: true
+    - uses: astral-sh/setup-uv@v3
+      with:
+        enable-cache: true
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+    - run: uv sync --locked --all-extras
+      working-directory: ./python
+    - name: Prepare python venv
+      run: |
+        source ${{ github.workspace }}/python/.venv/bin/activate
+    - name: Setup .NET 9.0
+      uses: actions/setup-dotnet@v4
+      with:
+        dotnet-version: '9.0.x'
+    - name: Install Temp Global.JSON
+      run: |
+        echo "{\"sdk\": {\"version\": \"9.0.101\"}}" > global.json
+    - name: Install .NET Aspire workload
+      run: dotnet workload install aspire
+    - name: Install dev certs
+      run:  dotnet --version && dotnet dev-certs https --trust
+    - name: Restore dependencies
+      run: |
+        dotnet restore -bl
+    - name: Build
+      run: |
+        echo "Build AutoGen"
+        dotnet build --no-restore --configuration Release -bl /p:SignAssembly=true
+    - name: Integration Test
+      run: dotnet --version && dotnet test --no-build -bl --configuration Release --filter type=integration
+    - name: Restore the global.json
+      run: rm global.json && git checkout -- global.json
+
   aot-test: # this make sure the AutoGen.Core is aot compatible
     strategy:
       fail-fast: false  # ensures the entire test matrix is run, even if one permutation fails
@@ -147,9 +203,17 @@ jobs:
     - name: list available kernels
       run: |
         python -m jupyter kernelspec list
-    - name: Setup .NET
+    - uses: astral-sh/setup-uv@v3
+      with:
+        enable-cache: true
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+
+    - name: Setup .NET 8.0
       uses: actions/setup-dotnet@v4
       with:
+        dotnet-version: '8.0.x'
         global-json-file: dotnet/global.json
     - name: Restore dependencies
       run: |
@@ -159,7 +223,7 @@ jobs:
         echo "Build AutoGen"
         dotnet build --no-restore --configuration Release -bl /p:SignAssembly=true
     - name: OpenAI Test
-      run: dotnet test --no-build -bl --configuration Release
+      run: dotnet test --no-build -bl --configuration Release --filter type!=integration
       env:
         AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
         AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}

dotnet/AutoGen.sln

@@ -132,6 +132,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AutoGen.Extension
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AutoGen.Agents.Tests", "test\Microsoft.AutoGen.Agents.Tests\Microsoft.AutoGen.Agents.Tests.csproj", "{394FDAF8-74F9-4977-94A5-3371737EB774}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.AutoGen.Integration.Tests", "test\Microsoft.AutoGen.Integration.Tests\Microsoft.AutoGen.Integration.Tests.csproj", "{D04C6153-8EAF-4E54-9852-52CEC1BE8D31}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -342,6 +344,10 @@ Global
 		{394FDAF8-74F9-4977-94A5-3371737EB774}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{394FDAF8-74F9-4977-94A5-3371737EB774}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{394FDAF8-74F9-4977-94A5-3371737EB774}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D04C6153-8EAF-4E54-9852-52CEC1BE8D31}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D04C6153-8EAF-4E54-9852-52CEC1BE8D31}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D04C6153-8EAF-4E54-9852-52CEC1BE8D31}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D04C6153-8EAF-4E54-9852-52CEC1BE8D31}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -402,6 +408,7 @@ Global
 		{64EF61E7-00A6-4E5E-9808-62E10993A0E5} = {7EB336C2-7C0A-4BC8-80C6-A3173AB8DC45}
 		{65059914-5527-4A00-9308-9FAF23D5E85A} = {18BF8DD7-0585-48BF-8F97-AD333080CE06}
 		{394FDAF8-74F9-4977-94A5-3371737EB774} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
+		{D04C6153-8EAF-4E54-9852-52CEC1BE8D31} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {93384647-528D-46C8-922C-8DB36A382F0B}

dotnet/Directory.Packages.props

@@ -15,6 +15,7 @@
   <ItemGroup>
     <PackageVersion Include="Aspire.Hosting" Version="9.0.0" />
     <PackageVersion Include="Aspire.Hosting.Python" Version="9.0.0" />
+    <PackageVersion Include="Aspire.Hosting.Testing" Version="9.0.0" />
     <PackageVersion Include="AspNetCore.Authentication.ApiKey" Version="8.0.1" />
     <PackageVersion Include="Aspire.Azure.AI.OpenAI" Version="8.0.1-preview.8.24267.1" />
     <PackageVersion Include="Aspire.Hosting.AppHost" Version="9.0.0" />
@@ -31,12 +32,17 @@
     <PackageVersion Include="Azure.ResourceManager.ContainerInstance" Version="1.2.1" />
     <PackageVersion Include="Azure.Storage.Files.Shares" Version="12.21.0" />
     <PackageVersion Include="CloudNative.CloudEvents.SystemTextJson" Version="2.7.1" />
+    <PackageVersion Include="GitHubActionsTestLogger" Version="2.4.1">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageVersion>
     <PackageVersion Include="Grpc.AspNetCore" Version="2.67.0" />
     <PackageVersion Include="Grpc.Core" Version="2.46.6" />
     <PackageVersion Include="Grpc.Net.ClientFactory" Version="2.67.0" />
     <PackageVersion Include="Grpc.Tools" Version="2.67.0" />
     <PackageVersion Include="Grpc.Net.Client" Version="2.65.0" />
     <PackageVersion Include="Google.Protobuf" Version="3.28.3" />
+    <PackageVersion Include="MartinCostello.Logging.XUnit" Version="0.4.0" />
     <PackageVersion Include="Microsoft.AspNetCore.App" Version="8.0.4" />
     <PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="8.0.8" />
     <PackageVersion Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.22.0" />
@@ -54,6 +60,7 @@
     <PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="$(MicrosoftExtensionConfiguration)" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="$(MicrosoftExtensionDependencyInjection)" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="$(MicrosoftExtensionDependencyInjection)" />
+    <PackageVersion Include="Microsoft.Extensions.Diagnostics.Testing" Version="9.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Hosting" Version="9.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="9.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionLogging)" />
@@ -118,5 +125,6 @@
     <PackageVersion Include="Moq" Version="4.20.72" />
     <PackageVersion Include="Microsoft.PowerShell.SDK" Version="7.4.5" />
     <PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="8.0.11" />
+    <PackageVersion Include="coverlet.collector" Version="6.0.2" />
   </ItemGroup>
-</Project>
+</Project>

dotnet/test/Microsoft.AutoGen.Integration.Tests/HelloAppHostIntegrationTests.cs

@@ -0,0 +1,120 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// HelloAppHostIntegrationTests.cs
+
+using System.Text.Json;
+using Xunit.Abstractions;
+
+namespace Microsoft.AutoGen.Integration.Tests;
+
+public class HelloAppHostIntegrationTests(ITestOutputHelper testOutput)
+{
+    [Theory, Trait("type", "integration")]
+    [MemberData(nameof(AppHostAssemblies))]
+    public async Task AppHostRunsCleanly(string appHostPath)
+    {
+        var appHost = await DistributedApplicationTestFactory.CreateAsync(appHostPath, testOutput);
+        await using var app = await appHost.BuildAsync().WaitAsync(TimeSpan.FromSeconds(15));
+
+        await app.StartAsync().WaitAsync(TimeSpan.FromSeconds(120));
+        await app.WaitForResourcesAsync().WaitAsync(TimeSpan.FromSeconds(120));
+
+        app.EnsureNoErrorsLogged();
+        await app.StopAsync().WaitAsync(TimeSpan.FromSeconds(15));
+    }
+
+    [Theory, Trait("type", "integration")]
+    [MemberData(nameof(TestEndpoints))]
+    public async Task AppHostLogsHelloAgentE2E(TestEndpoints testEndpoints)
+    {
+        var appHostName = testEndpoints.AppHost!;
+        var appHostPath = $"{appHostName}.dll";
+        var appHost = await DistributedApplicationTestFactory.CreateAsync(appHostPath, testOutput);
+        await using var app = await appHost.BuildAsync().WaitAsync(TimeSpan.FromSeconds(15));
+
+        await app.StartAsync().WaitAsync(TimeSpan.FromSeconds(120));
+        await app.WaitForResourcesAsync().WaitAsync(TimeSpan.FromSeconds(120));
+        if (testEndpoints.WaitForResources?.Count > 0)
+        {
+            // Wait until each resource transitions to the required state
+            var timeout = TimeSpan.FromMinutes(5);
+            foreach (var (ResourceName, TargetState) in testEndpoints.WaitForResources)
+            {
+                await app.WaitForResource(ResourceName, TargetState).WaitAsync(timeout);
+            }
+        }
+        //sleep 5 seconds to make sure the app is running
+        await Task.Delay(5000);
+        app.EnsureNoErrorsLogged();
+        app.EnsureLogContains("HelloAgents said Goodbye");
+        app.EnsureLogContains("Wild Hello from Python!");
+
+        await app.StopAsync().WaitAsync(TimeSpan.FromSeconds(15));
+    }
+    public static TheoryData<string> AppHostAssemblies()
+    {
+        var appHostAssemblies = GetSamplesAppHostAssemblyPaths();
+        var theoryData = new TheoryData<string, bool>();
+        return new(appHostAssemblies.Select(p => Path.GetRelativePath(AppContext.BaseDirectory, p)));
+    }
+
+    public static TheoryData<TestEndpoints> TestEndpoints() =>
+        new([
+            new TestEndpoints("Hello.AppHost", new() {
+                { "backend", ["/"] }
+            }),
+        ]);
+
+    private static IEnumerable<string> GetSamplesAppHostAssemblyPaths()
+    {
+        // All the AppHost projects are referenced by this project so we can find them by looking for all their assemblies in the base directory
+        return Directory.GetFiles(AppContext.BaseDirectory, "*.AppHost.dll")
+            .Where(fileName => !fileName.EndsWith("Aspire.Hosting.AppHost.dll", StringComparison.OrdinalIgnoreCase));
+    }
+}
+
+public class TestEndpoints : IXunitSerializable
+{
+    // Required for deserialization
+    public TestEndpoints() { }
+
+    public TestEndpoints(string appHost, Dictionary<string, List<string>> resourceEndpoints)
+    {
+        AppHost = appHost;
+        ResourceEndpoints = resourceEndpoints;
+    }
+
+    public string? AppHost { get; set; }
+
+    public List<ResourceWait>? WaitForResources { get; set; }
+
+    public Dictionary<string, List<string>>? ResourceEndpoints { get; set; }
+
+    public void Deserialize(IXunitSerializationInfo info)
+    {
+        AppHost = info.GetValue<string>(nameof(AppHost));
+        WaitForResources = JsonSerializer.Deserialize<List<ResourceWait>>(info.GetValue<string>(nameof(WaitForResources)));
+        ResourceEndpoints = JsonSerializer.Deserialize<Dictionary<string, List<string>>>(info.GetValue<string>(nameof(ResourceEndpoints)));
+    }
+
+    public void Serialize(IXunitSerializationInfo info)
+    {
+        info.AddValue(nameof(AppHost), AppHost);
+        info.AddValue(nameof(WaitForResources), JsonSerializer.Serialize(WaitForResources));
+        info.AddValue(nameof(ResourceEndpoints), JsonSerializer.Serialize(ResourceEndpoints));
+    }
+
+    public override string? ToString() => $"{AppHost} ({ResourceEndpoints?.Count ?? 0} resources)";
+
+    public class ResourceWait(string resourceName, string targetState)
+    {
+        public string ResourceName { get; } = resourceName;
+
+        public string TargetState { get; } = targetState;
+
+        public void Deconstruct(out string resourceName, out string targetState)
+        {
+            resourceName = ResourceName;
+            targetState = TargetState;
+        }
+    }
+}

dotnet/test/Microsoft.AutoGen.Integration.Tests/Infrastructure/DistributedApplicationExtension.cs

@@ -0,0 +1,301 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// DistributedApplicationExtension.cs
+
+using System.Security.Cryptography;
+using Aspire.Hosting;
+using Aspire.Hosting.Python;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Testing;
+
+namespace Microsoft.AutoGen.Integration.Tests;
+
+public static partial class DistributedApplicationExtensions
+{
+    /*  /// <summary>
+      /// Ensures all parameters in the application configuration have values set.
+      /// </summary>
+       public static TBuilder WithRandomParameterValues<TBuilder>(this TBuilder builder)
+          where TBuilder : IDistributedApplicationTestingBuilder
+      {
+          var parameters = builder.Resources.OfType<ParameterResource>().Where(p => !p.IsConnectionString).ToList();
+          foreach (var parameter in parameters)
+          {
+              builder.Configuration[$"Parameters:{parameter.Name}"] = parameter.Secret
+                  ? PasswordGenerator.Generate(16, true, true, true, false, 1, 1, 1, 0)
+                  : Convert.ToHexString(RandomNumberGenerator.GetBytes(4));
+          }
+
+          return builder;
+      } */
+
+    /// <summary>
+    /// Sets the container lifetime for all container resources in the application.
+    /// </summary>
+    public static TBuilder WithContainersLifetime<TBuilder>(this TBuilder builder, ContainerLifetime containerLifetime)
+        where TBuilder : IDistributedApplicationTestingBuilder
+    {
+        var containerLifetimeAnnotations = builder.Resources.SelectMany(r => r.Annotations
+            .OfType<ContainerLifetimeAnnotation>()
+            .Where(c => c.Lifetime != containerLifetime))
+            .ToList();
+
+        foreach (var annotation in containerLifetimeAnnotations)
+        {
+            annotation.Lifetime = containerLifetime;
+        }
+
+        return builder;
+    }
+
+    /// <summary>
+    /// Replaces all named volumes with anonymous volumes so they're isolated across test runs and from the volume the app uses during development.
+    /// </summary>
+    /// <remarks>
+    /// Note that if multiple resources share a volume, the volume will instead be given a random name so that it's still shared across those resources in the test run.
+    /// </remarks>
+    public static TBuilder WithRandomVolumeNames<TBuilder>(this TBuilder builder)
+        where TBuilder : IDistributedApplicationTestingBuilder
+    {
+        // Named volumes that aren't shared across resources should be replaced with anonymous volumes.
+        // Named volumes shared by mulitple resources need to have their name randomized but kept shared across those resources.
+
+        // Find all shared volumes and make a map of their original name to a new randomized name
+        var allResourceNamedVolumes = builder.Resources.SelectMany(r => r.Annotations
+            .OfType<ContainerMountAnnotation>()
+            .Where(m => m.Type == ContainerMountType.Volume && !string.IsNullOrEmpty(m.Source))
+            .Select(m => (Resource: r, Volume: m)))
+            .ToList();
+        var seenVolumes = new HashSet<string>();
+        var renamedVolumes = new Dictionary<string, string>();
+        foreach (var resourceVolume in allResourceNamedVolumes)
+        {
+            var name = resourceVolume.Volume.Source!;
+            if (!seenVolumes.Add(name) && !renamedVolumes.ContainsKey(name))
+            {
+                renamedVolumes[name] = $"{name}-{Convert.ToHexString(RandomNumberGenerator.GetBytes(4))}";
+            }
+        }
+
+        // Replace all named volumes with randomly named or anonymous volumes
+        foreach (var resourceVolume in allResourceNamedVolumes)
+        {
+            var resource = resourceVolume.Resource;
+            var volume = resourceVolume.Volume;
+            var newName = renamedVolumes.TryGetValue(volume.Source!, out var randomName) ? randomName : null;
+            var newMount = new ContainerMountAnnotation(newName, volume.Target, ContainerMountType.Volume, volume.IsReadOnly);
+            resource.Annotations.Remove(volume);
+            resource.Annotations.Add(newMount);
+        }
+
+        return builder;
+    }
+
+    /// <summary>
+    /// Waits for the specified resource to reach the specified state.
+    /// </summary>
+    public static Task WaitForResource(this DistributedApplication app, string resourceName, string? targetState = null, CancellationToken cancellationToken = default)
+    {
+        targetState ??= KnownResourceStates.Running;
+        var resourceNotificationService = app.Services.GetRequiredService<ResourceNotificationService>();
+
+        return resourceNotificationService.WaitForResourceAsync(resourceName, targetState, cancellationToken);
+    }
+
+    /// <summary>
+    /// Waits for all resources in the application to reach one of the specified states.
+    /// </summary>
+    /// <remarks>
+    /// If <paramref name="targetStates"/> is null, the default states are <see cref="KnownResourceStates.Running"/> and <see cref="KnownResourceStates.Hidden"/>.
+    /// </remarks>
+    public static async Task WaitForResourcesAsync(this DistributedApplication app, IEnumerable<string>? targetStates = null, CancellationToken cancellationToken = default)
+    {
+        var logger = app.Services.GetRequiredService<ILoggerFactory>().CreateLogger(nameof(WaitForResourcesAsync));
+
+        targetStates ??= [KnownResourceStates.Running, KnownResourceStates.Hidden, .. KnownResourceStates.TerminalStates];
+        var applicationModel = app.Services.GetRequiredService<DistributedApplicationModel>();
+        var resourceNotificationService = app.Services.GetRequiredService<ResourceNotificationService>();
+
+        var resourceTasks = new Dictionary<string, Task<(string Name, string State)>>();
+
+        foreach (var resource in applicationModel.Resources)
+        {
+            resourceTasks[resource.Name] = GetResourceWaitTask(resource.Name, targetStates, cancellationToken);
+        }
+
+        logger.LogInformation("Waiting for resources [{Resources}] to reach one of target states [{TargetStates}].",
+            string.Join(',', resourceTasks.Keys),
+            string.Join(',', targetStates));
+
+        while (resourceTasks.Count > 0)
+        {
+            var completedTask = await Task.WhenAny(resourceTasks.Values);
+            var (completedResourceName, targetStateReached) = await completedTask;
+
+            if (targetStateReached == KnownResourceStates.FailedToStart)
+            {
+                throw new DistributedApplicationException($"Resource '{completedResourceName}' failed to start.");
+            }
+
+            resourceTasks.Remove(completedResourceName);
+
+            logger.LogInformation("Wait for resource '{ResourceName}' completed with state '{ResourceState}'", completedResourceName, targetStateReached);
+
+            // Ensure resources being waited on still exist
+            var remainingResources = resourceTasks.Keys.ToList();
+            for (var i = remainingResources.Count - 1; i > 0; i--)
+            {
+                var name = remainingResources[i];
+                if (!applicationModel.Resources.Any(r => r.Name == name))
+                {
+                    logger.LogInformation("Resource '{ResourceName}' was deleted while waiting for it.", name);
+                    resourceTasks.Remove(name);
+                    remainingResources.RemoveAt(i);
+                }
+            }
+
+            if (resourceTasks.Count > 0)
+            {
+                logger.LogInformation("Still waiting for resources [{Resources}] to reach one of target states [{TargetStates}].",
+                    string.Join(',', remainingResources),
+                    string.Join(',', targetStates));
+            }
+        }
+
+        logger.LogInformation("Wait for all resources completed successfully!");
+
+        async Task<(string Name, string State)> GetResourceWaitTask(string resourceName, IEnumerable<string> targetStates, CancellationToken cancellationToken)
+        {
+            var state = await resourceNotificationService.WaitForResourceAsync(resourceName, targetStates, cancellationToken);
+            return (resourceName, state);
+        }
+    }
+
+    /// <summary>
+    /// Gets the app host and resource logs from the application.
+    /// </summary>
+    public static (IReadOnlyList<FakeLogRecord> AppHostLogs, IReadOnlyList<FakeLogRecord> ResourceLogs) GetLogs(this DistributedApplication app)
+    {
+        var environment = app.Services.GetRequiredService<IHostEnvironment>();
+        var logCollector = app.Services.GetFakeLogCollector();
+        var logs = logCollector.GetSnapshot();
+        var appHostLogs = logs.Where(l => l.Category?.StartsWith($"{environment.ApplicationName}.Resources") == false).ToList();
+        var resourceLogs = logs.Where(l => l.Category?.StartsWith($"{environment.ApplicationName}.Resources") == true).ToList();
+
+        return (appHostLogs, resourceLogs);
+    }
+
+    /// <summary>
+    /// Get all logs from the whole test run.
+    /// </summary>
+    /// <param name="app"></param>
+    /// <returns>List</returns>
+    public static IReadOnlyList<FakeLogRecord> GetAllLogs(this DistributedApplication app)
+    {
+        var logCollector = app.Services.GetFakeLogCollector();
+        return logCollector.GetSnapshot();
+    }
+
+    /// <summary>
+    /// Asserts that no errors were logged by the application or any of its resources.
+    /// </summary>
+    /// <remarks>
+    /// Some resource types are excluded from this check because they tend to write to stderr for various non-error reasons.
+    /// </remarks>
+    /// <param name="app"></param>
+    public static void EnsureNoErrorsLogged(this DistributedApplication app)
+    {
+        var environment = app.Services.GetRequiredService<IHostEnvironment>();
+        var applicationModel = app.Services.GetRequiredService<DistributedApplicationModel>();
+        var assertableResourceLogNames = applicationModel.Resources.Where(ShouldAssertErrorsForResource).Select(r => $"{environment.ApplicationName}.Resources.{r.Name}").ToList();
+
+        var (appHostlogs, resourceLogs) = app.GetLogs();
+
+        Assert.DoesNotContain(appHostlogs, log => log.Level >= LogLevel.Error);
+        Assert.DoesNotContain(resourceLogs, log => log.Category is { Length: > 0 } category && assertableResourceLogNames.Contains(category) && log.Level >= LogLevel.Error);
+
+        static bool ShouldAssertErrorsForResource(IResource resource)
+        {
+#pragma warning disable ASPIREHOSTINGPYTHON001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
+            return resource
+                is
+                    // Container resources tend to write to stderr for various reasons so only assert projects and executables
+                    (ProjectResource or ExecutableResource)
+                    // Node & Python resources tend to have modules that write to stderr so ignore them
+                    and not (PythonAppResource)
+                // Dapr resources write to stderr about deprecated --components-path flag
+                && !resource.Name.EndsWith("-dapr-cli");
+#pragma warning restore ASPIREHOSTINGPYTHON001
+        }
+    }
+
+    /// <summary>
+    /// Asserts that the application and resource logs contain the specified message.
+    /// </summary>
+    /// <param name="app"></param>
+    /// <param name="message"></param>
+    public static void EnsureLogContains(this DistributedApplication app, string message)
+    {
+        var resourceLogs = app.GetAllLogs();
+        Assert.Contains(resourceLogs, log => log.Message.Contains(message));
+    }
+
+    /// <summary>
+    /// Creates an <see cref="HttpClient"/> configured to communicate with the specified resource.
+    /// </summary>
+    public static HttpClient CreateHttpClient(this DistributedApplication app, string resourceName, bool useHttpClientFactory)
+        => app.CreateHttpClient(resourceName, null, useHttpClientFactory);
+
+    /// <summary>
+    /// Creates an <see cref="HttpClient"/> configured to communicate with the specified resource.
+    /// </summary>
+    public static HttpClient CreateHttpClient(this DistributedApplication app, string resourceName, string? endpointName, bool useHttpClientFactory)
+    {
+        if (useHttpClientFactory)
+        {
+            return app.CreateHttpClient(resourceName, endpointName);
+        }
+
+        // Don't use the HttpClientFactory to create the HttpClient so, e.g., no resilience policies are applied
+        var httpClient = new HttpClient
+        {
+            BaseAddress = app.GetEndpoint(resourceName, endpointName)
+        };
+
+        return httpClient;
+    }
+
+    /// <summary>
+    /// Creates an <see cref="HttpClient"/> configured to communicate with the specified resource with custom configuration.
+    /// </summary>
+    public static HttpClient CreateHttpClient(this DistributedApplication app, string resourceName, string? endpointName, Action<IHttpClientBuilder> configure)
+    {
+        var services = new ServiceCollection()
+            .AddHttpClient()
+            .ConfigureHttpClientDefaults(configure)
+            .BuildServiceProvider();
+        var httpClientFactory = services.GetRequiredService<IHttpClientFactory>();
+
+        var httpClient = httpClientFactory.CreateClient();
+        httpClient.BaseAddress = app.GetEndpoint(resourceName, endpointName);
+
+        return httpClient;
+    }
+
+    private static bool DerivesFromDbContext(Type type)
+    {
+        var baseType = type.BaseType;
+
+        while (baseType is not null)
+        {
+            if (baseType.FullName == "Microsoft.EntityFrameworkCore.DbContext" && baseType.Assembly.GetName().Name == "Microsoft.EntityFrameworkCore")
+            {
+                return true;
+            }
+
+            baseType = baseType.BaseType;
+        }
+
+        return false;
+    }
+}

dotnet/test/Microsoft.AutoGen.Integration.Tests/Infrastructure/DistributedApplicationTestFactory.cs

@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// DistributedApplicationTestFactory.cs
+
+using System.Reflection;
+using Microsoft.Extensions.Logging;
+using Xunit.Abstractions;
+
+namespace Microsoft.AutoGen.Integration.Tests;
+
+internal static class DistributedApplicationTestFactory
+{
+    /// <summary>
+    /// Creates an <see cref="IDistributedApplicationTestingBuilder"/> for the specified app host assembly.
+    /// </summary>
+    public static async Task<IDistributedApplicationTestingBuilder> CreateAsync(string appHostAssemblyPath, ITestOutputHelper? testOutput)
+    {
+        var appHostProjectName = Path.GetFileNameWithoutExtension(appHostAssemblyPath) ?? throw new InvalidOperationException("AppHost assembly was not found.");
+
+        var appHostAssembly = Assembly.LoadFrom(Path.Combine(AppContext.BaseDirectory, appHostAssemblyPath));
+
+        var appHostType = appHostAssembly.GetTypes().FirstOrDefault(t => t.Name.EndsWith("_AppHost"))
+            ?? throw new InvalidOperationException("Generated AppHost type not found.");
+
+        var builder = await DistributedApplicationTestingBuilder.CreateAsync(appHostType);
+
+        //builder.WithRandomParameterValues();
+        builder.WithRandomVolumeNames();
+        builder.WithContainersLifetime(ContainerLifetime.Session);
+
+        builder.Services.AddLogging(logging =>
+        {
+            logging.ClearProviders();
+            logging.AddSimpleConsole();
+            logging.AddFakeLogging();
+            if (testOutput is not null)
+            {
+                logging.AddXUnit(testOutput);
+            }
+            logging.SetMinimumLevel(LogLevel.Trace);
+            logging.AddFilter("Aspire", LogLevel.Trace);
+            logging.AddFilter(builder.Environment.ApplicationName, LogLevel.Trace);
+        });
+
+        return builder;
+    }
+}

dotnet/test/Microsoft.AutoGen.Integration.Tests/Microsoft.AutoGen.Integration.Tests.csproj

@@ -0,0 +1,52 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Aspire.Hosting.Testing" />
+    <PackageReference Include="coverlet.collector" />
+    <PackageReference Include="GitHubActionsTestLogger">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="MartinCostello.Logging.XUnit" />
+    <PackageReference Include="Microsoft.Extensions.Diagnostics.Testing" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Using Include="System.Net" />
+    <Using Include="Microsoft.Extensions.DependencyInjection" />
+    <Using Include="Aspire.Hosting.ApplicationModel" />
+    <Using Include="Aspire.Hosting.Testing" />
+    <Using Include="Xunit" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\samples\Hello\Hello.AppHost\Hello.AppHost.csproj" />
+  </ItemGroup>
+
+  <!-- Properties, Items, and targets to ensure Python apps are initialized -->
+  <PropertyGroup>
+    <PythonVirtualEnvironmentName>.venv</PythonVirtualEnvironmentName>
+    <PythonVenvRoot>$(RepoRoot)..\python</PythonVenvRoot>
+
+  </PropertyGroup>
+
+  <Target Name="CreatePythonVirtualEnvironments" AfterTargets="PrepareForBuild">
+    <PropertyGroup>
+      <VenvDir>$(PythonVenvRoot)\$(PythonVirtualEnvironmentName)\</VenvDir>
+      <CreateVenv Condition="!Exists('$(VenvDir)')">True</CreateVenv>
+    </PropertyGroup>  
+    <Message Importance="Normal" Text="Initializing virtual environment for $(PythonVenvRoot)" />
+
+    <Exec Command="uv sync --all-extras" WorkingDirectory="$(PythonVenvRoot)" Condition=" '$(CreateVenv)' == 'True' " />
+  </Target>
+
+
+</Project>