Skip to content

Commit 0d9ff40

Browse files
kzukzu
committed
Add tool-focused extension to inspect calls and results
It's sometimes useful to be able to inspect (wether in tests or in production) the invocations that were performed by the model, including typed results. The introduced `ToolJsonOptions` provide a mechanism to automatically inject a `$type` for result types so they can be later inspected by the `FindCall<TResult>` extension for `ChatResponse`. To simplify this scenario, we also provide a `ToolFactory` that automatically sets things up for this scenario, additionally making the tool name default to the naming convention for tools, rather than the .NET method name.
1 parent e42de34 commit 0d9ff40

6 files changed

Lines changed: 391 additions & 0 deletions

File tree

‎readme.md‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,54 @@ var openai = new OpenAIClient(
136136
OpenAIClientOptions.Observable(requests.Add, responses.Add));
137137
```
138138

139+
## Tool Results
140+
141+
Given the following tool:
142+
143+
```csharp
144+
MyResult RunTool(string name, string description, string content) { ... }
145+
```
146+
147+
You can use the `ToolFactory` and `FindCall<MyResult>` extension method to
148+
locate the function invocation, its outcome and the typed result for inspection:
149+
150+
```csharp
151+
AIFunction tool = ToolFactory.Create(RunTool);
152+
var options = new ChatOptions
153+
{
154+
ToolMode = ChatToolMode.RequireSpecific(tool.Name), // 👈 forces the tool to be used
155+
Tools = [tool]
156+
};
157+
158+
var response = await client.GetResponseAsync(chat, options);
159+
var result = response.FindCalls<MyResult>(tool).FirstOrDefault();
160+
161+
if (result != null)
162+
{
163+
// Successful tool call
164+
Console.WriteLine($"Args: '{result.Call.Arguments.Count}'");
165+
MyResult typed = result.Result;
166+
}
167+
else
168+
{
169+
Console.WriteLine("Tool call not found in response.");
170+
}
171+
```
172+
173+
If the typed result is not found, you can also inspect the raw outcomes by finding
174+
untyped calls to the tool and checking their `Outcome.Exception` property:
175+
176+
```csharp
177+
var result = response.FindCalls(tool).FirstOrDefault();
178+
if (result.Outcome.Exception is not null)
179+
{
180+
Console.WriteLine($"Tool call failed: {result.Outcome.Exception.Message}");
181+
}
182+
else
183+
{
184+
Console.WriteLine($"Tool call succeeded: {result.Outcome.Result}");
185+
}
186+
```
139187

140188
## Console Logging
141189

‎src/AI.Tests/ToolsTests.cs‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
using System.ComponentModel;
2+
using Microsoft.Extensions.AI;
3+
using static ConfigurationExtensions;
4+
5+
namespace Devlooped.Extensions.AI;
6+
7+
public class ToolsTests(ITestOutputHelper output)
8+
{
9+
public record ToolResult(string Name, string Description, string Content);
10+
11+
[SecretsFact("OPENAI_API_KEY")]
12+
public async Task RunToolResult()
13+
{
14+
var chat = new Chat()
15+
{
16+
{ "system", "You make up a tool run by making up a name, description and content based on whatever the user says." },
17+
{ "user", "I want to create an order for a dozen eggs" },
18+
};
19+
20+
var client = new OpenAIChatClient(Configuration["OPENAI_API_KEY"]!, "gpt-4.1",
21+
OpenAI.OpenAIClientOptions.WriteTo(output))
22+
.AsBuilder()
23+
.UseFunctionInvocation()
24+
.Build();
25+
26+
var tool = ToolFactory.Create(RunTool);
27+
var options = new ChatOptions
28+
{
29+
ToolMode = ChatToolMode.RequireSpecific(tool.Name),
30+
Tools = [tool]
31+
};
32+
33+
var response = await client.GetResponseAsync(chat, options);
34+
var result = response.FindCalls<ToolResult>(tool).FirstOrDefault();
35+
36+
Assert.NotNull(result);
37+
Assert.NotNull(result.Call);
38+
Assert.Equal(tool.Name, result.Call.Name);
39+
Assert.NotNull(result.Outcome);
40+
Assert.Null(result.Outcome.Exception);
41+
}
42+
43+
[SecretsFact("OPENAI_API_KEY")]
44+
public async Task RunToolTerminateResult()
45+
{
46+
var chat = new Chat()
47+
{
48+
{ "system", "You make up a tool run by making up a name, description and content based on whatever the user says." },
49+
{ "user", "I want to create an order for a dozen eggs" },
50+
};
51+
52+
var client = new OpenAIChatClient(Configuration["OPENAI_API_KEY"]!, "gpt-4.1",
53+
OpenAI.OpenAIClientOptions.WriteTo(output))
54+
.AsBuilder()
55+
.UseFunctionInvocation()
56+
.Build();
57+
58+
var tool = ToolFactory.Create(RunToolTerminate);
59+
var options = new ChatOptions
60+
{
61+
ToolMode = ChatToolMode.RequireSpecific(tool.Name),
62+
Tools = [tool]
63+
};
64+
65+
var response = await client.GetResponseAsync(chat, options);
66+
var result = response.FindCalls<ToolResult>(tool).FirstOrDefault();
67+
68+
Assert.NotNull(result);
69+
Assert.NotNull(result.Call);
70+
Assert.Equal(tool.Name, result.Call.Name);
71+
Assert.NotNull(result.Outcome);
72+
Assert.Null(result.Outcome.Exception);
73+
}
74+
75+
[SecretsFact("OPENAI_API_KEY")]
76+
public async Task RunToolExceptionOutcome()
77+
{
78+
var chat = new Chat()
79+
{
80+
{ "system", "You make up a tool run by making up a name, description and content based on whatever the user says." },
81+
{ "user", "I want to create an order for a dozen eggs" },
82+
};
83+
84+
var client = new OpenAIChatClient(Configuration["OPENAI_API_KEY"]!, "gpt-4.1",
85+
OpenAI.OpenAIClientOptions.WriteTo(output))
86+
.AsBuilder()
87+
.UseFunctionInvocation()
88+
.Build();
89+
90+
var tool = ToolFactory.Create(RunToolThrows);
91+
var options = new ChatOptions
92+
{
93+
ToolMode = ChatToolMode.RequireSpecific(tool.Name),
94+
Tools = [tool]
95+
};
96+
97+
var response = await client.GetResponseAsync(chat, options);
98+
var result = response.FindCalls(tool).FirstOrDefault();
99+
100+
Assert.NotNull(result);
101+
Assert.NotNull(result.Call);
102+
Assert.Equal(tool.Name, result.Call.Name);
103+
Assert.NotNull(result.Outcome);
104+
Assert.NotNull(result.Outcome.Exception);
105+
}
106+
107+
[Description("Runs a tool to provide a result based on user input.")]
108+
ToolResult RunTool(
109+
[Description("The name")] string name,
110+
[Description("The description")] string description,
111+
[Description("The content")] string content)
112+
{
113+
// Simulate running a tool and returning a result
114+
return new ToolResult(name, description, content);
115+
}
116+
117+
[Description("Runs a tool to provide a result based on user input.")]
118+
ToolResult RunToolTerminate(
119+
[Description("The name")] string name,
120+
[Description("The description")] string description,
121+
[Description("The content")] string content)
122+
{
123+
FunctionInvokingChatClient.CurrentContext?.Terminate = true;
124+
// Simulate running a tool and returning a result
125+
return new ToolResult(name, description, content);
126+
}
127+
128+
[Description("Runs a tool to provide a result based on user input.")]
129+
ToolResult RunToolThrows(
130+
[Description("The name")] string name,
131+
[Description("The description")] string description,
132+
[Description("The content")] string content)
133+
{
134+
FunctionInvokingChatClient.CurrentContext?.Terminate = true;
135+
throw new ArgumentException("BOOM");
136+
}
137+
}

‎src/AI/ToolExtensions.cs‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
using System.Text.Json;
2+
using Microsoft.Extensions.AI;
3+
4+
namespace Devlooped.Extensions.AI;
5+
6+
/// <summary>
7+
/// Represents a tool call made by the AI, including the function call content and the result of the function execution.
8+
/// </summary>
9+
public record ToolCall(FunctionCallContent Call, FunctionResultContent Outcome);
10+
11+
/// <summary>
12+
/// Represents a tool call made by the AI, including the function call content, the result of the function execution,
13+
/// and the deserialized result of type <typeparamref name="TResult"/>.
14+
/// </summary>
15+
public record ToolCall<TResult>(FunctionCallContent Call, FunctionResultContent Outcome, TResult Result);
16+
17+
/// <summary>
18+
/// Extensions for inspecting chat messages and responses for tool
19+
/// usage and processing responses.
20+
/// </summary>
21+
public static class ToolExtensions
22+
{
23+
/// <summary>
24+
/// Looks for calls to a tool and their outcome.
25+
/// </summary>
26+
public static IEnumerable<ToolCall> FindCalls(this ChatResponse response, AIFunction tool)
27+
=> FindCalls(response.Messages, tool.Name);
28+
29+
/// <summary>
30+
/// Looks for calls to a tool and their outcome.
31+
/// </summary>
32+
public static IEnumerable<ToolCall> FindCalls(this IEnumerable<ChatMessage> messages, AIFunction tool)
33+
=> FindCalls(messages, tool.Name);
34+
35+
/// <summary>
36+
/// Looks for calls to a tool and their outcome.
37+
/// </summary>
38+
public static IEnumerable<ToolCall> FindCalls(this IEnumerable<ChatMessage> messages, string tool)
39+
{
40+
var calls = messages
41+
.Where(x => x.Role == ChatRole.Assistant)
42+
.SelectMany(x => x.Contents)
43+
.OfType<FunctionCallContent>()
44+
.Where(x => x.Name == tool)
45+
.ToDictionary(x => x.CallId);
46+
47+
var results = messages
48+
.Where(x => x.Role == ChatRole.Tool)
49+
.SelectMany(x => x.Contents)
50+
.OfType<FunctionResultContent>()
51+
.Where(x => calls.TryGetValue(x.CallId, out var call) && call.Name == tool)
52+
.Select(x => new ToolCall(calls[x.CallId], x));
53+
54+
return results;
55+
}
56+
57+
/// <summary>
58+
/// Looks for a user prompt in the chat response messages.
59+
/// </summary>
60+
/// <remarks>
61+
/// In order for this to work, the <see cref="AIFunctionFactory"/> must have been invoked using
62+
/// the <see cref="ToolJsonOptions.Default"/> or with a <see cref="JsonSerializerOptions"/> configured
63+
/// with <see cref="TypeInjectingResolverExtensions.WithTypeInjection(JsonSerializerOptions)"/> so
64+
/// that the tool result type can be properly inspected.
65+
/// </remarks>
66+
public static IEnumerable<ToolCall<TResult>> FindCalls<TResult>(this ChatResponse response, AIFunction tool)
67+
=> FindCalls<TResult>(response.Messages, tool.Name);
68+
69+
/// <summary>
70+
/// Looks for a user prompt in the chat response messages.
71+
/// </summary>
72+
/// <remarks>
73+
/// In order for this to work, the <see cref="AIFunctionFactory"/> must have been invoked using
74+
/// the <see cref="ToolJsonOptions.Default"/> or with a <see cref="JsonSerializerOptions"/> configured
75+
/// with <see cref="TypeInjectingResolverExtensions.WithTypeInjection(JsonSerializerOptions)"/> so
76+
/// that the tool result type can be properly inspected.
77+
/// </remarks>
78+
public static IEnumerable<ToolCall<TResult>> FindCalls<TResult>(this IEnumerable<ChatMessage> messages, AIFunction tool)
79+
=> FindCalls<TResult>(messages, tool.Name);
80+
81+
/// <summary>
82+
/// Looks for a user prompt in the chat response messages.
83+
/// </summary>
84+
/// <remarks>
85+
/// In order for this to work, the <see cref="AIFunctionFactory"/> must have been invoked using
86+
/// the <see cref="ToolJsonOptions.Default"/> or with a <see cref="JsonSerializerOptions"/> configured
87+
/// with <see cref="TypeInjectingResolverExtensions.WithTypeInjection(JsonSerializerOptions)"/> so
88+
/// that the tool result type can be properly inspected.
89+
/// </remarks>
90+
public static IEnumerable<ToolCall<TResult>> FindCalls<TResult>(this IEnumerable<ChatMessage> messages, string tool)
91+
{
92+
var calls = FindCalls(messages, tool)
93+
.Where(x => x.Outcome.Result is JsonElement element &&
94+
element.ValueKind == JsonValueKind.Object &&
95+
element.TryGetProperty("$type", out var type) &&
96+
type.GetString() == typeof(TResult).FullName)
97+
.Select(x => new ToolCall<TResult>(
98+
Call: x.Call,
99+
Outcome: x.Outcome,
100+
Result: JsonSerializer.Deserialize<TResult>((JsonElement)x.Outcome.Result!, ToolJsonOptions.Default) ??
101+
throw new InvalidOperationException($"Failed to deserialize result for tool '{tool}' to {typeof(TResult).FullName}.")));
102+
103+
return calls;
104+
}
105+
}

‎src/AI/ToolFactory.cs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
using Microsoft.Extensions.AI;
2+
3+
namespace Devlooped.Extensions.AI;
4+
5+
/// <summary>
6+
/// Creates tools for function calling that can leverage the <see cref="ToolExtensions"/>
7+
/// extension methods for locating invocations and their results.
8+
/// </summary>
9+
public static class ToolFactory
10+
{
11+
/// <summary>
12+
/// Invokes <see cref="AIFunctionFactory.Create(Delegate, string?, string?, System.Text.Json.JsonSerializerOptions?)"/>
13+
/// using the method name following the naming convention and serialization options from <see cref="ToolJsonOptions.Default"/>.
14+
/// </summary>
15+
public static AIFunction Create(Delegate method)
16+
=> AIFunctionFactory.Create(method,
17+
ToolJsonOptions.Default.PropertyNamingPolicy!.ConvertName(method.Method.Name),
18+
serializerOptions: ToolJsonOptions.Default);
19+
}

‎src/AI/ToolJsonOptions.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
using System.Diagnostics;
2+
using System.Text.Json;
3+
using System.Text.Json.Serialization;
4+
using System.Text.Json.Serialization.Metadata;
5+
6+
namespace Devlooped.Extensions.AI;
7+
8+
/// <summary>
9+
/// Provides a <see cref="JsonSerializerOptions"/> optimized for use with
10+
/// function calling and tools.
11+
/// </summary>
12+
public static class ToolJsonOptions
13+
{
14+
static ToolJsonOptions() => Default.MakeReadOnly();
15+
16+
/// <summary>
17+
/// Default <see cref="JsonSerializerOptions"/> for function calling and tools.
18+
/// </summary>
19+
public static JsonSerializerOptions Default { get; } = new(JsonSerializerDefaults.Web)
20+
{
21+
Converters =
22+
{
23+
new AdditionalPropertiesDictionaryConverter(),
24+
new JsonStringEnumConverter(),
25+
},
26+
DefaultIgnoreCondition =
27+
JsonIgnoreCondition.WhenWritingDefault |
28+
JsonIgnoreCondition.WhenWritingNull,
29+
Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
30+
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
31+
WriteIndented = Debugger.IsAttached,
32+
TypeInfoResolver = new TypeInjectingResolver(new DefaultJsonTypeInfoResolver())
33+
};
34+
}

0 commit comments

Comments
 (0)