Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 171 additions & 0 deletions packages/core/src/core/coreToolScheduler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2812,3 +2812,174 @@ describe('CoreToolScheduler plan mode with ask_user_question', () => {
expect(completedCalls[0].status).toBe('cancelled');
});
});

describe('CoreToolScheduler unicode-escaped tool args', () => {
class CapturingToolInvocation extends BaseToolInvocation<
Record<string, unknown>,
ToolResult
> {
constructor(params: Record<string, unknown>) {
super(params);
}

getDescription(): string {
return 'Capturing tool invocation';
}

async execute(_abortSignal: AbortSignal): Promise<ToolResult> {
return {
llmContent: 'Captured successfully',
returnDisplay: 'Captured successfully',
};
}
}

class CapturingTool extends BaseDeclarativeTool<
Record<string, unknown>,
ToolResult
> {
capturedBuildParams: Record<string, unknown> | undefined;

constructor(name: string) {
super(
name,
name,
'A tool used to capture normalized params in scheduler tests',
Kind.Read,
{
type: 'object',
properties: {
absolute_path: { type: 'string' },
command: { type: 'string' },
is_background: { type: 'boolean' },
},
},
);
}

protected createInvocation(
params: Record<string, unknown>,
): ToolInvocation<Record<string, unknown>, ToolResult> {
this.capturedBuildParams = params;
return new CapturingToolInvocation(params);
}
}

function createUnicodeDecodingScheduler(
tool: CapturingTool,
model = 'qwen3.5-plus',
) {
const mockToolRegistry = {
getTool: (name: string) => (name === tool.name ? tool : undefined),
getFunctionDeclarations: () => [],
tools: new Map(),
discovery: {},
registerTool: () => {},
getToolByName: () => tool,
getToolByDisplayName: () => tool,
getTools: () => [tool],
discoverTools: async () => {},
getAllTools: () => [tool],
getToolsByServer: () => [],
} as unknown as ToolRegistry;

const onAllToolCallsComplete = vi.fn();
const onToolCallsUpdate = vi.fn();

const mockConfig = {
getSessionId: () => 'test-session-id',
getUsageStatisticsEnabled: () => true,
getDebugMode: () => false,
getApprovalMode: () => ApprovalMode.DEFAULT,
getAllowedTools: () => [],
getModel: () => model,
getContentGeneratorConfig: () => ({
model,
authType: 'gemini',
}),
getShellExecutionConfig: () => ({
terminalWidth: 90,
terminalHeight: 30,
}),
storage: {
getProjectTempDir: () => '/tmp',
},
getTruncateToolOutputThreshold: () =>
DEFAULT_TRUNCATE_TOOL_OUTPUT_THRESHOLD,
getTruncateToolOutputLines: () => DEFAULT_TRUNCATE_TOOL_OUTPUT_LINES,
getToolRegistry: () => mockToolRegistry,
getUseModelRouter: () => false,
getGeminiClient: () => null,
getChatRecordingService: () => undefined,
isInteractive: () => true,
getIdeMode: () => false,
getExperimentalZedIntegration: () => false,
} as unknown as Config;

const scheduler = new CoreToolScheduler({
config: mockConfig,
onAllToolCallsComplete,
onToolCallsUpdate,
getPreferredEditor: () => 'vscode',
onEditorClose: vi.fn(),
});

return { scheduler, onAllToolCallsComplete, onToolCallsUpdate };
}

it('should decode unicode-escaped absolute_path for read_file calls on qwen3.5-plus', async () => {
const tool = new CapturingTool('read_file');
const { scheduler, onAllToolCallsComplete } =
createUnicodeDecodingScheduler(tool);

const abortController = new AbortController();
const request = {
callId: 'unicode-read-1',
name: 'read_file',
args: {
absolute_path: '/tmp/\\u4e2d\\u6587\\u4e2d\\u6587-1.md',
},
isClientInitiated: false,
prompt_id: 'prompt-unicode-read',
};

await scheduler.schedule([request], abortController.signal);

await vi.waitFor(() => {
expect(onAllToolCallsComplete).toHaveBeenCalled();
});

expect(tool.capturedBuildParams).toEqual({
absolute_path: '/tmp/中文中文-1.md',
});
});

it('should decode unicode-escaped command for run_shell_command calls on qwen3.5-plus', async () => {
const tool = new CapturingTool('run_shell_command');
const { scheduler, onAllToolCallsComplete } =
createUnicodeDecodingScheduler(tool);

const abortController = new AbortController();
const request = {
callId: 'unicode-shell-1',
name: 'run_shell_command',
args: {
command: "cat '/tmp/\\u4e2d\\u6587\\u4e2d\\u6587-1.md'",
is_background: false,
},
isClientInitiated: false,
prompt_id: 'prompt-unicode-shell',
};

await scheduler.schedule([request], abortController.signal);

await vi.waitFor(() => {
expect(onAllToolCallsComplete).toHaveBeenCalled();
});

expect(tool.capturedBuildParams).toEqual({
command: "cat '/tmp/中文中文-1.md'",
is_background: false,
});
});
});
107 changes: 95 additions & 12 deletions packages/core/src/core/coreToolScheduler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ import type {
PartListUnion,
} from '@google/genai';
import { ToolNames } from '../tools/tool-names.js';
import {
decodeUnicodeEscapedString,
shouldUseUnicodeEscapedPaths,
} from '../utils/unicodeEscaping.js';
import { getResponseTextFromParts } from '../utils/generateContentResponseUtilities.js';
import type { ModifyContext } from '../tools/modifiable-tool.js';
import {
Expand Down Expand Up @@ -306,6 +310,69 @@ const createErrorResponse = (
contentLength: error.message.length,
});

function getModelForUnicodeNormalization(config: Config): string | undefined {
const partialConfig = config as Partial<Config>;

if (typeof partialConfig.getModel === 'function') {
return partialConfig.getModel();
}

if (typeof partialConfig.getContentGeneratorConfig === 'function') {
return partialConfig.getContentGeneratorConfig()?.model;
}

return undefined;
}

function normalizeUnicodeEscapedToolArgs(
model: string | undefined,
toolName: string,
args: Record<string, unknown>,
): Record<string, unknown> {
if (!shouldUseUnicodeEscapedPaths(model)) {
return args;
}

let fieldsToNormalize: string[] = [];
switch (toolName) {
case ToolNames.READ_FILE:
fieldsToNormalize = ['absolute_path'];
break;
case ToolNames.WRITE_FILE:
case ToolNames.EDIT:
fieldsToNormalize = ['file_path'];
break;
case ToolNames.LS:
case ToolNames.GLOB:
case ToolNames.GREP:
fieldsToNormalize = ['path'];
break;
case ToolNames.SHELL:
fieldsToNormalize = ['command'];
break;
default:
return args;
}

let changed = false;
const normalizedArgs: Record<string, unknown> = { ...args };

for (const field of fieldsToNormalize) {
const value = normalizedArgs[field];
if (typeof value !== 'string') {
continue;
}

const decoded = decodeUnicodeEscapedString(value);
if (decoded !== value) {
normalizedArgs[field] = decoded;
changed = true;
}
}

return changed ? normalizedArgs : args;
}

export async function truncateAndSaveToFile(
content: string,
callId: string,
Expand Down Expand Up @@ -594,18 +661,21 @@ export class CoreToolScheduler {
return call;
}

const invocationOrError = this.buildInvocation(
call.tool,
const normalizedArgs = normalizeUnicodeEscapedToolArgs(
getModelForUnicodeNormalization(this.config),
call.tool.name,
args as Record<string, unknown>,
);

const invocationOrError = this.buildInvocation(call.tool, normalizedArgs);
if (invocationOrError instanceof Error) {
const response = createErrorResponse(
call.request,
invocationOrError,
ToolErrorType.INVALID_TOOL_PARAMS,
);
return {
request: { ...call.request, args: args as Record<string, unknown> },
request: { ...call.request, args: normalizedArgs },
status: 'error',
tool: call.tool,
response,
Expand All @@ -614,7 +684,7 @@ export class CoreToolScheduler {

return {
...call,
request: { ...call.request, args: args as Record<string, unknown> },
request: { ...call.request, args: normalizedArgs },
invocation: invocationOrError,
};
});
Expand Down Expand Up @@ -786,22 +856,32 @@ export class CoreToolScheduler {
};
}

const normalizedArgs = normalizeUnicodeEscapedToolArgs(
getModelForUnicodeNormalization(this.config),
reqInfo.name,
reqInfo.args,
);
const normalizedRequest =
normalizedArgs === reqInfo.args
? reqInfo
: { ...reqInfo, args: normalizedArgs };

const invocationOrError = this.buildInvocation(
toolInstance,
reqInfo.args,
normalizedRequest.args,
);
if (invocationOrError instanceof Error) {
const error = reqInfo.wasOutputTruncated
const error = normalizedRequest.wasOutputTruncated
? new Error(
`${invocationOrError.message} ${TRUNCATION_PARAM_GUIDANCE}`,
)
: invocationOrError;
return {
status: 'error',
request: reqInfo,
request: normalizedRequest,
tool: toolInstance,
response: createErrorResponse(
reqInfo,
normalizedRequest,
error,
ToolErrorType.INVALID_TOOL_PARAMS,
),
Expand All @@ -811,14 +891,17 @@ export class CoreToolScheduler {

// Reject file-modifying calls when truncated to prevent
// writing incomplete content.
if (reqInfo.wasOutputTruncated && toolInstance.kind === Kind.Edit) {
if (
normalizedRequest.wasOutputTruncated &&
toolInstance.kind === Kind.Edit
) {
const truncationError = new Error(TRUNCATION_EDIT_REJECTION);
return {
status: 'error',
request: reqInfo,
request: normalizedRequest,
tool: toolInstance,
response: createErrorResponse(
reqInfo,
normalizedRequest,
truncationError,
ToolErrorType.OUTPUT_TRUNCATED,
),
Expand All @@ -828,7 +911,7 @@ export class CoreToolScheduler {

return {
status: 'validating',
request: reqInfo,
request: normalizedRequest,
tool: toolInstance,
invocation: invocationOrError,
startTime: Date.now(),
Expand Down
11 changes: 11 additions & 0 deletions packages/core/src/core/prompts.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,17 @@ describe('Core System Prompt (prompts.ts)', () => {
vi.stubEnv('QWEN_WRITE_SYSTEM_MD', undefined);
});

it('should include unicode path handling instructions for qwen3.5-plus', () => {
const prompt = getCoreSystemPrompt(undefined, 'qwen3.5-plus');
expect(prompt).toContain('# Unicode Path Handling');
expect(prompt).toContain('\\u4e2d\\u6587\\u4e2d\\u6587-1.md');
});

it('should include unicode path handling instructions for qwen3.5-397B-A17B', () => {
const prompt = getCoreSystemPrompt(undefined, 'qwen3.5-397B-A17B');
expect(prompt).toContain('# Unicode Path Handling');
});

it('should return the base prompt when no userMemory is provided', () => {
vi.stubEnv('SANDBOX', undefined);
const prompt = getCoreSystemPrompt();
Expand Down
Loading