Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
622 changes: 622 additions & 0 deletions docs/design/multi-model-review.md

Large diffs are not rendered by default.

8 changes: 8 additions & 0 deletions packages/cli/src/config/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1035,6 +1035,14 @@ export async function loadCliConfig(
},
hooks: settings.hooks,
hooksConfig: settings.hooksConfig,
reviewConfig: settings.review
? {
models: settings.review.models as
| Array<string | Record<string, unknown>>
| undefined,
arbitratorModel: settings.review.arbitratorModel,
}
: undefined,
enableHooks:
argv.experimentalHooks === true || settings.hooksConfig?.enabled === true,
channel: argv.channel,
Expand Down
60 changes: 60 additions & 0 deletions packages/cli/src/config/settingsSchema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ export interface SettingDefinition {
mergeStrategy?: MergeStrategy;
/** Enum type options */
options?: readonly SettingEnumOption[];
/** Custom JSON Schema for array items (overrides default `{ type: 'string' }`) */
items?: Record<string, unknown>;
}

export interface SettingsSchema {
Expand Down Expand Up @@ -1247,6 +1249,64 @@ const SETTINGS_SCHEMA = {
},
},
},
// Multi-model code review configuration
review: {
type: 'object',
label: 'Code Review',
category: 'Tools',
requiresRestart: false,
default: {},
description:
'Multi-model code review configuration. When review.models is configured with 2+ models, /review will use multi-model review automatically.',
showInDialog: false,
properties: {
models: {
type: 'array',
label: 'Review Models',
category: 'Tools',
requiresRestart: false,
default: [] as Array<string | Record<string, unknown>>,
description:
'Models for multi-model review. Each entry can be a model ID string (resolved from modelProviders) or a full model config object with id, authType, baseUrl, envKey.',
showInDialog: false,
items: {
oneOf: [
{
type: 'string',
description: 'Model ID resolved from modelProviders',
},
{
type: 'object',
description: 'Inline model configuration',
properties: {
id: { type: 'string', description: 'Model identifier' },
authType: {
type: 'string',
description: 'Authentication type',
},
baseUrl: { type: 'string', description: 'API base URL' },
envKey: {
type: 'string',
description: 'Environment variable for API key',
},
},
required: ['id'],
},
],
},
},
arbitratorModel: {
type: 'string',
label: 'Arbitrator Model',
category: 'Tools',
requiresRestart: false,
default: undefined,
description:
'Model ID for the final arbitrator (resolved from modelProviders). Falls back to the current session model if not set. Recommended: a high-reasoning model.',
showInDialog: false,
},
},
},
} as const satisfies SettingsSchema;

export type SettingsSchemaType = typeof SETTINGS_SCHEMA;
Expand Down
283 changes: 283 additions & 0 deletions packages/core/src/config/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1313,6 +1313,289 @@ describe('BaseLlmClient Lifecycle', () => {
});
});

describe('Review Model Config Resolution', () => {
const MODEL = 'qwen3-coder-plus';
const TELEMETRY_SETTINGS = { enabled: false };

const reviewBaseParams: ConfigParameters = {
cwd: '/tmp',
targetDir: '/tmp',
debugMode: false,
question: '',
model: MODEL,
usageStatisticsEnabled: false,
telemetry: TELEMETRY_SETTINGS,
overrideExtensions: [],
};

beforeEach(() => {
vi.clearAllMocks();
vi.mocked(canUseRipgrep).mockResolvedValue(true);
vi.spyOn(QwenLogger.prototype, 'logStartSessionEvent').mockImplementation(
async () => undefined,
);
vi.mocked(resolveContentGeneratorConfigWithSources).mockImplementation(
(_config, authType, generationConfig) => ({
config: {
...generationConfig,
authType,
model: generationConfig?.model || MODEL,
apiKey: 'test-key',
} as ContentGeneratorConfig,
sources: {},
}),
);
});

describe('getReviewModels', () => {
it('should return empty array when no reviewConfig', () => {
const config = new Config({ ...reviewBaseParams });
expect(config.getReviewModels()).toEqual([]);
});

it('should return empty array when models is empty', () => {
const config = new Config({
...reviewBaseParams,
reviewConfig: { models: [] },
});
expect(config.getReviewModels()).toEqual([]);
});

it('should resolve string model IDs from modelProviders', () => {
const config = new Config({
...reviewBaseParams,
modelProvidersConfig: {
openai: [
{
id: 'gpt-4o',
name: 'GPT-4o',
baseUrl: 'https://api.openai.com/v1',
},
],
},
reviewConfig: { models: ['gpt-4o'] },
});
const models = config.getReviewModels();
expect(models).toHaveLength(1);
expect(models[0].id).toBe('gpt-4o');
expect(models[0].authType).toBe(AuthType.USE_OPENAI);
});

it('should throw for string model ID not found in modelProviders', () => {
const config = new Config({
...reviewBaseParams,
modelProvidersConfig: {},
reviewConfig: { models: ['non-existent'] },
});
expect(() => config.getReviewModels()).toThrow(
/Model 'non-existent' not found in modelProviders/,
);
});

it('should deduplicate string model IDs', () => {
const config = new Config({
...reviewBaseParams,
modelProvidersConfig: {
openai: [
{
id: 'gpt-4o',
name: 'GPT-4o',
baseUrl: 'https://api.openai.com/v1',
},
],
},
reviewConfig: { models: ['gpt-4o', 'gpt-4o'] },
});
const models = config.getReviewModels();
expect(models).toHaveLength(1);
});

it('should resolve inline object model configs', () => {
const config = new Config({
...reviewBaseParams,
reviewConfig: {
models: [
{
id: 'custom-model',
authType: 'openai',
baseUrl: 'https://custom.api/v1',
envKey: 'CUSTOM_KEY',
},
],
},
});
const models = config.getReviewModels();
expect(models).toHaveLength(1);
expect(models[0].id).toBe('custom-model');
expect(models[0].authType).toBe(AuthType.USE_OPENAI);
expect(models[0].baseUrl).toBe('https://custom.api/v1');
expect(models[0].envKey).toBe('CUSTOM_KEY');
});

it('should throw for inline object missing authType', () => {
const config = new Config({
...reviewBaseParams,
reviewConfig: {
models: [{ id: 'no-auth' }],
},
});
expect(() => config.getReviewModels()).toThrow(
/missing required field: authType/,
);
});

it('should throw for inline object with invalid authType', () => {
const config = new Config({
...reviewBaseParams,
reviewConfig: {
models: [{ id: 'bad-auth', authType: 'invalid-provider' }],
},
});
expect(() => config.getReviewModels()).toThrow(/invalid authType/);
});

it('should throw for inline object missing baseUrl when required', () => {
const config = new Config({
...reviewBaseParams,
reviewConfig: {
models: [{ id: 'no-url', authType: 'openai' }],
},
});
expect(() => config.getReviewModels()).toThrow(/requires a baseUrl/);
});

it('should allow missing baseUrl for qwen-oauth authType', () => {
const config = new Config({
...reviewBaseParams,
reviewConfig: {
models: [{ id: 'qwen-model', authType: 'qwen-oauth' }],
},
});
const models = config.getReviewModels();
expect(models).toHaveLength(1);
expect(models[0].baseUrl).toBe('');
});

it('should handle mixed string and object entries', () => {
const config = new Config({
...reviewBaseParams,
modelProvidersConfig: {
openai: [
{
id: 'gpt-4o',
name: 'GPT-4o',
baseUrl: 'https://api.openai.com/v1',
},
],
},
reviewConfig: {
models: [
'gpt-4o',
{
id: 'custom-model',
authType: 'openai',
baseUrl: 'https://custom.api/v1',
},
],
},
});
const models = config.getReviewModels();
expect(models).toHaveLength(2);
expect(models[0].id).toBe('gpt-4o');
expect(models[1].id).toBe('custom-model');
});

it('should deduplicate across string and object entries', () => {
const config = new Config({
...reviewBaseParams,
modelProvidersConfig: {
openai: [
{
id: 'gpt-4o',
name: 'GPT-4o',
baseUrl: 'https://api.openai.com/v1',
},
],
},
reviewConfig: {
models: [
'gpt-4o',
{
id: 'gpt-4o',
authType: 'openai',
baseUrl: 'https://api.openai.com/v1',
},
],
},
});
const models = config.getReviewModels();
expect(models).toHaveLength(1);
});

it('should use id as name when name is not provided in inline config', () => {
const config = new Config({
...reviewBaseParams,
reviewConfig: {
models: [
{
id: 'my-model',
authType: 'openai',
baseUrl: 'https://api.example.com/v1',
},
],
},
});
const models = config.getReviewModels();
expect(models[0].name).toBe('my-model');
});
});

describe('getArbitratorModel', () => {
it('should return undefined when no arbitratorModel configured', () => {
const config = new Config({ ...reviewBaseParams });
expect(config.getArbitratorModel()).toBeUndefined();
});

it('should return undefined when arbitratorModel is empty string', () => {
const config = new Config({
...reviewBaseParams,
reviewConfig: { arbitratorModel: '' },
});
expect(config.getArbitratorModel()).toBeUndefined();
});

it('should resolve arbitratorModel from modelProviders', () => {
const config = new Config({
...reviewBaseParams,
modelProvidersConfig: {
openai: [
{
id: 'gpt-4o',
name: 'GPT-4o',
baseUrl: 'https://api.openai.com/v1',
},
],
},
reviewConfig: { arbitratorModel: 'gpt-4o' },
});
const model = config.getArbitratorModel();
expect(model).toBeDefined();
expect(model?.id).toBe('gpt-4o');
});

it('should throw when arbitratorModel not found in modelProviders', () => {
const config = new Config({
...reviewBaseParams,
modelProvidersConfig: {},
reviewConfig: { arbitratorModel: 'non-existent' },
});
expect(() => config.getArbitratorModel()).toThrow(
/Arbitrator model 'non-existent' not found/,
);
});
});
});

describe('Model Switching and Config Updates', () => {
const baseParams: ConfigParameters = {
cwd: '/tmp',
Expand Down
Loading
Loading