Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 134 additions & 0 deletions ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
getEntityBreakdown,
handleExportCSV,
handleExportJSON,
resolveEntities,
} from "./utils";

vi.mock("@/utils/dataUtils", () => ({
Expand Down Expand Up @@ -1561,4 +1562,137 @@ describe("EntityUsageExport utils", () => {
window.Blob = originalBlob;
});
});

describe("resolveEntities and aggregated endpoint fallback", () => {
// Simulates the response from /user/daily/activity/aggregated which has
// empty entities but populated api_keys at the breakdown level.
// Derived from mockSpendData: flatten all entities' api_key_breakdowns
// into top-level api_keys, clear entities, and add a second key for team-1
// to test multi-key grouping.
const aggregatedSpendData: EntitySpendData = {
...mockSpendData,
results: mockSpendData.results.slice(0, 1).map((day) => ({
...day,
breakdown: {
entities: {},
api_keys: {
...Object.fromEntries(
Object.values(day.breakdown.entities as Record<string, any>).flatMap((e: any) =>
Object.entries(e.api_key_breakdown || {}),
),
),
// Extra key on team-1 to test multi-key-per-team aggregation
key1b: {
metrics: { spend: 5, api_requests: 50, successful_requests: 48, failed_requests: 2, total_tokens: 500 },
metadata: { team_id: "team-1", key_alias: "staging-key" },
},
},
models: { "gpt-4": { metrics: { spend: 35, api_requests: 350, total_tokens: 3500 } } },
},
})),
};

describe("resolveEntities", () => {
it("should return entities when populated", () => {
const breakdown = {
entities: { e1: { metrics: { spend: 1 } } },
api_keys: { k1: { metrics: { spend: 2 }, metadata: { team_id: "t1" } } },
};
const result = resolveEntities(breakdown);
expect(result).toBe(breakdown.entities);
});

it("should aggregate api_keys into entities when entities is empty", () => {
const breakdown = aggregatedSpendData.results[0].breakdown;
const result = resolveEntities(breakdown);

// Two teams: team-1 (key1+key2) and team-2 (key3)
expect(Object.keys(result)).toHaveLength(2);
expect(result["team-1"]).toBeDefined();
expect(result["team-2"]).toBeDefined();

// team-1 spend = 10.5 (key1) + 5 (key1b)
expect(result["team-1"].metrics.spend).toBe(15.5);
expect(result["team-1"].metrics.api_requests).toBe(150);
expect(result["team-1"].metrics.total_tokens).toBe(1500);

// team-2 spend = 20.3 (key2)
expect(result["team-2"].metrics.spend).toBe(20.3);
expect(result["team-2"].metrics.api_requests).toBe(200);
});

it("should use 'Unassigned' for keys without team_id", () => {
const breakdown = {
entities: {},
api_keys: {
k1: {
metrics: { spend: 7, api_requests: 10, successful_requests: 10, failed_requests: 0, total_tokens: 100 },
metadata: {},
},
},
};
const result = resolveEntities(breakdown);
expect(result["Unassigned"]).toBeDefined();
expect(result["Unassigned"].metrics.spend).toBe(7);
});

it("should handle missing or empty api_keys gracefully", () => {
expect(Object.keys(resolveEntities({ entities: {}, api_keys: {} }))).toHaveLength(0);
expect(Object.keys(resolveEntities({ entities: {} }))).toHaveLength(0);
});

it("should preserve api_key_breakdown on aggregated entities", () => {
const breakdown = aggregatedSpendData.results[0].breakdown;
const result = resolveEntities(breakdown);

// team-1 should have key1 and key1b in api_key_breakdown
expect(Object.keys(result["team-1"].api_key_breakdown)).toEqual(["key1", "key1b"]);
// team-2 should have key2
expect(Object.keys(result["team-2"].api_key_breakdown)).toEqual(["key2"]);
});
});

describe("getEntityBreakdown with aggregated data", () => {
it("should produce breakdown from api_keys when entities is empty", () => {
const result = getEntityBreakdown(aggregatedSpendData);
expect(result.length).toBeGreaterThan(0);

// Sorted by spend desc: team-2 (20.3) then team-1 (15.5)
expect(result[0].metrics.spend).toBe(20.3);
expect(result[1].metrics.spend).toBe(15.5);
});

});

describe("generateDailyData with aggregated data", () => {
it("should produce rows from api_keys when entities is empty", () => {
const result = generateDailyData(aggregatedSpendData, "Team");
expect(result.length).toBeGreaterThan(0);
expect(result[0]).toHaveProperty("Date");
expect(result[0]).toHaveProperty("Team");
});
});

describe("generateDailyWithKeysData with aggregated data", () => {
it("should produce rows from api_keys when entities is empty", () => {
const result = generateDailyWithKeysData(aggregatedSpendData, "Team");
expect(result.length).toBeGreaterThan(0);
Comment on lines +1672 to +1679
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Weak assertion for model data

This test only verifies that rows are produced and that the "Model" property exists — it does not verify actual spend or request values per model. The underlying generateDailyWithModelsData logic adds all API key metrics to every model in breakdown.models. With only one model in the test fixture ("gpt-4") the multiplication factor is 1, so values happen to be correct by coincidence. If a second model were added to aggregatedSpendData.breakdown.models, each model row would carry the full team spend rather than its own share, and this test would not catch it.

Consider adding assertions on the spend values per model to catch regressions if the metric attribution logic changes:

expect(result.find(r => r.Model === "gpt-4")).toBeDefined();
// Verify spend is not double-counted when multiple models exist


// Should have 3 key rows (key1, key1b, key2)
expect(result).toHaveLength(3);
const keyIds = result.map((r) => r["Key ID"]);
expect(keyIds).toContain("key1");
expect(keyIds).toContain("key1b");
expect(keyIds).toContain("key2");
});
});

describe("generateDailyWithModelsData with aggregated data", () => {
it("should produce rows from api_keys when entities is empty", () => {
const result = generateDailyWithModelsData(aggregatedSpendData, "Team");
expect(result.length).toBeGreaterThan(0);
expect(result[0]).toHaveProperty("Model");
});
});
});
});
53 changes: 48 additions & 5 deletions ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,57 @@ const extractTeamIdFromApiKeyBreakdown = (apiKeyBreakdown: Record<string, any> |
return null;
};

// Mirrors backend SpendMetrics fields (litellm/types/activity_tracking.py).
// If the backend adds a field, add it here too.
const METRIC_KEYS = [
"spend", "api_requests", "successful_requests", "failed_requests",
"total_tokens", "prompt_tokens", "completion_tokens",
"cache_read_input_tokens", "cache_creation_input_tokens",
] as const;

// When breakdown.entities is empty (aggregated endpoint), reconstruct entities
// from breakdown.api_keys by grouping on metadata.team_id.
const aggregateApiKeysIntoEntities = (breakdown: Record<string, any>): Record<string, any> => {
const apiKeys = breakdown.api_keys;
if (!apiKeys || Object.keys(apiKeys).length === 0) return {};

const grouped: Record<string, any> = {};

for (const [keyId, keyData] of Object.entries<any>(apiKeys)) {
const teamId = keyData?.metadata?.team_id || "Unassigned";
if (!grouped[teamId]) {
grouped[teamId] = {
metrics: Object.fromEntries(METRIC_KEYS.map((k) => [k, 0])),
api_key_breakdown: {},
};
}
const m = grouped[teamId].metrics;
const km = keyData?.metrics || {};
for (const k of METRIC_KEYS) {
m[k] += km[k] || 0;
}
grouped[teamId].api_key_breakdown[keyId] = keyData;
}

return grouped;
};

// Returns breakdown.entities if populated, otherwise falls back to
// reconstructing entities from breakdown.api_keys.
export const resolveEntities = (breakdown: Record<string, any>): Record<string, any> => {
const entities = breakdown.entities;
if (entities && Object.keys(entities).length > 0) return entities;
return aggregateApiKeysIntoEntities(breakdown);
};

export const getEntityBreakdown = (
spendData: EntitySpendData,
teamAliasMap: Record<string, string> = {},
): EntityBreakdown[] => {
const entitySpend: { [key: string]: EntityBreakdown } = {};

spendData.results.forEach((day) => {
Object.entries(day.breakdown.entities || {}).forEach(([entity, data]: [string, any]) => {
Object.entries(resolveEntities(day.breakdown)).forEach(([entity, data]: [string, any]) => {
// Extract team_id from api_key_breakdown metadata (not data.metadata which is empty)
const teamId = extractTeamIdFromApiKeyBreakdown(data.api_key_breakdown) || entity;
// Extract key_alias from the first API key that has one
Expand Down Expand Up @@ -80,7 +123,7 @@ export const generateDailyData = (
const dailyBreakdown: any[] = [];

spendData.results.forEach((day) => {
Object.entries(day.breakdown.entities || {}).forEach(([entity, data]: [string, any]) => {
Object.entries(resolveEntities(day.breakdown)).forEach(([entity, data]: [string, any]) => {
// Extract team_id from api_key_breakdown metadata (not data.metadata which is empty)
const teamId = extractTeamIdFromApiKeyBreakdown(data.api_key_breakdown);
const teamAlias = teamId ? teamAliasMap[teamId] || null : null;
Expand Down Expand Up @@ -129,7 +172,7 @@ export const generateDailyWithKeysData = (
} = {};

spendData.results.forEach((day) => {
Object.entries(day.breakdown.entities || {}).forEach(([entity, data]: [string, any]) => {
Object.entries(resolveEntities(day.breakdown)).forEach(([entity, data]: [string, any]) => {
const apiKeyBreakdown = data.api_key_breakdown || {};

// Iterate through each API key in the breakdown
Expand Down Expand Up @@ -202,7 +245,7 @@ export const generateDailyWithModelsData = (
spendData.results.forEach((day) => {
const dailyEntityModels: { [key: string]: { [key: string]: any } } = {};

Object.entries(day.breakdown.entities || {}).forEach(([entity, entityData]: [string, any]) => {
Object.entries(resolveEntities(day.breakdown)).forEach(([entity, entityData]: [string, any]) => {
if (!dailyEntityModels[entity]) {
dailyEntityModels[entity] = {};
}
Expand Down Expand Up @@ -230,7 +273,7 @@ export const generateDailyWithModelsData = (
});

Object.entries(dailyEntityModels).forEach(([entity, models]) => {
const entityData = day.breakdown.entities?.[entity];
const entityData = resolveEntities(day.breakdown)[entity];
Comment on lines 275 to +276
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Redundant resolveEntities call

resolveEntities(day.breakdown) is called a second time here (line 276), after already being called in the outer forEach at line 248. When the fallback path is taken, this re-runs the entire aggregateApiKeysIntoEntities aggregation for every entity in the second loop. Caching the result in a variable would prevent the redundant work.

Suggested change
Object.entries(dailyEntityModels).forEach(([entity, models]) => {
const entityData = day.breakdown.entities?.[entity];
const entityData = resolveEntities(day.breakdown)[entity];
const resolvedEntities = resolveEntities(day.breakdown);
Object.entries(resolvedEntities).forEach(([entity, entityData]: [string, any]) => {

Then replace both usages of resolveEntities(day.breakdown) in this function body with resolvedEntities.

// Extract team_id from api_key_breakdown metadata (not entityData.metadata which is empty)
const teamId = extractTeamIdFromApiKeyBreakdown(entityData?.api_key_breakdown);
const teamAlias = teamId ? teamAliasMap[teamId] || null : null;
Expand Down
Loading