[Test] UI - E2E: Add Playwright tests with local PostgreSQL

tests/ui_e2e_tests/constants.ts

@@ -0,0 +1,40 @@
+export const ADMIN_STORAGE_PATH = "admin.storageState.json";
+
+// Page enum — maps to ?page= query parameter values in the UI
+export enum Page {
+  ApiKeys = "api-keys",
+  Teams = "teams",
+  AdminSettings = "settings",
+}
+
+// Test user credentials — all users have password "test" (hashed in seed.sql)
+export enum Role {
+  ProxyAdmin = "proxy_admin",
+  ProxyAdminViewer = "proxy_admin_viewer",
+  InternalUser = "internal_user",
+  InternalUserViewer = "internal_user_viewer",
+  TeamAdmin = "team_admin",
+}
+
+export const users: Record<Role, { email: string; password: string }> = {
+  [Role.ProxyAdmin]: {
+    email: "admin",
+    password: process.env.LITELLM_MASTER_KEY || "sk-1234",
+  },
+  [Role.ProxyAdminViewer]: {
+    email: "adminviewer@test.local",
+    password: "test",
+  },
+  [Role.InternalUser]: {
+    email: "internal@test.local",
+    password: "test",
+  },
+  [Role.InternalUserViewer]: {
+    email: "viewer@test.local",
+    password: "test",
+  },
+  [Role.TeamAdmin]: {
+    email: "teamadmin@test.local",
+    password: "test",
+  },
+};

tests/ui_e2e_tests/fixtures/config.yml

@@ -0,0 +1,16 @@
+model_list:
+  - model_name: fake-openai-gpt-4
+    litellm_params:
+      model: openai/fake-gpt-4
+      api_base: os.environ/MOCK_LLM_URL
+      api_key: fake-key
+  - model_name: fake-anthropic-claude
+    litellm_params:
+      model: openai/fake-claude
+      api_base: os.environ/MOCK_LLM_URL
+      api_key: fake-key
+
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+  database_url: os.environ/DATABASE_URL
+  store_prompts_in_spend_logs: true

tests/ui_e2e_tests/fixtures/mock_llm_server/server.py

@@ -0,0 +1,118 @@
+"""
+Mock LLM server for UI e2e tests.
+Responds to OpenAI-format endpoints with canned responses.
+"""
+
+import time
+import json
+import uuid
+
+import uvicorn
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse
+
+
+app = FastAPI(title="Mock LLM Server")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/health")
+async def health():
+    return {"status": "ok"}
+
+
+@app.get("/v1/models")
+@app.get("/models")
+async def list_models():
+    return {
+        "object": "list",
+        "data": [
+            {"id": "fake-gpt-4", "object": "model", "owned_by": "mock"},
+            {"id": "fake-claude", "object": "model", "owned_by": "mock"},
+        ],
+    }
+
+
+@app.post("/v1/chat/completions")
+@app.post("/chat/completions")
+async def chat_completions(request: Request):
+    body = await request.json()
+    model = body.get("model", "mock-model")
+    stream = body.get("stream", False)
+
+    response_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
+    created = int(time.time())
+
+    if stream:
+        async def stream_generator():
+            chunk = {
+                "id": response_id,
+                "object": "chat.completion.chunk",
+                "created": created,
+                "model": model,
+                "choices": [
+                    {
+                        "index": 0,
+                        "delta": {"role": "assistant", "content": "This is a mock response."},
+                        "finish_reason": None,
+                    }
+                ],
+            }
+            yield f"data: {json.dumps(chunk)}\n\n"
+
+            done_chunk = {
+                "id": response_id,
+                "object": "chat.completion.chunk",
+                "created": created,
+                "model": model,
+                "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
+            }
+            yield f"data: {json.dumps(done_chunk)}\n\n"
+            yield "data: [DONE]\n\n"
+
+        return StreamingResponse(
+            stream_generator(), media_type="text/event-stream"
+        )
+
+    return {
+        "id": response_id,
+        "object": "chat.completion",
+        "created": created,
+        "model": model,
+        "choices": [
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": "This is a mock response."},
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18},
+    }
+
+
+@app.post("/v1/embeddings")
+@app.post("/embeddings")
+async def embeddings(request: Request):
+    body = await request.json()
+    inputs = body.get("input", [""])
+    if isinstance(inputs, str):
+        inputs = [inputs]
+    return {
+        "object": "list",
+        "data": [
+            {"object": "embedding", "index": i, "embedding": [0.0] * 1536}
+            for i in range(len(inputs))
+        ],
+        "model": body.get("model", "mock-embedding"),
+        "usage": {"prompt_tokens": 5, "total_tokens": 5},
+    }
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="127.0.0.1", port=8090)

tests/ui_e2e_tests/fixtures/seed.sql

@@ -0,0 +1,103 @@
+-- UI E2E Test Database Seed
+-- Run with: psql $DATABASE_URL -f seed.sql
+
+-- ============================================================
+-- 1. Budget Table (must be first — referenced by org FK)
+-- ============================================================
+INSERT INTO "LiteLLM_BudgetTable" (
+    budget_id, max_budget, created_by, updated_by
+) VALUES (
+    'e2e-budget-org', 1000.0, 'e2e-proxy-admin', 'e2e-proxy-admin'
+) ON CONFLICT (budget_id) DO NOTHING;
+
+-- ============================================================
+-- 2. Organization
+-- ============================================================
+INSERT INTO "LiteLLM_OrganizationTable" (
+    organization_id, organization_alias, budget_id, metadata, models, spend,
+    model_spend, created_by, updated_by
+) VALUES (
+    'e2e-org-main', 'E2E Organization', 'e2e-budget-org', '{}'::jsonb,
+    ARRAY[]::text[], 0.0, '{}'::jsonb, 'e2e-proxy-admin', 'e2e-proxy-admin'
+) ON CONFLICT (organization_id) DO NOTHING;
+
+-- ============================================================
+-- 3. Users (password is scrypt hash of "test")
+-- ============================================================
+INSERT INTO "LiteLLM_UserTable" (
+    user_id, user_email, user_role, password, teams, models, metadata,
+    spend, model_spend, model_max_budget
+) VALUES
+(
+    'e2e-proxy-admin', 'admin@test.local', 'proxy_admin', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
+    ARRAY['e2e-team-crud']::text[], ARRAY[]::text[], '{}'::jsonb,
+    0.0, '{}'::jsonb, '{}'::jsonb
+),
+(
+    'e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
+    ARRAY[]::text[], ARRAY[]::text[], '{}'::jsonb,
+    0.0, '{}'::jsonb, '{}'::jsonb
+),
+(
+    'e2e-internal-user', 'internal@test.local', 'internal_user', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
+    ARRAY['e2e-team-crud', 'e2e-team-org']::text[], ARRAY[]::text[], '{}'::jsonb,
+    0.0, '{}'::jsonb, '{}'::jsonb
+),
+(
+    'e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
+    ARRAY[]::text[], ARRAY[]::text[], '{}'::jsonb,
+    0.0, '{}'::jsonb, '{}'::jsonb
+),
+(
+    'e2e-team-admin', 'teamadmin@test.local', 'internal_user', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
+    ARRAY['e2e-team-crud', 'e2e-team-delete']::text[], ARRAY[]::text[], '{}'::jsonb,
+    0.0, '{}'::jsonb, '{}'::jsonb
+)
+ON CONFLICT (user_id) DO NOTHING;
+
+-- ============================================================
+-- 4. Teams
+-- ============================================================
+INSERT INTO "LiteLLM_TeamTable" (
+    team_id, team_alias, organization_id, admins, members,
+    members_with_roles, metadata, models, spend, model_spend,
+    model_max_budget, blocked
+) VALUES
+(
+    'e2e-team-crud', 'E2E Team CRUD', NULL,
+    ARRAY['e2e-team-admin']::text[],
+    ARRAY['e2e-team-admin', 'e2e-internal-user']::text[],
+    '[{"role": "admin", "user_id": "e2e-team-admin"}, {"role": "user", "user_id": "e2e-internal-user"}]'::jsonb,
+    '{}'::jsonb,
+    ARRAY['fake-openai-gpt-4', 'fake-anthropic-claude']::text[],
+    0.0, '{}'::jsonb, '{}'::jsonb, false
+),
+(
+    'e2e-team-delete', 'E2E Team Delete', NULL,
+    ARRAY['e2e-team-admin']::text[],
+    ARRAY['e2e-team-admin']::text[],
+    '[{"role": "admin", "user_id": "e2e-team-admin"}]'::jsonb,
+    '{}'::jsonb,
+    ARRAY['fake-openai-gpt-4']::text[],
+    0.0, '{}'::jsonb, '{}'::jsonb, false
+),
+(
+    'e2e-team-org', 'E2E Team In Org', 'e2e-org-main',
+    ARRAY[]::text[],
+    ARRAY['e2e-internal-user']::text[],
+    '[{"role": "user", "user_id": "e2e-internal-user"}]'::jsonb,
+    '{}'::jsonb,
+    ARRAY['fake-openai-gpt-4']::text[],
+    0.0, '{}'::jsonb, '{}'::jsonb, false
+)
+ON CONFLICT (team_id) DO NOTHING;
+
+-- ============================================================
+-- 5. Team Memberships
+-- ============================================================
+INSERT INTO "LiteLLM_TeamMembership" (user_id, team_id, spend) VALUES
+    ('e2e-team-admin', 'e2e-team-crud', 0.0),
+    ('e2e-internal-user', 'e2e-team-crud', 0.0),
+    ('e2e-team-admin', 'e2e-team-delete', 0.0),
+    ('e2e-internal-user', 'e2e-team-org', 0.0)
+ON CONFLICT (user_id, team_id) DO NOTHING;

tests/ui_e2e_tests/globalSetup.ts

@@ -0,0 +1,32 @@
+import { chromium, expect } from "@playwright/test";
+import { users, Role, ADMIN_STORAGE_PATH } from "./constants";
+import * as fs from "fs";
+
+async function globalSetup() {
+  const browser = await chromium.launch();
+  const page = await browser.newPage();
+  await page.goto("http://localhost:4000/ui/login");
+  await page.getByPlaceholder("Enter your username").fill(users[Role.ProxyAdmin].email);
+  await page.getByPlaceholder("Enter your password").fill(users[Role.ProxyAdmin].password);
+  await page.getByRole("button", { name: "Login", exact: true }).click();
+  try {
+    // Wait for navigation away from login page into the dashboard
+    await page.waitForURL(
+      (url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"),
+      { timeout: 30_000 },
+    );
+    // Wait for sidebar to render as a signal that the dashboard is ready
+    await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible({ timeout: 30_000 });
+  } catch (e) {
+    // Save a screenshot for debugging before re-throwing
+    fs.mkdirSync("test-results", { recursive: true });
+    await page.screenshot({ path: "test-results/global-setup-failure.png", fullPage: true });
+    console.error("Global setup failed. Screenshot saved to test-results/global-setup-failure.png");
+    console.error("Current URL:", page.url());
+    throw e;
+  }
+  await page.context().storageState({ path: ADMIN_STORAGE_PATH });
+  await browser.close();
+}
+
+export default globalSetup;

tests/ui_e2e_tests/helpers/login.ts

@@ -0,0 +1,16 @@
+import { Page as PlaywrightPage, expect } from "@playwright/test";
+import { users, Role } from "../constants";
+
+export async function loginAs(page: PlaywrightPage, role: Role) {
+  const user = users[role];
+  await page.goto("/ui/login");
+  await page.getByPlaceholder("Enter your username").fill(user.email);
+  await page.getByPlaceholder("Enter your password").fill(user.password);
+  await page.getByRole("button", { name: "Login", exact: true }).click();
+  // Wait for navigation away from login page into the dashboard
+  await page.waitForURL((url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"), {
+    timeout: 30_000,
+  });
+  // Wait for sidebar to render as a signal that the dashboard is ready
+  await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible({ timeout: 30_000 });
+}

tests/ui_e2e_tests/helpers/navigation.ts

@@ -0,0 +1,6 @@
+import { Page as PlaywrightPage } from "@playwright/test";
+import { Page } from "../constants";
+
+export async function navigateToPage(page: PlaywrightPage, targetPage: Page) {
+  await page.goto(`/ui?page=${targetPage}`);
+}