feat(dashboards): Track dashboard generation validation success and fail metrics

static/app/actionCreators/dashboards.tsx

@@ -10,7 +10,7 @@ import type {Organization} from 'sentry/types/organization';
 import {defined} from 'sentry/utils';
 import getApiUrl from 'sentry/utils/api/getApiUrl';
 import {TOP_N} from 'sentry/utils/discover/types';
-import type {QueryClient} from 'sentry/utils/queryClient';
+import {fetchMutation, type QueryClient} from 'sentry/utils/queryClient';
 import {getQueryKey} from 'sentry/views/dashboards/hooks/useGetStarredDashboards';
 import {
   DisplayType,
@@ -88,6 +88,41 @@ export function createDashboard(
   return promise;
 }
 
+export function validateDashboard(
+  orgSlug: string,
+  dashboard: DashboardDetails
+): Promise<void> {
+  const {title, widgets, projects, environment, period, start, end, filters, utc} =
+    dashboard;
+
+  const url = getApiUrl('/organizations/$organizationIdOrSlug/dashboards/', {
+    path: {organizationIdOrSlug: orgSlug},
+  });
+
+  return fetchMutation({
+    url,
+    method: 'POST',
+    data: {
+      title,
+      widgets: widgets.map(widget => omit(widget, ['tempId'])),
+      projects,
+      environment,
+      period,
+      start,
+      end,
+      filters,
+      utc,
+    },
+    options: {
+      query: {
+        validateOnly: '1',
+        project: projects,
+        environment,
+      },
+    },
+  });
+}
+
 export function updateDashboardVisit(
   api: Client,
   orgId: string,

static/app/views/dashboards/createFromSeer.spec.tsx

@@ -93,6 +93,11 @@ describe('CreateFromSeer', () => {
       url: '/organizations/org-slug/trace-items/attributes/',
       body: [],
     });
+    MockApiClient.addMockResponse({
+      url: '/organizations/org-slug/dashboards/',
+      method: 'POST',
+      body: {},
+    });
   });
 
   it('shows loading state while session is processing', () => {

static/app/views/dashboards/createFromSeer.tsx

@@ -5,11 +5,13 @@ import * as Sentry from '@sentry/react';
 import {Alert} from '@sentry/scraps/alert';
 import {Flex} from '@sentry/scraps/layout';
 
+import {validateDashboard} from 'sentry/actionCreators/dashboards';
 import {addErrorMessage} from 'sentry/actionCreators/indicator';
 import ErrorBoundary from 'sentry/components/errorBoundary';
 import * as Layout from 'sentry/components/layouts/thirds';
 import {LoadingIndicator} from 'sentry/components/loadingIndicator';
 import {t} from 'sentry/locale';
+import type {Organization} from 'sentry/types/organization';
 import {parseQueryKey} from 'sentry/utils/api/apiQueryKey';
 import {MarkedText} from 'sentry/utils/marked/markedText';
 import {useApiQuery, useQueryClient} from 'sentry/utils/queryClient';
@@ -101,6 +103,27 @@ function extractMessages(
   return messages;
 }
 
+async function validateDashboardAndRecordMetrics(
+  organization: Organization,
+  newDashboard: DashboardDetails,
+  seerRunId: number | null
+) {
+  try {
+    await validateDashboard(organization.slug, newDashboard);
+    Sentry.metrics.count('dashboards.seer.validation', 1, {
+      attributes: {status: 'success', ...(seerRunId ? {seer_run_id: seerRunId} : {})},
+    });
+  } catch (error) {
+    Sentry.metrics.count('dashboards.seer.validation', 1, {
+      attributes: {status: 'failure', ...(seerRunId ? {seer_run_id: seerRunId} : {})},
+    });
+  }
+}
+
+function statusIsTerminal(status?: string | null) {
+  return status === 'completed' || status === 'error' || status === 'awaiting_user_input';
+}
+
 export default function CreateFromSeer() {
   const organization = useOrganization();
   const location = useLocation();
@@ -127,7 +150,7 @@ export default function CreateFromSeer() {
           return POLL_INTERVAL_MS;
         }
         const status = query.state.data?.[0]?.session?.status;
-        if (status === 'completed' || status === 'error') {
+        if (statusIsTerminal(status)) {
           return false;
         }
         return POLL_INTERVAL_MS;
@@ -143,7 +166,7 @@ export default function CreateFromSeer() {
     const prevUpdatedAt = prevUpdatedAtRef.current;
     prevUpdatedAtRef.current = sessionUpdatedAt;
 
-    const isTerminal = sessionStatus === 'completed' || sessionStatus === 'error';
+    const isTerminal = statusIsTerminal(sessionStatus);
 
     // Only trigger Dashboard rerender when transition to a new completed state
     if (prevUpdatedAt !== sessionUpdatedAt && isTerminal && session) {
@@ -152,22 +175,24 @@ export default function CreateFromSeer() {
       }
       const dashboardData = extractDashboardFromSession(session);
       if (dashboardData) {
-        setDashboard({
+        const newDashboard = {
           ...EMPTY_DASHBOARD,
           title: dashboardData.title,
           widgets: dashboardData.widgets,
-        });
+        };
+        setDashboard(newDashboard);
+
+        validateDashboardAndRecordMetrics(organization, newDashboard, seerRunId);
       }
     }
-  }, [isUpdating, sessionStatus, session, sessionUpdatedAt]);
+  }, [organization, seerRunId, isUpdating, sessionStatus, session, sessionUpdatedAt]);
 
   const blockMessages = useMemo(
     () => (session ? extractMessages(session) : []),
     [session]
   );
 
-  const isLoading =
-    !!seerRunId && sessionStatus !== 'completed' && sessionStatus !== 'error' && !isError;
+  const isLoading = !!seerRunId && !statusIsTerminal(sessionStatus) && !isError;
 
   // Prevent repeat errors on the same widget
   const reportedWidgetErrors = useRef(new Set<string>());