feat: Added smartscapeNode command for finding entities (#204)

* feat: Added smartscapeNode command for finding entities

* Update CHANGELOG.md

Co-authored-by: Manuel Warum <[email protected]>

* chore: Refactor the slice filter for entity search

---------

Co-authored-by: Manuel Warum <[email protected]>

Author: christian-kreuzberger-dtx

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Unreleased Changes
 
+### Tools
+
+- `find_entities_by_name` now uses `smartscapeNode` DQL command under the hood, and will fall back to `fetch dt.entity.${entityType}`.
+
+### Scopes
+
+- Added OAuth scope `storage:smartscape:read`
+
 ## 0.9.2
 
 - Improved error handling when initializing the connection for the first time

README.md

@@ -321,6 +321,7 @@ Depending on the features you are using, the following scopes are needed:
 - `storage:system:read` - needed for `execute_dql` tool to read System Data from Grail
 - `storage:user.events:read` - needed for `execute_dql` tool to read User events from Grail
 - `storage:user.sessions:read` - needed for `execute_dql` tool to read User sessions from Grail
+- `storage:smartscape:read` - needed for `execute_dql` tool to read Smartscape Data
 - `davis-copilot:conversations:execute` - execute conversational skill (chat with Copilot)
 - `davis-copilot:nl2dql:execute` - execute Davis Copilot Natural Language (NL) to DQL skill
 - `davis-copilot:dql2nl:execute` - execute DQL to Natural Language (NL) skill

src/capabilities/find-monitored-entity-by-name.ts

@@ -28,10 +28,37 @@ export const generateDqlSearchEntityCommand = (entityNames: string[], extendedSe
 };
 
 /**
- * Find a monitored entity by name via DQL
+ * Find a monitored entity via "smartscapeNodes" by name via DQL
  * @param dtClient
- * @param entityName
- * @returns A string with the entity details like id, name and type, or an error message if no entity was found
+ * @param entityNames Array of entitiy names to search for
+ * @returns An array with the entity details like id, name and type
+ */
+export const findMonitoredEntityViaSmartscapeByName = async (dtClient: HttpClient, entityNames: string[]) => {
+  const dql = `smartscapeNodes "*" | search "*${entityNames.join('*" OR "*')}*" | fields id, name, type`;
+  console.error(`Executing DQL: ${dql}`);
+
+  try {
+    const smartscapeResult = await executeDql(dtClient, { query: dql });
+
+    if (smartscapeResult && smartscapeResult.records && smartscapeResult.records.length > 0) {
+      // return smartscape results if we found something
+      return smartscapeResult;
+    }
+  } catch (error) {
+    // ignore errors here, as smartscapeNodes may not be ready for all environments/users
+    console.error('Error while querying smartscapeNodes:', error);
+  }
+
+  console.error('No results from smartscapeNodes');
+  return null;
+};
+
+/**
+ * Find a monitored entity via "dt.entity.${entityType}" by name via DQL
+ * @param dtClient
+ * @param entityNames Array of entitiy names to search for
+ * @param extendedSearch If true, search over all entity types, otherwise only basic ones
+ * @returns An array with the entity details like id, name and type
  */
 export const findMonitoredEntitiesByName = async (
   dtClient: HttpClient,

src/index.ts

@@ -31,7 +31,10 @@ import { updateWorkflow } from './capabilities/update-workflow';
 import { executeDql, verifyDqlStatement } from './capabilities/execute-dql';
 import { sendSlackMessage } from './capabilities/send-slack-message';
 import { sendEmail } from './capabilities/send-email';
-import { findMonitoredEntitiesByName } from './capabilities/find-monitored-entity-by-name';
+import {
+  findMonitoredEntitiesByName,
+  findMonitoredEntityViaSmartscapeByName,
+} from './capabilities/find-monitored-entity-by-name';
 import {
   chatWithDavisCopilot,
   explainDqlInNaturalLanguage,
@@ -464,26 +467,61 @@ const main = async () => {
       readOnlyHint: true,
     },
     async ({ entityNames, maxEntitiesToDisplay, extendedSearch }) => {
-      const dtClient = await createAuthenticatedHttpClient(scopesBase.concat('storage:entities:read'));
+      const dtClient = await createAuthenticatedHttpClient(
+        scopesBase.concat('storage:entities:read', 'storage:smartscape:read'),
+      );
+
+      const smartscapeResult = await findMonitoredEntityViaSmartscapeByName(dtClient, entityNames);
+
+      if (smartscapeResult && smartscapeResult.records && smartscapeResult.records.length > 0) {
+        // Filter valid entities first, to ensure we display up to maxEntitiesToDisplay entities
+        const validSmartscapeEntities = smartscapeResult.records.filter(
+          (entity): entity is { id: string; type: string; name: string; [key: string]: any } =>
+            !!(entity && entity.id && entity.type && entity.name),
+        );
+
+        let resp = `Found ${validSmartscapeEntities.length} monitored entities via Smartscape! Displaying the first ${Math.min(maxEntitiesToDisplay, validSmartscapeEntities.length)} valid entities:\n`;
+
+        validSmartscapeEntities.slice(0, maxEntitiesToDisplay).forEach((entity) => {
+          resp += `- Entity '${entity.name}' of entity-type '${entity.type}' has entity id '${entity.id}' and tags ${entity['tags'] ? JSON.stringify(entity['tags']) : 'none'} - DQL Filter: '| filter dt.smartscape.${String(entity.type).toLowerCase()} == "${entity.id}"'\n`;
+        });
 
+        resp +=
+          '\n\n**Next Steps:**\n' +
+          '1. Fetch more details about the entity, using the `execute_dql` tool with the following DQL Statement: "smartscapeNodes \"<entity-type>\" | filter id == <entity-id>"\n' +
+          '2. Perform a sanity check that found entities are actually the ones you are looking for, by comparing name and by type (hosts vs. containers vs. apps vs. functions) and technology (Java, TypeScript, .NET) with what is available in the local source code repo.\n' +
+          '3. Find and investigate available metrics for relevant entities, by using the `execute_dql` tool with the following DQL statement: "fetch metric.series | filter dt.smartscape.<entity-type> == <entity-id> | limit 20"\n' +
+          '4. Find out whether any problems exist for this entity using the `list_problems` or `list_vulnerabilities` tool, and the provided DQL-Filter\n' +
+          '5. Explore dependency & relationships with: "smartscapeEdges \"*\" | filter source_id == <entity-id> or target_id == <entity-id>" to list inbound/outbound edges (depends_on, dependency_of, owned_by, part_of) for graph context\n';
+
+        return resp;
+      }
+
+      // If no result from Smartscape, try the classic entities API
       const result = await findMonitoredEntitiesByName(dtClient, entityNames, extendedSearch);
 
       if (result && result.records && result.records.length > 0) {
-        let resp = `Found ${result.records.length} monitored entities! Displaying the first ${maxEntitiesToDisplay} entities:\n`;
+        // Filter valid entities first, to ensure we display up to maxEntitiesToDisplay entities
+        const validClassicEntities = result.records.filter(
+          (entity): entity is { id: string; type: string; name: string; [key: string]: any } =>
+            !!(entity && entity.id && entity.type && entity.name),
+        );
+
+        let resp = `Found ${validClassicEntities.length} monitored entities! Displaying the first ${Math.min(maxEntitiesToDisplay, validClassicEntities.length)} entities:\n`;
 
         // iterate over dqlResponse and create a string with the problem details, but only show the top maxEntitiesToDisplay problems
-        result.records.slice(0, maxEntitiesToDisplay).forEach((entity) => {
+        validClassicEntities.slice(0, maxEntitiesToDisplay).forEach((entity) => {
           if (entity && entity.id) {
             const entityType = getEntityTypeFromId(String(entity.id));
-            resp += `- Entity '${entity['entity.name']}' of type '${entity['entity.type']}' has entity id '${entity.id}' and tags ${entity['tags'] ? entity['tags'] : 'none'} - Use the DQL Filter: '| filter ${entityType} == "${entity.id}"'\n`;
+            resp += `- Entity '${entity['entity.name']}' of entity-type '${entity['entity.type']}' has entity id '${entity.id}' and tags ${entity['tags'] ? entity['tags'] : 'none'} - DQL Filter: '| filter ${entityType} == "${entity.id}"'\n`;
           }
         });
 
         resp +=
           '\n\n**Next Steps:**\n' +
-          '1. Try to fetch more details about the entity, using the `execute_dql` tool with "describe(dt.entity.<entity-type>)", and "fetch dt.entity.<entity-type> | filter id == <entity-id> | fieldsAdd <field-1>, <field2>, ..."\n' +
+          '1. Fetch more details about the entity, using the `execute_dql` tool with the following DQL Statements: "describe(dt.entity.<entity-type>)", and "fetch dt.entity.<entity-type> | filter id == <entity-id> | fieldsAdd <field-1>, <field-2>, ..."\n' +
           '2. Perform a sanity check that found entities are actually the ones you are looking for, by comparing name and by type (hosts vs. containers vs. apps vs. functions) and technology (Java, TypeScript, .NET) with what is available in the local source code repo.\n' +
-          '3. Find and investigate available metrics for relevant entities, by using the `execute_dql` tool with the following DQL statement: "fetch metric.series | filter dt.entity.<entity-type> == <entity-id>"\n' +
+          '3. Find and investigate available metrics for relevant entities, by using the `execute_dql` tool with the following DQL statement: "fetch metric.series | filter dt.entity.<entity-type> == <entity-id> | limit 20"\n' +
           '4. Find out whether any problems exist for this entity using the `list_problems` or `list_vulnerabilities` tool, and the provided DQL-Filter\n';
 
         return resp;
@@ -558,14 +596,14 @@ const main = async () => {
 
   tool(
     'execute_dql',
-    'Get Logs, Metrics, Spans or Events from Dynatrace GRAIL by executing a Dynatrace Query Language (DQL) statement. ' +
+    'Get data like Logs, Metrics, Spans, Events, or Entity Data from Dynatrace GRAIL by executing a Dynatrace Query Language (DQL) statement. ' +
       'Use the "generate_dql_from_natural_language" tool upfront to generate or refine a DQL statement based on your request. ' +
       'To learn about possible fields available for filtering, use the query "fetch dt.semantic_dictionary.models | filter data_object == \"logs\""',
     {
       dqlStatement: z
         .string()
         .describe(
-          'DQL Statement (Ex: "fetch [logs, spans, events], from: now()-4h, to: now() | filter <some-filter> | summarize count(), by:{some-fields}.", or for metrics: "timeseries { avg(<metric-name>), value.A = avg(<metric-name>, scalar: true) }"). ' +
+          'DQL Statement (Ex: "fetch [logs, spans, events, metric.series, ...], from: now()-4h, to: now() [| filter <some-filter>] [| summarize count(), by:{some-fields}]", or for metrics: "timeseries { avg(<metric-name>), value.A = avg(<metric-name>, scalar: true) }", or for entities via smartscape: "smartscapeNodes \"[*, HOST, PROCESS, ...]\" [| filter id == "<ENTITY-ID>"]"). ' +
             'When querying data for a specific entity, call the `find_entity_by_name` tool first to get an appropriate filter like `dt.entity.service == "SERVICE-1234"` or `dt.entity.host == "HOST-1234"` to be used in the DQL statement. ',
         ),
     },
@@ -591,6 +629,7 @@ const main = async () => {
           'storage:user.events:read', // Read User events from Grail
           'storage:user.sessions:read', // Read User sessions from Grail
           'storage:security.events:read', // Read Security events from Grail
+          'storage:smartscape:read', // Read Smartscape Entities from Grail
         ),
       );
       const response = await executeDql(dtClient, { query: dqlStatement }, grailBudgetGB);