@@ -22,7 +22,7 @@ class UIUXViewportTool(BaseTool):
2222 "Performs two UX checks in the current viewport: (1) Typo/grammar/text accuracy using page text; "
2323 "(2) Layout/visual rendering using screenshot + viewport structure. Returns both analyses."
2424 )
25- page : Any = Field (..., description = "Playwright Page instance" )
25+ ui_tester_instance : Any = Field (..., description = "UITester instance to access driver and page " )
2626 llm_config : dict | None = Field (default = None , description = "LLM configuration for independent client" )
2727 case_recorder : Any | None = Field (default = None , description = "Optional CentralCaseRecorder to record ux_verify step" )
2828
@@ -52,21 +52,64 @@ def _annotate_b64_image(self, image_b64: str, rect: List[int]) -> str:
5252 return image_b64
5353
5454 async def _arun (self , assertion : str ) -> str :
55- if not self .page :
56- return "[FAILURE] Error: Page instance not provided for UX collection."
55+ if not self .ui_tester_instance :
56+ return "[FAILURE] Error: UITester instance not provided for UX collection."
5757
5858 try :
5959 logging .debug (f"Executing UX verification: { assertion } )
6060
61- dp = DeepCrawler (self .page )
62- # Viewport-only crawl; do NOT scroll; collect text-rich structure
63- crawl_result = await dp .crawl (highlight = False , filter_text = True , viewport_only = True , include_styles = True )
61+ # Dynamically get current page from driver (handles get_new_page updates)
62+ page = self .ui_tester_instance .driver .get_page ()
63+
64+ dp = DeepCrawler (page )
65+ # Crawl for interactive elements with layout info (for layout check)
66+ crawl_result = await dp .crawl (highlight = False , filter_text = False , viewport_only = False , include_styles = True )
6467 id_map = crawl_result .raw_dict ()
65- viewport_structure = dp .get_text ()
68+
69+ # Get full page text directly from page for text/typo check (more comprehensive)
70+ viewport_structure = await page .evaluate ("""
71+ () => {
72+ // Extract all visible text from the page
73+ const textElements = [];
74+ const walker = document.createTreeWalker(
75+ document.body,
76+ NodeFilter.SHOW_TEXT,
77+ {
78+ acceptNode: function(node) {
79+ const parent = node.parentElement;
80+ if (!parent) return NodeFilter.FILTER_REJECT;
81+
82+ // Skip script, style, and hidden elements
83+ const style = window.getComputedStyle(parent);
84+ if (style.display === 'none' ||
85+ style.visibility === 'hidden' ||
86+ parent.tagName === 'SCRIPT' ||
87+ parent.tagName === 'STYLE') {
88+ return NodeFilter.FILTER_REJECT;
89+ }
90+
91+ const text = node.textContent.trim();
92+ return text.length > 0 ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_REJECT;
93+ }
94+ }
95+ );
96+
97+ let node;
98+ while (node = walker.nextNode()) {
99+ const text = node.textContent.trim();
100+ if (text && text.length > 0) {
101+ textElements.push(text);
102+ }
103+ }
104+
105+ // Deduplicate and return as JSON
106+ return JSON.stringify([...new Set(textElements)]);
107+ }
108+ """ )
66109 logging .debug (f"Viewport Text Structure: { viewport_structure } )
67110
68111 screenshot = None
69- img_bytes = await self . page .screenshot (full_page = False )
112+ img_bytes = await page .screenshot (full_page = True )
70113 screenshot = f"data:image/png;base64,{ base64 .b64encode (img_bytes ).decode ('utf-8' )}
71114
72115 try :
@@ -127,7 +170,7 @@ async def _arun(self, assertion: str) -> str:
127170 "- Conciseness: Keep each error description concise and direct, avoid explanations.\n "
128171 )
129172
130- logging .debug (f"UX text typo analysis prompt: { text_prompt } )
173+ # logging.debug(f"UX text typo analysis prompt: {text_prompt}")
131174
132175 typo_response = await llm_client .get_llm_response (
133176 LLMPrompt .page_default_prompt ,
@@ -146,7 +189,7 @@ async def _arun(self, assertion: str) -> str:
146189 # 2) Layout/visual analysis (screenshot + structure)
147190 layout_prompt = self ._build_layout_prompt (layout_user_case , id_map , len (screenshot ))
148191
149- logging .debug (f"UX layout analysis prompt: { layout_prompt } )
192+ # logging.debug(f"UX layout analysis prompt: {layout_prompt}")
150193
151194 images = [screenshot ] if isinstance (screenshot , str ) else None
152195 layout_response = await llm_client .get_llm_response (
0 commit comments