
    epj                       U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlmZmZmZmZmZ ddlmZmZmZmZ  ej        e          ZdadqdZ eh d	          Z eh d
          Z eh d           eh d           eh d           eh d           eh d           eddh           eh d           eh d           eddh          h	ZdddddddddZ drdZ! ej"        dej#                   ej"        dej#                   ej"        dej#                   ej"        d ej#                   ej"        d!ej#                   ej"        d"ej#                  gZ$dsd%Z% ej&                    Z'i Z(d&e)d'<   da*d(e)d)<   i Z+d*e)d+<   i Z,d,e)d-<   i Z-d.e)d/<    ej&                    Z.i Z/d0e)d1<   i Z0d2e)d3<   dtd5Z1dudvd8Z2dwd:Z3dqd;Z4 ej5        e4           dqd<Z6 G d= d>e          Z7dxdBZ8	 dudydDZ9dzdEZ:d{dGZ;d|dJZ<d|dKZ=d}dLZ>dMZ?dNZ@dOZAd~dRZBddUZCe?fddYZDdZZEeEfdd_ZFdd`ZGddaZHddcZIddeZJdddkZKddnZLddoZMddpZNdS )uB  Entry point for the `computer_use` tool.

Universal (any-model) desktop control across macOS, Windows, and Linux via
cua-driver's background computer-use primitive. Replaces #4562's
Anthropic-native `computer_20251124` approach — the schema here is standard
OpenAI function-calling so every tool-capable model can drive it.

Linux is the most recent runtime (X11 + Wayland, via cua-driver-rs's
AT-SPI tree path); it is enabled here alongside macOS and Windows. When a
host's display server or accessibility stack isn't reachable, cua-driver's
`health_report` (surfaced by `hermes computer-use doctor`) reports the
exact blocked check rather than the toolset silently failing.

Return contract
---------------
For text-only results (wait, key, list_apps, focus_app, failures, etc.):
  JSON string.

For captures / actions with `capture_after=True`:
  A dict wrapped as the OpenAI-style multi-part tool-message content:

      {
        "_multimodal": True,
        "content": [
            {"type": "text", "text": "<human-readable summary + SOM index>"},
            {"type": "image_url",
             "image_url": {"url": "data:image/png;base64,<b64>"}},
        ],
        "text_summary": "<text used for fallback string content>",
      }

  run_agent.py's tool-message builder inspects `_multimodal` and emits a
  list-shaped `content` for OpenAI-compatible providers. The Anthropic
  adapter splices the base64 image into a `tool_result` block (see
  `agent/anthropic_adapter.py`). Every provider that supports multi-part
  tool content gets the image; text-only providers see the summary only.
    )annotationsN)AnyDictListOptionalTuple)ActionResultCaptureResultComputerUseBackend	UIElementreturnNonec                
    | a dS )a  Register a callback for computer_use approval prompts (used by CLI).

    Matches the terminal_tool._approval_callback pattern. The callback
    receives (action, args, summary) and returns one of:
      "approve_once" | "approve_session" | "always_approve" | "deny".
    N)_approval_callback)cbs    =/home/thesage/.hermes/hermes-agent/tools/computer_use/tool.pyset_approval_callbackr   E   s         >   waitcapture	list_appslist_windowscua_browser_state>   keydragtypeclickscroll	focus_app	set_valueright_clickdouble_clickmiddle_clickcua_browser_typecua_browser_clickcua_browser_dialogcua_browser_pointercua_browser_preparecua_browser_downloadcua_browser_navigatecua_browser_set_input_files>   cmdshift	backspace>   r,   optionr.   >   qr,   ctrl>   r0   r,   r-   >   r0   r,   r-   r/   winl>   r1   deleter/   >   delr1   r/   r/   f4r,   r1   )commandcontrolaltu   ⌘u   ⌥windowssupermetakeysstr	frozensetc                v    d t          j        d|           D             }d |D             }t          |          S )Nc                    g | ]<}|                                 |                                                                 =S  )striplower.0ps     r   
<listcomp>z$_canon_key_combo.<locals>.<listcomp>z   s7    TTT1!''))TQWWYY__TTTr   z\s*[+\-]\s*c                D    g | ]}t                               ||          S rB   )_KEY_ALIASESgetrE   s     r   rH   z$_canon_key_combo.<locals>.<listcomp>{   s(    333\a##333r   )resplitr?   )r=   partss     r   _canon_key_comborO   u   sE    
 UT(F(FTTTE33U333EUr   zcurl\s+[^|]*\|\s*bashzcurl\s+[^|]*\|\s*shzwget\s+[^|]*\|\s*bashz\bsudo\s+rm\s+-[rf]z\brm\s+-rf\s+/\s*$z:\s*\(\)\s*\{\s*:\|:\s*&\s*\}textOptional[str]c                V    t           D ] }|                    |           r	|j        c S !d S N)_BLOCKED_TYPE_PATTERNSsearchpattern)rP   pats     r   _is_blocked_typerX      s=    %  ::d 	;	4r   zDict[Tuple[str, str], bool]_AUX_VISION_ROUTE_CACHEzOptional[ComputerUseBackend]_backendzDict[str, ComputerUseBackend]	_backendszDict[str, threading.RLock]_backend_call_lockszDict[str, str]_backend_permission_modeszDict[str, bool]_session_auto_approvezDict[str, set]_always_allow
session_idc                    	 ddl m}m}  ||           rdS  |d          }|r ||          rdS n# t          $ r Y nw xY wdS )u  Map Hermes's explicit approval bypass onto Cua's immutable mode.

    Hermes has TWO session-identity namespaces: the tool-dispatch path passes
    the DB ``session_id`` (``agent.session_id``), while gateway ``/yolo``
    keys approval state off the gateway ``session_key`` (set per turn via the
    ``set_current_session_key`` contextvar in tools/approval.py). CLI and TUI
    use the DB id for both. Checking ONLY ``session_id`` here would make a
    gateway ``/yolo`` toggle silently invisible to computer_use (works in
    CLI, dead on messaging platforms), so we consult both namespaces —
    bypass in either means the user explicitly opted out of approvals for
    this run. Fails closed on any resolution error.
    r   )get_current_session_key%is_approval_bypass_active_for_sessionunrestricted )defaultstandard)tools.approvalrb   rc   	Exception)r`   rb   rc   current_keys       r   _cua_permission_moderk      s    	
 	
 	
 	
 	
 	
 	
 	

 10<< 	"!>--b999 	"@@MM 	"!>    :s   3 3 
A A re   r   c                   t          | pd          }	 d }d }t          5  t          |          }|dk    rDt          =|t          vr4t          t          |<   t          j                    t          |<   |t          |<   t          	                    |          }|t          	                    |d          |k    r|cd d d            S t          
                    |          }t          
                    |d           }t          
                    |d            |dk    rd ant          j        	                    dd                                          }|dv rddlm}  ||	          }n'|d
k    rt!                      }nt#          d|          |                                 |t          |<   t          j                    t          |<   |t          |<   |dk    r|a|cd d d            S d d d            n# 1 swxY w Y   	 |/|5  |                                 d d d            n# 1 swxY w Y   n||                                 n# t(          $ r Y nw xY wB)Nre   Trg   HERMES_COMPUTER_USE_BACKENDcua>   
cua-driverre   rn   r   )CuaDriverBackend)permission_modenoopz$Unknown HERMES_COMPUTER_USE_BACKEND=)r>   _backend_lockrk   rZ   r[   	threadingRLockr\   r]   rK   poposenvironrD   tools.computer_use.cua_backendrp   _NoopBackendRuntimeErrorstartstopri   )	r`   sidstale_backend
stale_lockrq   cachedbackend_namerp   backends	            r   _get_backendr      s/   
jB

C=6:04
 .	 .	 3377ObyyX1c6J6J "*	#+4?+<+<#C(1@)#.]]3''F!,00jAA_TT!.	 .	 .	 .	 .	 .	 .	 .	$ !*c 2 2044S$??
)--c4888"99#H!z~~15   %''   #<<<OOOOOO..OOOGG!V++*nnGG&O|OO   !(	#+4?+<+<#C(1@)#."99&H].	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	 .	f	% ) )!&&((() ) ) ) ) ) ) ) ) ) ) ) ) ) )*""$$$ 	 	 	D	y=sO   BG/DG//G36G3;I ?H I  H$$I 'H$(I 
IIboolc                   t          | pd          }t          5  t                              |d          }t                              |d          }t
                              |d           |dk    r	|t          }|dk    rt          |u rdaddd           n# 1 swxY w Y   t          5  t                              |d           t                              |d           ddd           n# 1 swxY w Y   |dS 	 |/|5  |
                                 ddd           n# 1 swxY w Y   n|
                                 n-# t          $ r  t                              d|d           Y nw xY wdS )uZ  Release one session-owned computer-use backend.

    This is the production lifecycle seam for hosts and policy plugins. It
    removes the exact session backend, its call lock, and its recorded
    permission mode before stopping the backend, so new lookups cannot retain
    the stale target/ref namespace — and stops a private embedded daemon when
    Hermes YOLO selected unrestricted mode. Approval state is cleared even
    when no backend was started.

    Returns ``True`` when a backend was found and released, ``False`` when the
    session was already absent. Safe to call repeatedly.
    re   NFz2computer_use backend release failed for session %sT)exc_info)r>   rs   r[   rv   r\   r]   rZ   _approval_lockr^   r_   r}   ri   loggerdebug)r`   r~   r   	call_locks       r   release_computer_use_sessionr     sF    jB

C	 	 	--T**'++C66	!%%c4000 "99G"99W,,H	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 
 % %!!#t,,,#t$$$% % % % % % % % % % % % % % % u
                   LLNNN 
 
 
@ 	 	
 	
 	
 	
 	

 4sZ   A2BBB)7C,,C03C0<E  D!E !D%%E (D%)E 'E,+E,c                    t           5  d t                                          D             } t          G|                     t          t                    t          t                              d          f           dat                                           t                                           t                                           ddd           n# 1 swxY w Y   t          5  t                                           t                                           ddd           n# 1 swxY w Y   |                                 D ]}\  }}	 |/|5  |                                 ddd           n# 1 swxY w Y   n|                                 L# t          $ r%}t                               d|           Y d}~vd}~ww xY wdS )u  Stop all cached backends so cua-driver children don't outlive us.

    Each session backend holds a long-lived ``cua-driver`` subprocess, so
    without this a driver can survive the Hermes process that spawned it
    (#28152 item 3). #69903 kept the orphan from burning a core by disabling
    the cursor overlay; the process itself still lingered.

    Mirrors ``browser_tool``'s ``atexit.register(_emergency_cleanup_all_sessions)``
    — same spawn-and-drive-a-subprocess shape. atexit only, no signal handlers:
    a ``SystemExit`` raised from a prompt_toolkit key binding corrupts its
    coroutine state and makes the process unkillable. Never raises, since an
    exception escaping atexit prints a traceback on every exit.
    c                h    i | ]/\  }}t          |          |t                              |          f0S rB   )idr\   rK   )rF   r~   r   s      r   
<dictcomp>z,_shutdown_backend_atexit.<locals>.<dictcomp>P  sH     
 
 
W wKK'#6#:#:3#?#?@
 
 
r   Nre   z%cua-driver atexit teardown failed: %s)rs   r[   itemsrZ   
setdefaultr   r\   rK   clearr]   r   r^   r_   valuesr}   ri   r   r   )uniquer   r   es       r   _shutdown_backend_atexitr   >  s{   " 
 * *
 
 ) 1 1
 
 
 8.222667   !!###!'')))* * * * * * * * * * * * * * * 
  ##%%%               %mmoo E E	E$ # #LLNNN# # # # # # # # # # # # # # #  	E 	E 	ELL@!DDDDDDDD	EE Es`   B?CCC%3D$$D(+D(	FE."F.E2	2F5E2	6F
F>F99F>c                 T    t                       t                                           dS )uC   Test helper — tear down the cached backend and per-session state.N)r   rY   r   rB   r   r   reset_backend_for_testsr   p  s&    !!#####r   c                      e Zd ZdZd%dZd%dZd%dZd&dZ	 	 	 	 d'd(dZd)dZ	d)dZ
d)dZd*dZd+dZd,dZd,dZd-d.d!Zd/d0d$Zd
S )1rz   z5Test/CI stub. Records calls; returns trivial results.r   r   c                "    g | _         d| _        d S NF)calls_startedselfs    r   __init__z_NoopBackend.__init__y  s    79
r   c                    d| _         d S NTr   r   s    r   r|   z_NoopBackend.start}  s    TT]]]r   c                    d| _         d S r   r   r   s    r   r}   z_NoopBackend.stop~  s    EDMMMr   r   c                    dS r   rB   r   s    r   is_availablez_NoopBackend.is_available  s    44r   somNmoder>   apprQ   pidOptional[int]	window_idr
   c           	     t    | j                             d||||df           t          |ddd g |pdd          S )Nr   )r   r   r   r   i   i   re   )r   widthheightpng_b64elementsr   window_title)r   appendr
   )r   r   r   r   r   s        r   r   z_NoopBackend.capture  se     	
#c	JJ
 	 	 	 $d3&(ciRbJ J J 	Jr   r	   c                \    | j                             d|f           t          dd          S )Nr   Tokactionr   r   r	   r   kws     r   r   z_NoopBackend.click  s/    
7B-(((tG4444r   c                \    | j                             d|f           t          dd          S )Nr   Tr   r   r   s     r   r   z_NoopBackend.drag  s/    
62,'''tF3333r   c                \    | j                             d|f           t          dd          S )Nr   Tr   r   r   s     r   r   z_NoopBackend.scroll  s/    
8R.)))tH5555r   rP   c                d    | j                             dd|i|f           t          dd          S )Nr   rP   Tr   r   )r   rP   r   s      r   	type_textz_NoopBackend.type_text  s:    
6FD#7B#78999tF3333r   r=   c                d    | j                             dd|i|f           t          dd          S )Nr   r=   Tr   r   )r   r=   r   s      r   r   z_NoopBackend.key  s:    
564"62"67888tE2222r   List[Dict[str, Any]]c                >    | j                             di f           g S )Nr   r   r   r   s    r   r   z_NoopBackend.list_apps  s"    
;+,,,	r   c                >    | j                             di f           g S )Nr   r   r   s    r   r   z_NoopBackend.list_windows  s"    
>2.///	r   Fraise_windowc                b    | j                             d||df           t          dd          S )Nr   )r   raiseTr   r   )r   r   r   s      r   r   z_NoopBackend.focus_app  s8    
;l(K(KLMMMtK8888r   valueelementc                b    | j                             d||df           t          dd          S )Nr    r   r   Tr   r   )r   r   r   s      r   r    z_NoopBackend.set_value  s8    
;%G(L(LMNNNtK8888r   r   r   r   r   )r   NNN)
r   r>   r   rQ   r   r   r   r   r   r
   )r   r	   )rP   r>   r   r	   )r=   r>   r   r	   )r   r   )F)r   r>   r   r   r   r	   rS   )r   r>   r   r   r   r	   )__name__
__module____qualname____doc__r   r|   r}   r   r   r   r   r   r   r   r   r   r   r    rB   r   r   rz   rz   v  s>       ??    21111111//// !!#'J J J J J5 5 5 54 4 4 46 6 6 64 4 4 43 3 3 3      9 9 9 9 99 9 9 9 9 9 9r   rz   argsDict[str, Any]r   c                   |                      d          pd                                                                }|st          j        ddi          S t          |                     d          pd          }|dv rA|                      dd          }t          |          }|rt          j        d|d	d
          S |dk    r|                      dd          }t          |          }t          D ]`}|	                    |          rIt          |          t          |          k    r)t          j        dt          |           dd
          c S a|                      d          r0|                      d          dk    rt          j        ddd          S |t          v rt          || |          }	|	|	S |                      d          s|dk    r*|                      d          rt          d| |          }	|	|	S 	 t          |          }
n1# t          $ r$}t          j        d| dd
          cY d}~S d}~ww xY w	 t           5  t"                              |t'          j                              }ddd           n# 1 swxY w Y   |5  t+          |
||           cddd           S # 1 swxY w Y   dS # t          $ r@}t,                              d|           t          j        d| d| i          cY d}~S d}~ww xY w)u   Main entry point — dispatched by tools.registry.

    Returns either a JSON string (text-only) or a dict marked `_multimodal`
    (image + summary) which run_agent.py wraps into the tool message.
    r   re   errorzmissing `action`r`   >   r   r$   rP   zblocked pattern in type text: z:Dangerous shell patterns cannot be typed via computer_use.)r   hintr   r=   zblocked key combo: z.Destructive system shortcuts are hard-blocked.bring_to_frontdelivery_mode
foregroundz2bring_to_front requires delivery_mode='foreground'"bring_to_front_requires_foreground)r   codeNr   r   )r`   z"computer_use backend unavailable: zIf the cua-driver binary is missing, run `hermes computer-use install`. If a Python dependency is missing, the error above shows the exact install command.zcomputer_use %s failedz	 failed: )rK   rC   rD   jsondumpsr>   rX   rO   _BLOCKED_KEY_COMBOSissubsetlensorted_DESTRUCTIVE_ACTIONS_request_approvalr   ri   rs   r\   r   rt   ru   	_dispatchr   	exception)r   kwargsr   r`   rP   rW   r=   comboblockederrr   r   r   s                r   handle_computer_user     s+    hhx  &B--//5577F 9z7$67888 VZZ--344J ---xx##t$$ 	:A#AAT    
 xx## &&* 	 	G&& 3w<<3u::+E+EzD6'??DDL# #     
 xx !! dhh&?&?<&O&OzI8
 
   	 %%%j99?J xx !! +$((>":": 0$
CC?J*555   z=!==j
 
   	 	 	 	 	 	> 	V 	V+66z9?CTCTUUI	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	4 	4Wfd33	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 > > >16:::z7v$;$;$;$;<========>s   9H
 

H8H3-H83H8<J4 -I<0J4 <J  J4 J J4 	J'J4 'J++J4 .J+/J4 4
K>>5K93K>9K>r   c                d   |                     d          dk    }| |rdndf}t          5  t                               |          r	 ddd           dS |t                               |t	                                v r	 ddd           dS 	 ddd           n# 1 swxY w Y   t
          }|dS t          | |          }	  || ||          }n4# t          $ r'}t          	                    d|           d}Y d}~nd}~ww xY w|dk    rdS |dk    s|d	k    rkt          5  t          
                    |t	                                                    |           |d	k    r
d
t          |<   ddd           n# 1 swxY w Y   dS |dk    rt          j        d| d          S t          j        d| d          S )uP  Return None if approved, or a JSON error string if denied.

    Approval is scoped by (action, delivery_mode) AND by session_id.
    Foreground delivery is a visible focus change, so a prior background
    approval — even ``approve_session`` on the same action — must NOT
    silently authorize it (NousResearch/hermes-agent#67052).
    ``always_approve`` (the blanket "auto-approve everything" unlock) still
    covers foreground, since the user explicitly opted into unattended
    operation. State is keyed on session_id so concurrent runs don't leak
    unlocks into one another.
    r   r   
backgroundNzapproval callback failed: %sdenyapprove_onceapprove_sessionalways_approveTtimeoutun   approval prompt timed out — the user did not respond. Silence is not consent; do not retry without the user.)r   r   zdenied by user)rK   r   r^   r_   setr   _summarize_actionri   r   warningr   addr   r   )	r   r   r`   is_foreground	scope_keyr   summaryverdictr   s	            r   r   r     s    HH_--=MHLII	   $$Z00 	        ))*cee<<<<	        =              
 
B	z t--G"VT7++   5q999 .  t###w2B'B'B 	9 	9$$Z77;;IFFF***48%j1	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 t)zI 
 
   	 : 0FCCDDDsB   B*BBB=C 
C<C77C<AE11E58E5c                n   |                     d          dk    rdnd}| dv rV|                     d          |  d|d          | S |                     d          }|r|  d	t          |           | S | |z   S | d
k    r^|                     d          p|                     d          }|                     d          p|                     d          }d| d| | S | dk    r2d|                     dd           d|                     dd           | S | dk    r>|                     dd          }d|d d         t          |          dk    rdndz   |z   S | dk    rd|                     dd          | S | d k    r3d!|                     d"d          |                     d#          rd$ndz   S | |z   S )%Nr   r   u;    [FOREGROUND — briefly raises the window / changes focus]re   >   r   r!   r"   r#   r   z
 element #
coordinatez at r   from_elementfrom_coordinate
to_elementto_coordinatezdrag u    → r   zscroll 	direction?z xamount   r   rP   ztype <   z...r   zkey r=   r   zfocus r   r   z (raise))rK   tupler   )r   r   fgcoordsrcdstrP   s          r   r   r   4  s   88O$$44 
G	F:< III88I*==Y====&& 	544%,,4444{hh~&&E$((3D*E*Ehh|$$A(A(A*s***b***R+s33RRtxx!7L7LRbRRRxx##$tCRCy$$TRRH2MM2dhhvr**2b222/++//.AYAY3a::_abbB;r   r   c                   t          |                    d                    }|dk    rt          |                    dd                    }|dvrt          j        dd|di          S ||                    d	          d
}|                    d          |                    d          >|                    |                    d          |                    d          d            | j        dfi |}t          |t          |                    d                              S |dk    rGt          |                    dd                    }| 
                    |          }t          |          S |dk    r8|                                 }	t          j        |	t          |	          d          S |dk    r8|                                 }
t          j        |
t          |
          d          S |dk    ru|                    d	          }|st          j        ddi          S |                     |t          |                    d                              }t!          | ||          S |dk    rIi }dD ]%\  }}|                    |          ||         ||<   &t          j         | j        dfi |          S |dk    rt          j        |                     |                    d          |                    d          |                    dd          |                    d           t          |                    d!                    "                    S d#d$d%d&d'd(d)d*}|                    |          }|Ti }d+d,d-d.d/d0d1d2}||         D ]"}|                    |          ||         ||<   #|d3v rW|                    d4          B|d4         }t'          |t(          t*          f          rt          |          d5k    r|\  |d6<   |d7<   |                    d8          }|                    d9          }|                    d:          }||vr|d&k    r||}|d'k    r||}|||d:<   |||d:<   |                    d;          |d;         |d<   t          j        |                     ||                    d<          |=                    S |                    d>          }t          |                    d?                    }|d@v r|                    dA          }dB}|dCk    rd5}n|dDk    rdE}n|dFk    rdG}n|pdH}|                    dI          }|                    d4          pdJ}|r|dK         |dK         |dB         fndJ\  }}|                     ||nd |||pdH||                    dL          ||M          }t!          | ||          S |dNk    rA|                    dO          d uo|                    dP          d u} |                    dQ          o|                    dR          }!| s|!st          j        ddSi          S |                     |                    dO          |                    dP          |                    dQ          rt+          |dQ                   nd |                    dR          rt+          |dR                   nd |                    dAdH          |                    dL          ||T          }t!          | ||          S |dUk    r|                    d4          pdJ}|                     |                    dVdW          t5          |                    dXdY                    |                    dI          |r|dK         |dK         nd |r|dB         |dB         nd |                    dL          ||Z          }t!          | ||          S |d[k    r=|                     |                    d\d]          ||^          }t!          | ||          S |d_k    r=|                     |                    d`d]          ||^          }t!          | ||          S |dak    ru|                    db          }"|"t          j        ddci          S |                     t          |"          |                    dI          d          }t!          | ||          S t          j        dde|i          S )gNcapture_afterr   r   r   >   axr   visionr   z	bad mode z; use som|vision|axr   r   r   r   r   )r   r   max_elements)r  r   secondsg      ?r   )appscountr   )r:   r  r   zfocus_app requires `app`r   )r   r   ))r   r   )r   r   )tab_idr  )snapshot_formatr  )queryr  )	scope_refr  )continuationr  r(   profile_modeisolated_newprofile_nameallow_launch)r   r   r  r  r  browser_navigatebrowser_clickbrowser_typebrowser_pointerbrowser_dialogbrowser_set_input_filesbrowser_download)r*   r%   r$   r'   r&   r+   r)   )url)refinput_routexy)r$  rP   )	r$  destination_refr%  r&  r'  to_xto_ydelta_xdelta_y)	dialog_idprompt_textr   )r$  files)r$  destination_root)r  r  r  r  r   r!  r"  >   r  r  r      r&  r'  browser_pointer_actionbrowser_dialog_actionr   browser_type_moder  )r  r   r   r   >   r   r!   r"   r#   button   r"   r!   rightr#   middleleftr   )NNr   	modifiers)r   r&  r'  r5  click_countr:  r   r   r   r   r   r   r   zFdrag requires from_coordinate/to_coordinate or from_element/to_element)r   r   from_xyto_xyr5  r:  r   r   r   r   downr  r  )r   r  r   r&  r'  r:  r   r   r   rP   re   )r   r   r   r=   r    r   zset_value requires `value`r   zunknown action rB   )r   rK   r>   r   r   updater   _capture_response_coerce_max_elementsfloatr   _text_responser   r   r   r   _maybe_follow_capturetyped_browser_statetyped_browser_prepare
isinstancelistr  typed_browser_actionr   r   r   intr   r   r    )#r   r   r   r  r   capture_kwargscapr  resr  r:   r   
state_argspublicinternalbrowser_toolsdriver_tool	call_argsallowed_fieldsfieldr   pointer_actiondialog_actionnested_actionr   r   r5  r;  r   r  r&  r'  has_elements
has_coordsr   s#                                      r   r   r   N  s	   /2233M488FE**++...:w(OD(O(O(OPQQQ26txx)O)O88E??&$((;*?*?*K!!xx!XXk22# #    go//// 3GQ_H`H`3a3abbbbC0011ll7##c"""  ""z4#d))<<===&&((zgGEEFFFhhuoo 	E:w(BCDDD$txx7O7O2P2PQQ$Wc=AAA $$$%'
!
 
	4 
	4FH xx+'+F|
8$z5'5CC
CCDDD&&&z'77hh{++..AA.11dhh~6677 8 
 
   	 !3,*0.'@ 2 M  ##F++K$&	 (=+  (8 ;
 
 $K0 	/ 	/Exx*#';	% ???&&2l+J*tUm44 <ZA9M9M1;.	#	#":;;!899 **--///N4J!....=3H -%"0Ih$"/Ih88'((4 $%8 9Ifz'6688H%% 7 
 
   	 HH_--M$((#34455NIII(##^##KK}$$FF~%%FF%vF((9%%&&6,',Wq1Ea%(##<1mm&2GG1V-v;hh{++'	  
 
 %Wc=AAAxx//t;b@V@V^b@bXX/00NTXXo5N5N
 	J 	:a    ll.11xx--6:hh?P6Q6Q[E$01222W[26((?2K2KU%_-...QU88Hf--hh{++'  
 
 %Wc=AAA&&6,nnhh{F33txx!,,--HHY''BE!H$8eAhhdBE!H$8eAhhdhh{++'  
 
 %Wc=AAA 4 4.;N   \ \$Wc=AAAkk$((62..(5n  V V$Wc=AAA!!=:w(DEFFFc%jj$((9:M:MNN$Wc=AAA:w <& < <=>>>r   rM  r	   c                   | j         dk    s	| j        du rddiS | j         dk    rddiS | j         dk    s| j        r| j        =dd	i}t	          | j        t                    r| j                            d
          |d
<   |S ddiS )a!  Choose the next ladder step from semantic evidence, in precedence order.

    An escalation recommendation is advisory. It never overrides a confirmed
    effect and it never turns an unverifiable action into permission to repeat
    input. The model must first obtain fresh evidence.
    	confirmedTdecisiondoneunverifiableverify_fresh_statesuspected_noopNescalaterecommended)effectverifiedr   r   rG  
escalationdictrK   )rM  r]  s     r   _classify_action_resultrh    s     z[  CLD$8$8F##
z^##011
z%%%SV%sx7K$.
#;cnd++ 	H&)n&8&8&G&GH]#,--r   c                z   | j         | j        d}| j        r
| j        |d<   | j        
| j        |d<   | j        
| j        |d<   | j        
| j        |d<   | j        
| j        |d<   | j        
| j        |d<   | j        
| j        |d<   | j	        
| j	        |d	<   | j
        r
| j
        |d
<   t          |           |d<   |S )Nr   messagere  rd  rf  pathdegradedr   r   r<   r   )r   r   rj  re  rd  rf  rk  rl  r   r   r<   rh  )rM  payloads     r   _action_payloadrn  /  s    %(VszBBG
{ ) [	
 |!l

zJ
~! #
x(
|!l

$#&#4 
x(
x #(055GINr   c                D    t          j        t          |                     S rS   )r   r   rn  )rM  s    r   rC  rC  K  s    :oc**+++r   d   i     	image_b64Optional[Tuple[int, int]]c                   | sdS 	 t          j        | d          }n# t          $ r Y dS w xY w|                    d          rct	          |          dk    rP	 t          j        d|dd                   \  }}t          |          t          |          fS # t          $ r Y dS w xY w|                    d          rt	          |          d	k    rvd
}|dz   t	          |          k     r]||         dk    r|dz  })||dz            }|d
z  }|dk    r9|t	          |          k     r&||         }|dz  }|dk    r|t	          |          k     &|dv r}|dk    rn|d
z   t	          |          k    rnt                              |||d
z            d          }|d
k     s||z   t	          |          k    rn|dv rv|dk    rpt                              ||dz   |dz            d          }t                              ||dz   |dz            d          }t          |          t          |          fS ||z  }|dz   t	          |          k     ]dS )a!  Return (width, height) for common inline screenshot formats.

    Some providers reject images below 8x8 before the model sees the tool
    result. Inspecting the encoded bytes here lets computer_use fall back to
    its AX/SOM text payload instead of sending an unusable placeholder.
    NFvalidates   PNG

   z>II   s      r1  	      r6  >            big>                                             r     )	base64	b64decoderi   
startswithr   structunpackrJ  
from_bytes)rr  rawr   r   imarkersegment_lens          r   _image_dimensions_from_b64r  [  s     ty5999   tt ~~*++ CB	"M%RU<<ME6u::s6{{** 	 	 	44	 ~~k"" s3xx!||!ec#hh1v~~QQZFFAD..QS\\QQ D..QS\\ %%~~1us3xx..Qq1uWu==KQ!k/CHH"<"<    ""AE!a%K(8%@@s1q5Q;'7??5zz3v;;..A3 !ec#hh4 4s    
++=B 
B#"B#r   rJ  c                    | t           S 	 t          |           }n# t          t          f$ r
 t           cY S w xY w|dk     rt           S |t          k    rt          S |S )ap  Validate the caller-supplied ``max_elements``.

    Falls back to :data:`_DEFAULT_MAX_ELEMENTS` for missing / non-integer /
    sub-1 inputs so the cap can never be silently disabled by a malformed
    tool-call argument. Clamps oversized values to
    :data:`_MAX_ALLOWED_MAX_ELEMENTS` so a caller cannot bypass the
    safeguard by passing a very large integer.
    Nr6  )_DEFAULT_MAX_ELEMENTSrJ  	TypeError
ValueError_MAX_ALLOWED_MAX_ELEMENTS)r   ns     r   rA  rA    ss     }$$%JJz" % % %$$$$%1uu$$$$$((Hs    66rL  r
   r  c                    t          | j                  }| j        d |         }t          d|t          |          z
            }| j        rt	          | j        pd          nd }|r|d         n| j        }|r|d         n| j        }t          |o!|d         t          k     p|d         t          k               }t          |          }	d| j
         d| d| | j        r
d| j         ndz   | j        r
d| j        ndz   | d	g}
|	r|
                    |	           |r8|
                    d
|d          d|d          dt           dt           d	           d                    |
          }| j        r!| j
        dk    r|st!                      rt#          | |          }||S |
                    d           |r)|
                    dt          |           d| d           | j
        ||| j        | j        d |D             |d                    |
          dd	}|r||d<   t%          j        |          S | j        }|s(| j        d d         }|                    d          rdnd}dd|dddd| d | j         id!g|| j
        |||| j        d"d#S |r)|
                    dt          |           d| d           d                    |
          }| j
        ||| j        | j        d$ |D             ||d%}|r||d<   t%          j        |          S )&Nr   re   r6  zcapture mode= r&  z app=z window=z interactable element(s):z  (screenshot omitted: z is below the z provider minimum)
r  u     (vision unavailable: the auxiliary vision model could not be reached; screenshot omitted. Element-index actions still work — drive via the element list above.)z  (response truncated to z of z5 elements; raise max_elements or pass app= to narrow)c                ,    g | ]}t          |          S rB   _element_to_dictrF   r   s     r   rH   z%_capture_response.<locals>.<listcomp>  s!    KKKQ-a00KKKr   T)	r   r   r   r   r   r   total_elementsr   vision_unavailabletruncated_elementsrq  /9j/
image/jpegz	image/pngrP   )r   rP   	image_urlr#  zdata:z;base64,)r   r  )r   r   r   r   	png_bytes)_multimodalcontenttext_summaryr<   c                ,    g | ]}t          |          S rB   r  r  s     r   rH   z%_capture_response.<locals>.<listcomp>   s!    CCCQ%a((CCCr   )r   r   r   r   r   r   r  r   )r   r   maxr   r  r   r   r   _MIN_PROVIDER_IMAGE_DIMENSION_format_elementsr   r   r   extendr   join _should_route_through_aux_vision!_route_capture_through_aux_visionr   r   image_mime_typer  png_bytes_len)rL  r  r  visible_elementsr  image_dimensionsresponse_widthresponse_heightimage_too_smallelement_indexsummary_linesr   routedrm  _mime_b64_prefixs                   r   r@  r@    sy   &&N|M\M2Q5E1F1F FGGHK]1#+2CDDDY],<K%a((#)N-=M&q))3:O 	
Q"?? C"%BB	 O %%566MEEE>EEOEE #/37R	1.1.>F*c&***B	H 444	M  ,]+++  
 &6q&9    <LQ<O    9   <Y     	
 	
 	

 ii&&G
{ A
sx4''' ,-- "	'6sGDDF!   >  
 " $$&4D0E0E & &%& & &   ')w # 0KK:JKKK"099]33&*
 
G " C0B,-:g&&& # 	T+bqb/K$/$:$:6$B$BSLLE
  11$$&Je&J&JS[&J&JKM M
 $ X/!/c>OQ Q

 

 
	
  
:,<(=(= : :> : : :	
 	
 	
 ii&&G!w(CC2BCCC(	 	G  ;(:$%:gr   i  r  bytesextmax_dimc                   	 ddl m} ddlm} |                     ||                     }t          |j                  |k    r| S |                    ||f            |            }|                    ||dk    rdnd           |	                                S # t          $ r'}t                              d|           | cY d	}~S d	}~ww xY w)
u   Downscale encoded image bytes so the longest side is <= max_dim.

    Returns the original bytes unchanged when the image already fits or when
    Pillow is unavailable/fails — no worse than the pre-shrink behavior.
    r   )BytesIO)Image.jpgJPEGPNG)formatz*computer_use: vision downscale skipped: %sN)ior  PILr  openr  size	thumbnailsavegetvalueri   r   r   )r  r  r  r  r  imgoutexcs           r   _shrink_capture_for_visionr  3  s    jj&&sx==G##Jw()))giisf}}VV%@@@||~~   A3GGG





s%   AB AB 
C$C CCc                 p   	 ddl m} m} ddlm} ddlm} n3# t          $ r&}t          	                    d|           Y d}~dS d}~ww xY w	  |            pd} |             pd}n3# t          $ r&}t          	                    d	|           Y d}~dS d}~ww xY wt          |          t          |          f}t                              |          }||S 	  |            }	t           ||||	                    }
n3# t          $ r&}t          	                    d
|           Y d}~dS d}~ww xY w|
t          |<   |
S )u  Return True when ``_capture_response`` should hand the PNG to aux vision.

    Reads the active main provider/model and the loaded config and asks the
    routing helper. Any failure (config import, runtime override missing,
    etc.) returns False so the existing multimodal envelope continues to be
    returned — fail open on the routing decision so a broken config can
    never silently drop the screenshot for vision-capable main models.
    r   )_read_main_model_read_main_providerload_config)"should_route_capture_to_aux_visionz2computer_use: aux-vision routing import failed: %sNFre   z7computer_use: aux-vision routing config read failed: %sz4computer_use: aux-vision routing decision failed: %s)agent.auxiliary_clientr  r  hermes_cli.configr  !tools.computer_use.vision_routingr  ri   r   r   r>   rY   rK   r   )r  r  r  r  r  providermodel	cache_keyr   cfgr]  s              r   r  r  H  s   PPPPPPPP111111	
 	
 	
 	
 	
 	
 	
    I3OOOuuuuu&&((.B  ""(b   NPSTTTuuuuu XE

+I$((33Fkmm::8UCPPQQ   KSQQQuuuuu *2I&OsD    
AAAA$ $
B.BB$C9 9
D)D$$D)c                    	 ddl m}   |             pi                     d          pi                     dd          }n# t          $ r Y dS w xY wt	          |pd                                                                          }|dv r|ndS )zDMode for ``capture_after`` follow-ups. Default ``som`` (screenshot).r   r  computer_usecapture_after_moder   >   r  r   r  )r  r  rK   ri   r>   rC   rD   )r  r  r   s      r   _capture_after_moder  n  s    111111#((88>BCC %
 
    uus|e""$$**,,D22244=s   ;> 
AAr   c                   | j         sdS 	 ddl}ddl}ddl}ddlm} ddlm} ddlm	} n3# t          $ r&}t                              d|           Y d}~dS d}~ww xY wd}		 	 |                    | j         d          }
nk# t          $ r^}t                              d	|           Y d}~|	6	 |                    t          |	                     dS # t          $ r Y dS w xY wdS d}~ww xY w| j        pd
}|dk    s$|s%| j         dd                             d          rd}nd} |dd          }|                    dd           |d|                                j         | z  }	t+          |
|          }
|	                    |
           d| } | |t          |	          |                    }nk# t          $ r^}t                              d|           Y d}~|	6	 |                    t          |	                     dS # t          $ r Y dS w xY wdS d}~ww xY w	 |	4	 |                    t          |	                     nL# t          $ r Y n@w xY wn;# |	4	 |                    t          |	                     w # t          $ r Y w w xY ww xY wd
}t1          |t                    r	 t3          j        |          }t1          |t6                    r6t          |                    d          pd
                                          }n0# t<          t2          j        f$ r |                                }Y nw xY w|sdS t3          j         | j!        | j"        | j#        | j$        | j%        d | j&        D             ||dd	          S )a\  Pre-analyse the captured PNG via ``vision_analyze`` and return a text result.

    The captured base64 PNG is materialised to ``$HERMES_HOME/cache/vision/``
    and handed to ``vision_analyze_tool`` with a generic describe prompt.
    The resulting text description is merged into the existing AX/SOM
    summary so the main model receives a single text payload that mentions
    every interactable element AND a description of what the screenshot
    looked like.

    Returns:
      A JSON-encoded text response on success.
      ``None`` on failure (caller falls back to the multimodal envelope).
    Nr   )get_hermes_dir)
_run_async)vision_analyze_toolz*computer_use: aux-vision import failed: %sFru  z1computer_use: failed to decode capture base64: %sre   r  rq  r  r  z.pngzcache/visiontemp_vision_imagesT)parentsexist_okcomputer_use_ak  Describe what is visible in this desktop application screenshot in concise but specific terms. Mention the app name and window title if visible, the overall layout, any labelled buttons, menus or text fields, and any prominent text content the user would need to know about. Do not invent details that are not actually visible.

AX/SOM index for cross-reference:
zacomputer_use: auxiliary.vision pre-analysis failed (%s); returning to caller without aux analysisanalysisc                ,    g | ]}t          |          S rB   r  r  s     r   rH   z5_route_capture_through_aux_vision.<locals>.<listcomp>  s!    ???Q%a((???r   zauxiliary.vision)	r   r   r   r   r   r   r   vision_analysisvision_analysis_routed_via)'r   r  rw   uuidhermes_constantsr  model_toolsr  tools.vision_toolsr  ri   r   r   r  unlinkr>   r  r  mkdiruuid4hexr  write_bytesr   rG  r   loadsrg  rK   rC   r  JSONDecodeErrorr   r   r   r   r   r   r   )rL  r   _base64_os_uuidr  r  r  r  temp_image_pathr  _mime_for_extr  	cache_dirpromptresult_jsonanalysis_textparseds                     r   r  r  |  s   " ; t
    333333******:::::::   A3GGGttttt O.	##CK%#@@CC 	 	 	LLLcRRR444J &

3//00000    '&O	 +1rL(((3;rPQr?C]C]^dCeCe(CCC"N>3GHH	t444#&Nekkmm6G&N&N&NN(c22##C(((< 3:< < 	 !jO 4 4f==
 
    7	
 	
 	

 ttt&

3//00000    '& 	 &

3//0000    '?&

3//0000    ' M+s## 0	0Z,,F&$'' J #FJJz$:$:$@b A A G G I I4/0 	0 	0 	0'--//MMM	0  t:*w(??#,???(&8
 
 
 
 
s   * 
AAA!A> =F2 >
C&C!#F2 *"C 
CC!C&&CF2 1I 2
H<HI "H 
HHHI !"I 
IIJ"I<;J<
J	JJ		J(AL *L54L5
do_capturec                   |st          |          S |j        st          |          S 	 t          | dd           pi }|                    d          }|                    d          }t	                      }|||                     |||          }n&|                     |t          | dd                     }nA# t          $ r4}t                              d|           t          |          cY d }~S d }~ww xY wt          |          }	t          |	t                    r|	                    d          rot          j        t          |                    }
|
d	z   |	d
         d         d         z   |	d
         d         d<   |
d	z   |	d         z   |	d<   t          |          |	d<   |	S 	 t          j        |	          }n # t           t          j        f$ r d|	i}Y nw xY w|                    t          |                     t          j        |          S )N_last_targetr   r   )r   r   r   	_last_appr  zfollow-up capture failed: %sr  z

r  r   rP   r  action_resultr   )rC  r   getattrrK   r  r   ri   r   r   r@  rG  rg  r   r   rn  r   r  r  r?  )r   rM  r  targetr   r   r   rL  r   respprefixdatas               r   rD  rD    s+     #c""" 6 #c"""# .$77=2jjJJ{++	"$$?y4//t	/JJCC//t+t1T1T/UUC # # #5q999c""""""""# S!!D$ $((="9"9  OC0011%+f_tIq7I&7Q%QY6"%n1EE^ / 4 4_!z$t+, ! ! !4 !KK$$%%%:ds0   BB8 8
C6)C1+C61C6"F7 7GG(   r   List[UIElement]	max_lines	List[str]c                l   g }| d |         D ]j}|j                             dd          d d         }|                    d|j         d|j         d|d|j         |j        rd|j         dndz              kt          |           |k    r)|                    d	t          |           |z
   d
           |S )Nr  r  r  z  #z @ z []re   z  ... +z( more (call capture with app= to narrow))labelreplacer   indexroleboundsr   r   )r   r  r  r   r  s        r   r  r    s    Cjyj! 7 7c**3B3/

BBB16BBEBBBB'(u4m15mmmm"6 	7 	7 	7 	7
8}}y  

`S]]Y6```aaaJr   r   r   c                ^    | j         | j        | j        t          | j                  | j        dS )N)r  r  r  r   r   )r  r  r  rH  r   r   )r   s    r   r  r    s0    qx..u  r   c                 B    t           j        dvrdS ddlm}   |             S )a  Return True iff computer_use can run on this host.

    Conditions: macOS, Windows, or Linux + cua-driver binary installed (or
    override via env). cua-driver runs on all three; the Linux path is
    headed/X11 today (Wayland via XWayland), pure-Wayland progress tracked
    upstream. Linux users see specific blocked checks via
    `hermes computer-use doctor` if their session is incomplete (e.g. no
    DISPLAY set).
    )darwinwin32linuxFr   cua_driver_binary_available)sysplatformry   r'  r&  s    r   check_computer_use_requirementsr*  +  s8     |777uJJJJJJ&&(((r   c                     ddl m}  | S )Nr   COMPUTER_USE_SCHEMA)tools.computer_use.schemar-  r,  s    r   get_computer_use_schemar/  ;  s    ======r   r   )r=   r>   r   r?   )rP   r>   r   rQ   )r`   r>   r   r>   )re   )r`   r>   r   r   )r`   r>   r   r   )r   r   r   r   )r   r>   r   r   r`   r>   r   rQ   )r   r>   r   r   r   r>   )r   r   r   r>   r   r   r   r   )rM  r	   r   r   )rM  r	   r   r>   )rr  r>   r   rs  )r   r   r   rJ  )rL  r
   r  rJ  r   r   )r  r  r  r>   r  rJ  r   r  r   )r   r>   )rL  r
   r   r>   r   rQ   )r   r   rM  r	   r  r   r   r   )r  )r   r  r  rJ  r   r  )r   r   r   r   )r   r   )Or   
__future__r   atexitr  r   loggingrw   rL   r  r(  rt   typingr   r   r   r   r   tools.computer_use.backendr	   r
   r   r   	getLoggerr   r   r   r   r?   _SAFE_ACTIONSr   r   rJ   rO   compile
IGNORECASErT   rX   Lockrs   rY   __annotations__rZ   r[   r\   r]   r   r^   r_   rk   r   r   r   registerr   rz   r   r   r   r   rh  rn  rC  r  r  r  r  rA  r@  _MAX_VISION_DIMr  r  r  r  rD  r  r  r*  r/  rB   r   r   <module>r=     s  $ $ $L # " " " " "     				 				  



     3 3 3 3 3 3 3 3 3 3 3 3 3 3            
	8	$	$      	     
 !y " " "    I+++,,I,,,--I"""##I###$$I---.. IuclI***++I'''((Ix   xue     BJ'77BJ%r}55BJ'77BJ%r}55BJ$bm44BJ/??     	   8:  9 9 9 9)- - - - -+-	 - - - -24  4 4 4 4,.  . . . .  !!)+  + + + + " " " " "   <@ @ @ @ @F/ / / /d,E ,E ,E ,E^ ( ) ) )$ $ $ $;9 ;9 ;9 ;9 ;9% ;9 ;9 ;9DG> G> G> G>V )+/E /E /E /E /Ed   4F? F? F? F?Z. . . .(   8, , , ,   !  ! 3 3 3 3l   , ?T     R  />    *# # # #L> > > >f f f fR* * * *Z       ) ) ) )      r   