
    Pmj)                        d Z ddlmZ ddlmZmZ ddlmZmZ ddl	m
Z
mZmZmZmZ e G d d                      Ze G d d	                      Ze G d
 d                      Z G d de          ZdS )zAbstract backend interface for computer use.

Any implementation (cua-driver over MCP, pyautogui, noop, future Linux/Windows)
must return the shape described below. All methods synchronous; async is
handled inside the backend implementation if needed.
    )annotations)ABCabstractmethod)	dataclassfield)AnyDictListOptionalTuplec                      e Zd ZU dZded<   ded<   dZded<   dZd	ed
<   dZded<   dZded<   dZ	ded<    e
e          Zded<   dZded<   ddZdS )	UIElementz/One interactable element on the current screen.intindexstrrole label)r   r   r   r   zTuple[int, int, int, int]boundsappr   pid	window_iddefault_factoryDict[str, Any]
attributesNOptional[str]element_tokenreturnTuple[int, int]c                :    | j         \  }}}}||dz  z   ||dz  z   fS )N   )r   )selfxywhs        @/home/thesage/.hermes/hermes-agent/tools/computer_use/backend.pycenterzUIElement.center#   s,    [
1a16z1qAv:%%    )r   r    )__name__
__module____qualname____doc____annotations__r   r   r   r   r   r   dictr   r   r)    r*   r(   r   r      s         99JJJIIIEOOOO(4F4444CMMMMCLLLLI!&t!<!<!<J<<<< $(M''''& & & & & &r*   r   c                      e Zd ZU dZded<   ded<   ded<   dZded	<    ee
          Zded<   dZ	ded<   dZ
ded<   dZded<   dZded<   dS )CaptureResultu  Result of a screen capture call.

    At least one of png_b64 / elements is populated depending on capture mode:
      * mode="vision" → png_b64 only
      * mode="ax"     → elements only
      * mode="som"    → both (default): PNG already has numbered overlays
                         drawn by the backend, and `elements` holds the
                         matching index → element mapping.
    r   moder   widthheightNr   png_b64r   zList[UIElement]elementsr   r   window_titler   png_bytes_lenimage_mime_type)r+   r,   r-   r.   r/   r7   r   listr8   r   r9   r:   r;   r1   r*   r(   r3   r3   (   s           IIIJJJKKK!G!!!! %d ; ; ;H;;;;CMMMMLM &*O))))))r*   r3   c                      e Zd ZU dZded<   ded<   dZded<   dZd	ed
<    ee          Z	ded<   dZ
ded<   dZded<   dZded<   dZded<   dZded<   dZded<   dZded<   dS )ActionResultu=  Result of any action (click / type / scroll / drag / key / wait).

    Beyond the transport-level ``ok`` flag, this carries cua-driver's
    structured action verdict so the model can follow the documented
    verify → escalate ladder (NousResearch/hermes-agent#67052). ``ok`` stays
    tool/transport success only — it is NOT the semantic verdict. Read
    ``effect`` / ``escalation`` to decide the next rung. All structured
    fields are optional and additive: an older driver that omits
    ``structuredContent`` leaves them ``None`` and behavior is unchanged.
    boolokr   actionr   messageNzOptional[CaptureResult]capturer   r   metazOptional[bool]verifiedr   effectOptional[Dict[str, Any]]
escalationpathdegradeddelivery_modecode)r+   r,   r-   r.   r/   rB   rC   r   r0   rD   rE   rF   rH   rI   rJ   rK   rL   r1   r*   r(   r>   r>   F   s         	 	 HHHKKKG (,G++++ 5666D6666  $H#### F     ,0J////D#H#####'M'''' Dr*   r>   c            
         e Zd ZdZedJd            ZedJd            ZedKd            Ze	 	 	 	 dLdMd            Zed	d	d	ddd	d	dddNd"            Z	ed	d	d	d	dd	d	dd#dOd)            Z
ed*d	d	d	d	d	dd+dPd.            Zed	dd/dQd1            Zed	dd/dRd3            ZedSd5            ZdSd6ZedTdUd8            ZedVdWd:            ZedXd<            ZdYd?ZdYd@Zd	d	dAdZdFZd[dIZd	S )\ComputerUseBackendz<Lifecycle: `start()` before first use, `stop()` at shutdown.r   Nonec                    d S Nr1   r#   s    r(   startzComputerUseBackend.startr   s     Sr*   c                    d S rQ   r1   rR   s    r(   stopzComputerUseBackend.stopu   s    Cr*   r?   c                    dS )zReturn True if the backend can be used on this host right now.

        Used by check_fn gating and by the post-setup wizard.
        Nr1   rR   s    r(   is_availablezComputerUseBackend.is_availablex         r*   somNr4   r   r   r   r   Optional[int]r   r3   c                    d S rQ   r1   )r#   r4   r   r   r   s        r(   rC   zComputerUseBackend.capture   s	     r*   left   F)elementr$   r%   buttonclick_count	modifiersrK   bring_to_frontr^   r$   r%   r_   r`   r   ra   Optional[List[str]]rK   rb   r>   c                   d S rQ   r1   )	r#   r^   r$   r%   r_   r`   ra   rK   rb   s	            r(   clickzComputerUseBackend.click   	     sr*   )from_element
to_elementfrom_xyto_xyr_   ra   rK   rb   rg   rh   ri   Optional[Tuple[int, int]]rj   c                   d S rQ   r1   )	r#   rg   rh   ri   rj   r_   ra   rK   rb   s	            r(   dragzComputerUseBackend.drag   rf   r*      )amountr^   r$   r%   ra   rK   rb   	directionro   c                   d S rQ   r1   )	r#   rp   ro   r^   r$   r%   ra   rK   rb   s	            r(   scrollzComputerUseBackend.scroll   rf   r*   )rK   rb   textc                   d S rQ   r1   )r#   rs   rK   rb   s       r(   	type_textzComputerUseBackend.type_text   s    ADr*   keysc                   dS )z7Send a key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return'.Nr1   )r#   rv   rK   rb   s       r(   keyzComputerUseBackend.key   rX   r*   List[Dict[str, Any]]c                    dS )z9Return running apps with bundle IDs, PIDs, window counts.Nr1   rR   s    r(   	list_appszComputerUseBackend.list_apps   rX   r*   c                    g S )zReturn visible native windows with PID and window identifiers.

        Optional compatibility hook: backends that predate window discovery
        remain instantiable and simply report no windows.
        r1   rR   s    r(   list_windowszComputerUseBackend.list_windows   s	     	r*   raise_windowc                    dS )zJRoute input to `app` (by name or bundle ID). Default: focus without raise.Nr1   )r#   r   r~   s      r(   	focus_appzComputerUseBackend.focus_app   rX   r*   valuec                    dS )zSet a native value on an element (e.g. AXPopUpButton selection).

        `element` is the 1-based SOM index returned by a prior capture call.
        Nr1   )r#   r   r^   s      r(   	set_valuezComputerUseBackend.set_value   rX   r*   r   c                     ddddddS )NFrefusedtyped_browser_unavailablezOThis computer-use backend has no typed browser route; use native capture/input.T)r@   statusrL   rB   native_fallback_requiredr1   r1   r*   r(   _typed_browser_unavailablez-ComputerUseBackend._typed_browser_unavailable   s"     /h(,
 
 	
r*   kwargsr   c                *    |                                  S )z@Optional exact-bind/read hook; native-only backends fail closed.r   r#   r   s     r(   typed_browser_statez&ComputerUseBackend.typed_browser_state       ..000r*   c                *    |                                  S )z6Optional setup hook; native-only backends fail closed.r   r   s     r(   typed_browser_preparez(ComputerUseBackend.typed_browser_prepare   r   r*   )tab_idargsdriver_toolr   r   rG   c               *    |                                  S )z9Optional mutation hook; native-only backends fail closed.r   )r#   r   r   r   s       r(   typed_browser_actionz'ComputerUseBackend.typed_browser_action   s     ..000r*   secondsfloatc           	         ddl }|                    t          dt          |d                               t	          ddd|dd	
          S )z#Default implementation: time.sleep.r   Ng        g      >@Twaitzwaited z.2fs)r@   rA   rB   )timesleepmaxminr>   )r#   r   r   s      r(   r   zComputerUseBackend.wait   sU    

3sC..//000tF<Tg<T<T<T<TUUUUr*   )r   rO   )r   r?   )rY   NNN)
r4   r   r   r   r   rZ   r   rZ   r   r3   )r^   rZ   r$   rZ   r%   rZ   r_   r   r`   r   ra   rc   rK   r   rb   r?   r   r>   )rg   rZ   rh   rZ   ri   rk   rj   rk   r_   r   ra   rc   rK   r   rb   r?   r   r>   )rp   r   ro   r   r^   rZ   r$   rZ   r%   rZ   ra   rc   rK   r   rb   r?   r   r>   )rs   r   rK   r   rb   r?   r   r>   )rv   r   rK   r   rb   r?   r   r>   )r   ry   )F)r   r   r~   r?   r   r>   rQ   )r   r   r^   rZ   r   r>   )r   r   )r   r   r   r   )r   r   r   r   r   rG   r   r   )r   r   r   r>   )r+   r,   r-   r.   r   rS   rU   rW   rC   re   rm   rr   ru   rx   r{   r}   r   r   staticmethodr   r   r   r   r   r1   r*   r(   rN   rN   o   s       FF    ^  ^   ^  !!#'    ^  "&)-'+$     ^  '+$(-1+/)-'+$     ^ 
 !%)-'+$     ^ EI).E E E E E ^E ?C#(F F F F F ^F
 H H H ^H    Y Y Y Y ^Y     ^ 
 
 
 \
1 1 1 11 1 1 1 !%)-1 1 1 1 1 1V V V V V Vr*   rN   N)r.   
__future__r   abcr   r   dataclassesr   r   typingr   r	   r
   r   r   r   r3   r>   rN   r1   r*   r(   <module>r      sb    # " " " " " # # # # # # # # ( ( ( ( ( ( ( ( 3 3 3 3 3 3 3 3 3 3 3 3 3 3 & & & & & & & &0 * * * * * * * *: % % % % % % % %PJV JV JV JV JV JV JV JV JV JVr*   