Pmjj_dZddlmZddlZddlZddlZddlZddlmZddl m Z m Z m Z ej eZdZehdZehdZejd Zd d dIdZdJdZdKdZdLdZedGddZdMdZdMd ZdNd"Zdd#d$dOd'ZdPd*ZdQd,Z dQd-Z!dRd.Z"dRd/Z#dSd0Z$dRd1Z%dTd2Z&d3Z'dUd4Z(dUd5Z)dTd6Z*dVd8Z+dWd:Z,dXd<Z-Gd=d>Z.dYd?Z/dZdCZ0d[dDZ1d\dFZ2d]dGZ3ddHgZ4dS)^aCGateway Health & Diagnostics OTLP export runtime. This exporter emits operator-owned gateway service-health metrics plus narrow redacted diagnostic events. It is deliberately in-process and fail-open so it works under systemd, launchd, s6, containers, tmux, nohup, or a simple shell without a sidecar/watchdog dependency. ) annotationsN) dataclass)AnyDictOptionalzhermes.gateway.diagnostics> cloud.regioncloud.platformcloud.providerservice.versionservice.namespacedeployment.environment.name service.nametelemetry.scopeservice.instance.id> nameversionplatformseverity new_state old_state subsystem error_code error_classz^[A-Za-z0-9._:/-]{1,128}$ilimitrawrrintreturnstrcz ddlm}|t|pdpdd|S#t$rYdSwxYw)Nr)redact_for_exportz [redacted]z[redaction-unavailable])agent.monitoring.redactionr!r Exception)rrr!s L/home/thesage/.hermes/hermes-agent/agent/monitoring/gateway_health_export.py_redact_stringr&/sf)@@@@@@!!#ciR..11A\6E6JJ )))((()s ), ::Dict[str, str]cbi}t|ts|S|D]\}}t|}|tvs| |dkrddlm}||||<;t|}t|set|d|kr{|||<|S)zIAllowlist bounded resource labels and reject values changed by redaction.Nrr_safe_instance_idr) isinstancedictitemsr_RESOURCE_ATTRIBUTE_KEYSagent.monitoring.gateway_healthr*_SAFE_RESOURCE_VALUE fullmatchr&)rattrskeyvaluer*texts r%_safe_resource_attributesr77sE c4  iikk   U#hh . . .%-  ' ' ' I I I I I I**511E#J 5zz#--d33   $c * * *d 2 2 c LconfigDict[str, Any]telemetry_scopect|}t|d}ddlm}d|d<|t ||d<||d<|S)zCBuild the safe OTLP resource shared by metrics and diagnostic logs.resource_attributesrr)zhermes-gatewayrrr)_gateway_health_configr7getr0r* _install_id)r9r;ghr3r*s r%_runtime_resource_attributesrBMsu  ' 'B %bff-B&C&C D DEAAAAAA,E.#4#4[5H5H#I#IE  .E  Lr8eventci}tD]F}||}|t|trt |n||d|<G|S)Nzhermes.)_DIAGNOSTIC_ATTRIBUTE_KEYSr?r,rr&)rCr3r4r5s r%_diagnostic_log_attributesrF[sdE)\\ # = :DUC:P:P![!6!6!6V[ooo Lr8T)slotsceZdZUded<dZded<dZded<dZded <dZded <dZded <dZ d ed <dZ ded<ddZ dS)GatewayHealthExportRuntimeboolenableddisabledrreasonNrstreamermetric_provider log_handler log_streamerzOptional[threading.Thread]threadzOptional[threading.Event] stop_eventrNonec|j|j|j|jd|j= t j|jn#t$rYnwxYw ddl m }|}| d|j | |j |j| |jn#t$rYnwxYwd|j |j|jfDdfd }rAt!j|d d }||d d|_ d|_d|_d|_d|_dS)Ng?)timeoutr get_emitterg?cg|]}||SN).0items r% z7GatewayHealthExportRuntime.shutdown..s(    r8rrTc\D]'} |#t$rY$wxYwdSrZ)shutdownr$)r] closeabless r%_closez3GatewayHealthExportRuntime.shutdown.._closesP"  MMOOOO D  s  ))z%hermes-gateway-health-export-shutdownTtargetrdaemong@rrT)rSsetrRjoinrPlogging getLogger removeHandlerr$agent.monitoring.emitterrXflushrN unsubscriberQrO threadingThreadstart)selfrXemitterrbworkerras @r%r`z#GatewayHealthExportRuntime.shutdownps ? & O   ! ! ! ; " K  T  * * *   ' !##11$2BCCCC      < < < < < <!kmmG MM#M & & &}(##DM222 ,##D$5666    D    "mT->@TU           %%<F LLNNN KKK $ $ $  # s% +A88 BB A(C22 C?>C?rf) __name__ __module__ __qualname____annotations__rMrNrOrPrQrRrSr`r[r8r%rIrIesMMMFHOKL)-F----,0J0000444444r8rIcb|pidpi}|dpiS)N monitoringgateway_health_exportr?)r9mons r%r>r>s6 rrJr?)r9rArs r%_enabledrsX  ' 'B   D y!!Rdhhy&9&9Rdhhz>R>R S SSr8F) auto_installpromptrrcH|r% ddlm}|d|n#t$rYnwxYw ddlm}ddlm}ddlm}ddl m }m }m }dd l m} dd lm} dd lm} dd lm} dd lm} ddlm}ddlm}|||| | |||| | | ||d S#t$r}t7d||d}~wwxYw)Nr)ensurez export.otlpr)OTLPLogExporter)OTLPMetricExporter) Observation)INVALID_SPAN_IDINVALID_TRACE_ID TraceFlags) LogRecord)SeverityNumber)LoggerProvider)BatchLogRecordProcessor) MeterProvider)PeriodicExportingMetricReader)Resource) rrrrrrrrrrrrrzOTLP metrics SDK unavailable: )tools.lazy_depsrr$4opentelemetry.exporter.otlp.proto.http._log_exporterr6opentelemetry.exporter.otlp.proto.http.metric_exporterropentelemetry.metricsropentelemetry.tracerrropentelemetry._logsropentelemetry._logs.severityropentelemetry.sdk._logsropentelemetry.sdk._logs.exportropentelemetry.sdk.metricsr opentelemetry.sdk.metrics.exportropentelemetry.sdk.resourcesr RuntimeError)rr _lazy_ensurerrrrrrrrrrrrrexcs r%_require_metrics_sdkrs  > > > > > > Lv 6 6 6 6 6    D LXXXXXX]]]]]]555555UUUUUUUUUU111111??????::::::JJJJJJ;;;;;;RRRRRR888888."4&",. 0$,'>*-J     LLLACAABBKLs$ %%AA?? B! BB! headers_envOptional[Dict[str, str]]ci}|piD]E\}}tjt |}|r||t |<F|SrZ)r.osenvironr?r)rresolved header_nameenv_namevals r%_resolve_headersrse!H"-"3!:!:!&&>,3|,,,,- == Or8c|dr|dtd dzS|dr|dtd dzS|S)Nrz/v1/logsrrrs r%_logs_endpointrsu&&;,3|,,,,- ::''<-3}----.;; Or8cP ddlm}t|S#t$rYdSwxYw)Nr __version__unknown) hermes_clirrr$rs r%_versionrsI******; yys  %%cd ddlm}t|pdS#t$rYdSwxYw)Nrget_active_profile_namedefault)hermes_cli.profilesrrr$rs r%_profilersX??????**,,9 ::: yys ! //cb ddlm}t||S#t$rYdSwxYw)Nr)ensure_install_idr)agent.monitoring.policyrrr$)r9rs r%r@r@sU======$$V,,--- yys  ..ctjdrdStjdstjdrdStjdstjdrdStjdrd Sd S) N INVOCATION_IDsystemd S6_CMD_ARG0 S6_VERSIONs6 containerz /.dockerenvLAUNCHD_SOCKETlaunchdmanual)rrr?pathexistsr[r8r%_supervision_moder s z~~o&&y z~~m$$ |(D(Dt z~~k""bgnn]&C&C{ z~~&''y 8r8c ddlm} ddlm}|pi}n#t$ri}YnwxYw||dt t |ttS)Nr)build_gateway_health_snapshot)read_runtime_statusT)gateway_runningprofile install_idrsupervision_mode) r0rgateway.statusrr$rr@rr)r9rrruntimes r%_read_gateway_snapshotrsMMMMMM666666%%''-2  ( ( v&& *,,    s  **c"ddlm}|S)Nrbuild_cron_health_snapshot)agent.monitoring.cron_healthrrs r%_read_cron_snapshotr*s#GGGGGG % % ' ''r8cd} ddlm}|tdt|z }n,#t$rt ddYnwxYw ddlm}|tdt| z }n,#t$rt ddYnwxYw|S)aCount live background/subagent work that ``active_agents`` does NOT include. ``hermes.gateway.active_agents`` counts foreground turns + in-flight cron jobs + API runs, but deliberately excludes backgrounded ``delegate_task`` subagents, ``terminal(background=true)`` processes, kanban workers, and the runner's own background tasks (they are tracked only for the scale-to-zero suspend guard, ``_scale_to_zero_has_live_background_work``). Without this metric a peer churning through delegated subagents shows ``active_agents=0`` on the fleet dashboard. Best-effort and content-free: a single integer, no job/task identity. Returns 0 if a source can't be imported. Delegation is counted TASK-granular (``active_task_count``): a fan-out batch of N subagents contributes N, not 1, so the metric reflects real concurrent subagent load rather than dispatch-unit/pool-slot count. This intentionally differs from the async pool's capacity accounting (one batch = one slot). r)active_task_countz-background-work async-delegation count failedTexc_info)process_registryz-background-work process-registry count failed) tools.async_delegationrmaxrr$loggerdebugtools.process_registryr count_running)totalrrs r%_read_background_work_countr0s " EU<<<<<< Q--//00111 UUU Dt TTTTTUU;;;;;; Q,::<<==>>> UUU Dt TTTTTU Ls!.3&AA 8B&CCc ddlm}tdt|S#t$r t ddYdSwxYw)uCount live async delegation UNITS (dispatch/pool slots). Complements ``_read_background_work_count`` (which is task-granular): this counts each ``delegate_task`` dispatch as ONE regardless of fan-out width, matching the async pool's capacity accounting (a batch = one slot). Together the two metrics let an operator see both slot pressure (``background_delegations``, alert vs ``max_concurrent_children``) and real concurrent subagent load (``background_work``). Delegations only — it does not include ``terminal(background)`` / kanban work, which are already folded into ``background_work``. Best-effort; 0 if the source can't be imported. r active_countz#background-delegations count failedTr)rrrrr$rrrs r%"_read_background_delegations_countrQsp7777771c,,..))***  :T JJJqqs*-&AAc.t|} ddlm}|jrt |jdjni}|j|dt||j|dt|n`#t$rS}t dt|j tdd Yd}~nd}~wwxYw t}nb#t$rU}t d t|j td d |cYd}~Sd}~wwxYw|j|j|S) Nr) GatewayMetrichermes.gateway.background_workrr5 attributes%hermes.gateway.background_delegationszIbackground-work snapshot unavailable; metric not exported (error_type=%s)z"background-work snapshot tracebackTrzMcron health snapshot unavailable; cron telemetry not exported (error_type=%s)zcron health snapshot traceback)rr0rmetricsr-rappendrrr$rwarningtyperurrextend)r9gateway_snapshotrbaser cron_snapshots r%_read_runtime_snapshotrfs-f55JAAAAAA?O?W_t$,Q/:;;;]_ '' M5133        '' M<8::        JJJ W II      9D IIIIIIII J  +--      [ II      5 EEE  ##M$9::: s8BB$$ D.A C<<DD E3A E.(E3.E3rTct|}|ddsdS ddlm}t |}|jD]}||dS#t$r t ddYdSwxYw)Ndiagnostic_events_enabledTr)rsz#gateway health snapshot emit failedr) r>r?agent.monitoringrsreventsemitr$rr)r9rArssnapshotrCs r%_emit_snapshot_eventsr s  ' 'B 66-t 4 4K,,,,,,)&11_  E LL      KKK :T JJJJJJKs4A&B B sdkc t}|ddsdSt}tt |d}t |d}|d||pd}t dt|dd d z}|d || }td } |d|g|d | } | d} |dgd} dfd } | D]#}| || |g$| S)Nmetrics_enabledTrrrrheadersexport_interval_seconds<ir)export_interval_millisgateway_healthr;rr)metric_readersresourcezhermes.gateway.healthr)zhermes.gateway.upzhermes.gateway.statezhermes.gateway.active_agentszhermes.gateway.busyzhermes.gateway.drainablez hermes.gateway.restart_requestedrrzhermes.platform.upzhermes.platform.degradedz+hermes.cron.scheduler.heartbeat_age_secondsz.hermes.cron.scheduler.last_success_age_secondsz*hermes.cron.scheduler.catch_up_occurrenceszhermes.cron.jobs.enabledzhermes.cron.jobs.runningzhermes.cron.jobs.overduerrcdfd }|S)Nc t}fd|jDS#t$r!tddgcYSwxYw)NcRg|]#}|jk |j|j$Sr[r)r\mrrs r%r^zI_start_metric_provider..callback.._cb..s8gggqXYX^bfXfXf AGQ\::XfXfXfr8zgateway metric callback failedTr)rrr$rr)_optionsr rr9rs r%_cbz5_start_metric_provider..callback.._cbsp 1&99gggggHDTgggg    = MMM  s"&(AArZr[)rrrr9s` r%callbackz(_start_metric_provider..callbacks3         r8) callbacks)rr) r>r?rrrrrrrBcreate get_metercreate_observable_gauge)r9r rArrrexporter interval_msreaderresource_attrsprovidermeter metric_namesr  metric_namers` @r%_start_metric_providerr-s  ' 'B 66#T * *t   DDHHZ$8$8 9 9::Htxx 6677G(s'((GOtTTTHaRVV$=rBBCCDDtKK 1S0 1(S^ _ _ _F1 0N$s?#xZ''77H   6 7 7Em$KL&$VV  %%khh{>S>S=T%UUUU Or8rc|d}t|pd}|dvr|jS|dkr|jS|dvr|jS|dkr|jS|jS)Nrr>fatalcriticalerror>info informationr)rlowerFATALERRORINFODEBUGWARN)r rrsevs r%_severity_numberr;s)*N h#) $ $ * * , ,C ##### g~~## %%%"" g~~##  r8c*eZdZdZd dZdd Zdd Zd S)GatewayDiagnosticLogStreamerzEEmitter subscriber that sends gateway diagnostic events as OTLP logs.r9r:r cVt|}t|d}tt |d}t |d}|d|d||_|d|d ||pd |_|j |j|j t|_ |d |_ ||_d |_dS) Nrrgateway_diagnosticsrrr)rrrrrr)rrr?rrrBr" _provider _processoradd_log_record_processor get_logger_DEFAULT_DIAGNOSTIC_SCOPE_logger _LogRecord_sdkexported)rrr9r rrrr(s r%__init__z%GatewayDiagnosticLogStreamer.__init__sF##"488M#:#:;;!#dhhz&:&:";";<<5 $9   /-.J8N8N~8^8^___8#78 "C! "Hgo N N N   //@@@~001JKK k*  r8batchlist[Dict[str, Any]]rrTcddlm}|D]Q}|ddkrt|}||d}||j|n|j}d}||d|jd|jd |jd j t|d pd  t|j|d t|| }|||xjdz c_SdS)Nr)source_logger_for_exportrCgateway_diagnostic source_loggerzgateway diagnosticts_nsrrrrr) timestamptrace_idspan_id trace_flags severity_textseverity_numberbodyr)r0rMr?rFr@rCrErFrGDEFAULTrupperr;r&r rH) rrrJrMevr3rO otel_loggerrWrecords r%__call__z%GatewayDiagnosticLogStreamer.__call__scLLLLLL  Bvvg"666.r22E54RVVO5L5LMMM!,))-888\  (D__&&//#56 "34 Il3;!"&&"4"4"A BBHHJJ 0BFF:._run2sO//(++ * !& ) ) )//(++ * * * * *r8zhermes-gateway-health-exportTrcrf)rrr>r?rorprq)r9rSrmrRrls`` @r%_start_snapshot_threadrn/s1c088<<=[]^__``aaH******** T0NW[ \ \ \F LLNNN Mr8c8t|}|ddr|ddsdSddlm}|t t }t j}||jvr| ||S)NrTwarning_error_events_enabledr)GatewayDiagnosticLogHandler)rr) r>r?r0rqrrrirjhandlers addHandler)r9rArqhandlerroots r%_attach_log_handlerrv;s  ' 'B 66-t 4 4BFFCacgcron_executionrr|)r[s r%_gateway_health_eventryGs 66'??B BBr8ct|stddSt|}tdd}d}|dds|ddrO t d }n=#t $r0t d d tdd cYSwxYw|ddrj|h t|||_ nQ#t $rDt d d | tddcYSwxYw|ddr| ddl m }| |t|_|jt!dt#|||_nQ#t $rDt dd | tddcYSwxYw t)||_n,#t $rt dd YnwxYw|ddro t-|t/j|_t5||j|_n,#t $rt dd YnwxYw|S)z;Start P0 gateway health export if configured. Never raises.FrL)rKrMTrKNrrrzbmonitoring.gateway_health_export.enabled but OTLP SDK is unavailable; install 'hermes-agent[otlp]'rotlp_unavailablez+gateway health OTLP metrics failed to startmetrics_start_failedr) otlp_exporter) event_filterz*gateway health span streamer did not startz.gateway diagnostic OTLP export failed to startdiagnostics_start_failedz/gateway diagnostic log handler failed to attachz.gateway health snapshot thread failed to start)rrIr>r?rr$rrr-rOr`rr}start_streamingryrNrrerQrrvrPr roEventrSrnrR)r9rArr r}s r%start_gateway_health_exportrKsW F  L)% KKKK  ' 'B(iHHHG$(C vv&& X"&&1Ld*S*S X X&e444CC X X X NN/     .eDVWWW W W W  X vv&&\3? \&J?r)rrrrrr)rrrr')r9r:r;rrr')rCr:rr:)r9r:rr:)r9r:rrJ)rrJrrJrr:)rrrr')rrrr)rr)r9r:rr)r9r:)rr)r9r:rrT)r9r:r r:rr)r r:rrrr)r9r:r r:rr=)r9r:rSrfrrg)r9r:rr)r[r:rrJ)r9r:rrI)5rb __future__rrirrero dataclassesrtypingrrrrjrurrD frozensetr/rEcompiler1r&r7rBrFrIr>rrrrrrrrr@rrrrrrr r-r;r=rernrvryr__all__r[r8r%rs#""""" !!!!!!&&&&&&&&&&  8 $ $8$9 & & &  'Y ( ( (  "rz">??.1)))))),     >>>>>>>>B2222 $$$$ TTTT 26e#L#L#L#L#L#LL     "((( B*++++\ K K K K3333l    @@@@@@@@F        CCCC2222l!! r8