mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-19 10:33:29 +00:00
Record successful response writes, scheduled generation, file saves and SMTP outcomes in a durable content-free tally. Distinct active days include manual exports, while saved-schedule periods deduplicate retries and monthly reuse. Publish bounded scalars through the existing opt-out-controlled heartbeat and report missing coverage explicitly. Preserve pricing and customer entitlements.
1941 lines
105 KiB
Go
1941 lines
105 KiB
Go
// Package telemetry provides outbound usage telemetry for Pulse.
|
|
//
|
|
// Pulse sends a lightweight ping on startup and once every 24 hours to help the
|
|
// developer understand how many active installations exist and which features are
|
|
// in use. Telemetry is enabled by default and can be opted out at any time.
|
|
//
|
|
// # What is sent (the full list — nothing else)
|
|
//
|
|
// Contract and identity:
|
|
// - Payload schema version and the UTC time this payload was built
|
|
// - A rotating install ID (UUID, generated locally and rotated periodically, not tied to any account)
|
|
// - Pulse version identity (normalized version plus raw build string when it differs)
|
|
// - Platform: "docker" or "binary"
|
|
// - Coarse deployment method from a fixed list, never an image name or path
|
|
// - OS and architecture (e.g. "linux/amd64")
|
|
//
|
|
// Lifecycle and audience posture (closed buckets, booleans, and counts only):
|
|
// - Known install age, highest activation stage, time to first monitored resource, and estate-size buckets
|
|
// - Whether authentication is configured, number of configured connections, and whether monitoring is active
|
|
// - Whether a core outcome was observed in the current aggregate windows
|
|
// - Whether the local API, UI document, and referenced frontend assets are
|
|
// being served, plus the immediately previous release observation
|
|
//
|
|
// Scale (counts only, no names):
|
|
// - Number of PVE nodes, PBS instances, PMG instances
|
|
// - Number of VMs, LXC containers
|
|
// - Number of Pulse Agent hosts, Docker hosts/containers, and Kubernetes clusters/nodes/pods/deployments
|
|
// - Number of storage resources, physical disks, Ceph clusters, and network shares
|
|
// - Number of TrueNAS systems/VMs/apps, VMware hosts/VMs/datastores, and availability targets
|
|
// - Number of availability targets assigned to external probes, and of distinct probe agents
|
|
//
|
|
// Feature usage (booleans and counts, no content):
|
|
// - Whether AI features are enabled
|
|
// - Whether Patrol, discovery, notifications, or AI action capability are enabled
|
|
// - Number of active alerts
|
|
// - Whether relay/remote access is enabled
|
|
// - Whether SSO/OIDC is configured
|
|
// - Whether multi-tenant mode is enabled
|
|
// - Whether a paid license is active
|
|
// - Whether any API tokens are configured
|
|
// - Aggregate alert fired/acknowledged/resolved counts over 30 days
|
|
// - Aggregate alert lifecycle counts by severity, active-alert age and
|
|
// resolution-duration buckets, repeat and snooze outcomes over 30 days
|
|
// - Tenant counts for alert delivery, flapping and intent-policy adoption,
|
|
// plus event-history and active-state persistence health
|
|
// - Aggregate notification attempt/delivery/failure counts over seven days
|
|
// - Coarse update funnel counters and last failure category over the current install-ID rotation window
|
|
// - Workload-history preview, scrub, range-change, and Details-selection
|
|
// session counts over the current install-ID rotation window
|
|
// - Patrol, Assistant, and external-agent usage counters over the current install-ID rotation window:
|
|
// configured/active/governed-action/approved-execution/resolved-loop state,
|
|
// Patrol control completed-loop and resolved-loop proof reported through
|
|
// legacy pro_activation metric keys for cohort continuity,
|
|
// operations-loop workflow starter request counts by surface, Assistant/Patrol AI calls,
|
|
// Patrol runs/new findings/investigations/resolved findings/autofixes,
|
|
// the fixed machine cause code when an enabled Patrol is blocked from running,
|
|
// external-agent readiness/usage, action plans, approval requests, rejected
|
|
// action decisions, approved action decisions, approved action attempts,
|
|
// and approved action successes
|
|
//
|
|
// # What is NOT sent
|
|
//
|
|
// - No IP addresses are included in the payload or stored in telemetry rows
|
|
// - No hostnames, node names, VM names, or any infrastructure identifiers
|
|
// - No Proxmox credentials, API tokens, or passwords
|
|
// - No alert content, AI prompts, chat messages, command text, action output, or token values
|
|
// - No action targets, resource IDs, finding IDs, approval actors, or approval reasons
|
|
// - No names, email addresses, account identifiers, or other intentionally identifying personal content
|
|
// - No listener addresses, self-check URLs, response bodies, asset names, or raw startup/self-check errors
|
|
// - No raw browser events, clickstream, guest/user identity, routes, selected
|
|
// ranges, cursor values, interaction timing, or browser identity
|
|
//
|
|
// # How to disable
|
|
//
|
|
// Set the environment variable PULSE_TELEMETRY=false, or toggle off
|
|
// "Outbound usage telemetry" in Settings → System → General.
|
|
//
|
|
// # Mock mode
|
|
//
|
|
// While mock/demo fixture mode is enabled, outbound pings are suppressed
|
|
// entirely: a mock-mode boot (e2e, CI, qual runs, demo containers) would
|
|
// otherwise report the synthetic fixture fleet as a real installation.
|
|
//
|
|
// # Test binaries
|
|
//
|
|
// Pings to the production endpoint are suppressed inside a Go test binary for
|
|
// the same reason: a test that boots the real server is not an installation,
|
|
// and its throwaway data directory mints a new install ID on every run. Tests
|
|
// that need to assert on ping content redirect pingEndpoint to a local server,
|
|
// which the guard deliberately allows.
|
|
package telemetry
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"math/rand"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/updates"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// productionPingEndpoint is the live receiver for outbound usage telemetry.
|
|
const productionPingEndpoint = "https://license.pulserelay.pro/v1/telemetry/ping"
|
|
|
|
// pingEndpoint is the URL that receives outbound usage telemetry pings.
|
|
// It is a var (not const) so that tests can redirect it to a local server.
|
|
var pingEndpoint = productionPingEndpoint
|
|
|
|
var errInstallIDUnavailable = errors.New("telemetry install id unavailable")
|
|
|
|
// errProductionEndpointUnderTest reports a ping suppressed because a test
|
|
// binary tried to reach the live receiver.
|
|
var errProductionEndpointUnderTest = errors.New("telemetry: refusing to post to the production endpoint from a test binary")
|
|
|
|
const (
|
|
// heartbeatInterval is the base interval between daily pings.
|
|
// Each cycle adds random jitter of ±maxHeartbeatJitter to prevent
|
|
// thundering-herd effects when many installations start simultaneously.
|
|
heartbeatInterval = 24 * time.Hour
|
|
|
|
// maxHeartbeatJitter is the maximum random offset added to each heartbeat.
|
|
maxHeartbeatJitter = 30 * time.Minute
|
|
|
|
// startupDelay is how long to wait after startup before sending the first
|
|
// ping, giving the monitor time to connect to nodes and populate state.
|
|
startupDelay = 2 * time.Minute
|
|
|
|
// httpTimeout is the maximum time for a single telemetry request.
|
|
httpTimeout = 10 * time.Second
|
|
|
|
// installIDFile is the filename persisted in the data directory.
|
|
installIDFile = ".install_id"
|
|
|
|
// lifecycleStateFile stores only local milestone timestamps and the highest
|
|
// coarse activation stage observed. It contains no user or infrastructure
|
|
// identifiers and is intentionally independent from the rotating install ID.
|
|
lifecycleStateFile = ".telemetry_lifecycle"
|
|
|
|
// serviceHealthStateFile retains only the current and immediately previous
|
|
// release observation. It lets a post-upgrade ping describe a before/after
|
|
// cohort without exporting listener addresses, URLs, errors, or request data.
|
|
serviceHealthStateFile = ".telemetry_service_health"
|
|
|
|
// installIDRotationWindow limits how long the same pseudonymous identifier
|
|
// can be reused before it is rotated locally.
|
|
installIDRotationWindow = 30 * 24 * time.Hour
|
|
|
|
// PulseIntelligenceTelemetryWindow is the rolling, content-free usage
|
|
// window used for Patrol, Assistant, and external-agent usage counters. It
|
|
// intentionally matches the install-ID rotation window so counters cannot
|
|
// be linked to one stable pseudonymous identifier indefinitely.
|
|
PulseIntelligenceTelemetryWindow = installIDRotationWindow
|
|
|
|
// TelemetrySchemaVersion identifies the exact outbound payload contract.
|
|
// Schema v3 makes notification_failures_7d a terminal-delivery count;
|
|
// schema v4 adds complete approved-action outcome accounting, fixed
|
|
// pre-dispatch refusal categories, and verified finding-resolution linkage.
|
|
// Schema v5 adds bounded, content-free notification failure classes.
|
|
// Schema v6 adds licensed-feature adoption counts (RBAC, audit logging,
|
|
// scheduled reporting, agent profiles, alert-triggered AI) and drops
|
|
// pulse_intelligence_patrol_autofixes_30d, which no code path could ever
|
|
// set to a non-zero value.
|
|
// Schema v7 replaces audit_logging_persistent and audit_events_30d with
|
|
// audit_reads_30d. The SQLite audit logger is installed on every install
|
|
// for defense in depth, so the boolean was true everywhere and the event
|
|
// count measured background write volume; neither discriminated an install
|
|
// that uses audit logging from one that merely has it. Reaching a
|
|
// license-gated audit read does.
|
|
// Schema v8 splits agent-side pre-mutation refusals into target-change,
|
|
// prerequisite, and invalid-contract categories so the legacy "other"
|
|
// bucket no longer hides actionable product failures.
|
|
// Schema v9 separates refusals that carried no machine reason code at all
|
|
// from refusals whose code is simply unrecognised. Agents that predate the
|
|
// typed refusal contract report the former, and folding them into "other"
|
|
// made a starved split indistinguishable from a broken one.
|
|
// Schema v10 adds the Patrol runtime blocked cause so an enabled Patrol
|
|
// that cannot run at all (no provider, unsuitable model, open circuit)
|
|
// is distinguishable in the fleet from one that runs and finds nothing.
|
|
// Field telemetry showed both present identically: high run counts with
|
|
// zero AI calls and zero findings.
|
|
// Schema v11 adds node connection test attempt and failure counts. The
|
|
// fleet shows a population that configures authentication and then never
|
|
// saves a connection, concentrated three to one in container deployments,
|
|
// and nothing recorded whether those installs tried to reach a node and
|
|
// failed or never attempted one at all.
|
|
// Schema v12 adds Patrol-origin action funnel counters so detection and
|
|
// investigation activity can be separated from proposal, decision,
|
|
// execution, and successful completion without exporting action identity.
|
|
// Schema v13 adds a local UI/API service observation and the immediately
|
|
// previous release observation. This separates a process that can emit
|
|
// telemetry from one that is actually serving its API and frontend assets,
|
|
// while retaining only fixed categories and release identity.
|
|
// Schema v14 adds identity-free alert quality outcomes and their tenant
|
|
// denominators. It uses closed severity, age, and resolution-time buckets,
|
|
// and reports only aggregate lifecycle, adoption, and persistence health.
|
|
// Schema v15 separates destination HTTP 5xx server failures from rejected
|
|
// HTTP 4xx responses. Both remain aggregate terminal-failure counters.
|
|
// Schema v16 adds four content-free workload-history adoption counters.
|
|
// Each browser reports each closed milestone at most once per session to a
|
|
// bounded local daily tally; no event stream or browser identity is sent.
|
|
// Schema v17 adds the closed Patrol provider class (local, cloud_byok,
|
|
// cloud_subscription, hosted_legacy, none), the effective Patrol
|
|
// autonomy level, coarse 30-day Patrol input and output token buckets, and
|
|
// per-outcome investigation counts. The 2026-09-01 Patrol assessment could
|
|
// not tell local from cloud installs, estimate what Patrol costs an
|
|
// install, or see how the investigations that produced no plan ended. No
|
|
// provider ID, model name, endpoint, account identity, exact token count,
|
|
// finding ID, or resource ID is exported.
|
|
// Schema v18 adds update_channel (stable or rc), update_check_outcome (a
|
|
// closed category for the last check on that channel) and update_available.
|
|
// A 6.4.3-rc.1 update check failed on every call for weeks while its
|
|
// telemetry looked healthy, because the update counters only see applied
|
|
// updates. No version, URL, or error text is exported.
|
|
// Schema v19 adds only the rolling count of successfully served metric
|
|
// history responses containing data older than seven days.
|
|
// Schema v20 adds aggregate reporting outcomes and distinct-period reuse.
|
|
TelemetrySchemaVersion = 20
|
|
)
|
|
|
|
type installIDRecord struct {
|
|
InstallID string `json:"install_id"`
|
|
IssuedAt time.Time `json:"issued_at"`
|
|
}
|
|
|
|
type lifecycleRecord struct {
|
|
FirstObservedAt time.Time `json:"first_observed_at"`
|
|
FirstMonitoredResourceAt *time.Time `json:"first_monitored_resource_at,omitempty"`
|
|
HighestObservedActivation string `json:"highest_observed_activation"`
|
|
}
|
|
|
|
// ServiceHealthObservation is the bounded result of Pulse's local UI/API
|
|
// self-check. FailureCategory must be one of the fixed ServiceHealthFailure*
|
|
// values. Callers must never put an address, URL, error string, or response
|
|
// content into this shape.
|
|
type ServiceHealthObservation struct {
|
|
Observed bool
|
|
Healthy bool
|
|
FailureCategory string
|
|
}
|
|
|
|
const (
|
|
ServiceHealthFailureListener = "listener"
|
|
ServiceHealthFailureStartup = "startup"
|
|
ServiceHealthFailureRuntime = "runtime"
|
|
ServiceHealthFailureAPIConnectivity = "api_connectivity"
|
|
ServiceHealthFailureTimeout = "timeout"
|
|
ServiceHealthFailureAPIStatus = "api_status"
|
|
ServiceHealthFailureUIStatus = "ui_status"
|
|
ServiceHealthFailureFrontendAssets = "frontend_assets"
|
|
ServiceHealthFailureUnknown = "unknown"
|
|
|
|
ServiceHealthCohortFirstObservation = "first_observation"
|
|
ServiceHealthCohortSameVersion = "same_version"
|
|
ServiceHealthCohortVersionChange = "version_change"
|
|
)
|
|
|
|
// Ping is the payload sent to the telemetry endpoint.
|
|
// Every field is documented here so users can audit exactly what leaves their server.
|
|
type Ping struct {
|
|
// Identity
|
|
SchemaVersion int `json:"schema_version"` // Versioned payload contract
|
|
SentAt string `json:"sent_at"` // UTC send/preview time; no client clock history
|
|
InstallID string `json:"install_id"` // Rotating UUID, not tied to any account
|
|
Version string `json:"version"` // Normalized Pulse version (e.g. "6.0.0-rc.1")
|
|
VersionRaw string `json:"version_raw,omitempty"` // Original version/build string when it differs
|
|
VersionChannel string `json:"version_channel"` // "stable", "rc", "dev", or "prerelease"
|
|
VersionBuild string `json:"version_build,omitempty"` // Build metadata when present (e.g. git describe suffix)
|
|
VersionDevelopment bool `json:"version_is_development"` // True for development/manual builds
|
|
VersionPublished bool `json:"version_is_published_release"` // True for published stable/RC asset versions
|
|
Platform string `json:"platform"` // "docker" or "binary"
|
|
OS string `json:"os"` // runtime.GOOS (e.g. "linux")
|
|
Arch string `json:"arch"` // runtime.GOARCH (e.g. "amd64")
|
|
Event string `json:"event"` // "startup" or "heartbeat"
|
|
DeploymentMethod string `json:"deployment_method"` // Closed coarse install method, never a path or image name
|
|
|
|
// Coarse lifecycle and audience posture. These are closed buckets and
|
|
// aggregate states only; no user, account, locale, or resource identity is
|
|
// included.
|
|
KnownInstallAgeBucket string `json:"known_install_age_bucket"`
|
|
ActivationStage string `json:"activation_stage"`
|
|
TimeToFirstMonitoredResourceBucket string `json:"time_to_first_monitored_resource_bucket"`
|
|
EstateSizeBucket string `json:"estate_size_bucket"`
|
|
AuthConfigured bool `json:"auth_configured"`
|
|
ConfiguredConnections int `json:"configured_connections"`
|
|
MonitoringActive bool `json:"monitoring_active"`
|
|
OutcomeObserved30d bool `json:"outcome_observed_30d"`
|
|
|
|
// Scale (counts only — no names, IPs, or identifiers)
|
|
PVENodes int `json:"pve_nodes"`
|
|
PBSInstances int `json:"pbs_instances"`
|
|
PMGInstances int `json:"pmg_instances"`
|
|
VMs int `json:"vms"`
|
|
Containers int `json:"containers"`
|
|
AgentHosts int `json:"agent_hosts"`
|
|
DockerHosts int `json:"docker_hosts"`
|
|
DockerContainers int `json:"docker_containers"`
|
|
KubernetesClusters int `json:"kubernetes_clusters"`
|
|
KubernetesNodes int `json:"kubernetes_nodes"`
|
|
KubernetesPods int `json:"kubernetes_pods"`
|
|
KubernetesDeployments int `json:"kubernetes_deployments"`
|
|
StoragePools int `json:"storage_pools"`
|
|
PhysicalDisks int `json:"physical_disks"`
|
|
CephClusters int `json:"ceph_clusters"`
|
|
NetworkShares int `json:"network_shares"`
|
|
TrueNASSystems int `json:"truenas_systems"`
|
|
TrueNASVMs int `json:"truenas_vms"`
|
|
TrueNASApps int `json:"truenas_apps"`
|
|
VMwareHosts int `json:"vmware_hosts"`
|
|
VMwareVMs int `json:"vmware_vms"`
|
|
VMwareDatastores int `json:"vmware_datastores"`
|
|
AvailabilityTargets int `json:"availability_targets"`
|
|
// Probe-assignment counts stay counts-only: no agent names, IDs, or
|
|
// addresses leave the install.
|
|
AvailabilityProbeTargets int `json:"availability_probe_targets"`
|
|
AvailabilityProbeAgents int `json:"availability_probe_agents"`
|
|
|
|
// Feature usage (booleans and counts — no content)
|
|
AIEnabled bool `json:"ai_enabled"`
|
|
PatrolEnabled bool `json:"patrol_enabled"`
|
|
DiscoveryEnabled bool `json:"discovery_enabled"`
|
|
NotificationsEnabled bool `json:"notifications_enabled"`
|
|
AIActionsEnabled bool `json:"ai_actions_enabled"`
|
|
AlertAIEnabled bool `json:"alert_ai_enabled"`
|
|
// AIProviderClass is the closed route class of the Patrol (or default)
|
|
// model: none, local, cloud_byok, cloud_subscription, hosted_legacy,
|
|
// or unknown. Never a provider ID, model name, endpoint, or account.
|
|
AIProviderClass string `json:"ai_provider_class"`
|
|
ActiveAlerts int `json:"active_alerts"`
|
|
RelayEnabled bool `json:"relay_enabled"`
|
|
SSOEnabled bool `json:"sso_enabled"`
|
|
MultiTenant bool `json:"multi_tenant"`
|
|
PaidLicense bool `json:"paid_license"`
|
|
HasAPITokens bool `json:"has_api_tokens"`
|
|
|
|
// Licensed-feature adoption. Counts only: no role names, schedule names,
|
|
// profile names, recipients, report contents, or audit event detail.
|
|
RBACCustomRoles int `json:"rbac_custom_roles"`
|
|
RBACUserAssignments int `json:"rbac_user_assignments"`
|
|
AuditReads30d int `json:"audit_reads_30d"`
|
|
ReportSchedules int `json:"report_schedules"`
|
|
ReportSchedulesEnabled int `json:"report_schedules_enabled"`
|
|
ReportSchedulesRun30d int `json:"report_schedules_run_30d"`
|
|
AgentProfiles int `json:"agent_profiles"`
|
|
UpdateAttempts30d int `json:"update_attempts_30d"`
|
|
UpdateSuccesses30d int `json:"update_successes_30d"`
|
|
UpdateFailures30d int `json:"update_failures_30d"`
|
|
// Last coarse update failure category; never raw error text.
|
|
UpdateLastFailureCategory string `json:"update_last_failure_category,omitempty"`
|
|
// Schema v18 update discovery: the channel the install follows and the
|
|
// closed outcome of its last check on that channel. A failed check never
|
|
// reaches the update counters above because nothing was applied.
|
|
UpdateChannel string `json:"update_channel"`
|
|
UpdateCheckOutcome string `json:"update_check_outcome"`
|
|
UpdateAvailable bool `json:"update_available"`
|
|
|
|
// Local release-service observation. The probe checks the local API, UI
|
|
// document, and referenced frontend assets. Only booleans, a fixed failure
|
|
// category, and normalized release versions leave the instance.
|
|
ServiceHealthObserved bool `json:"service_health_observed"`
|
|
ServiceHealthHealthy bool `json:"service_health_healthy"`
|
|
ServiceHealthFailureCategory string `json:"service_health_failure_category,omitempty"`
|
|
ServiceHealthCohort string `json:"service_health_cohort,omitempty"`
|
|
ServiceHealthPreviousVersion string `json:"service_health_previous_version,omitempty"`
|
|
ServiceHealthPreviousObserved bool `json:"service_health_previous_observed"`
|
|
ServiceHealthPreviousHealthy bool `json:"service_health_previous_healthy"`
|
|
|
|
// Node connection test outcomes over the install-ID rotation window.
|
|
// ConfiguredConnections counts only connections that were saved, so an
|
|
// install that tried to reach a node and could not is indistinguishable
|
|
// from one that never opened the add-node dialog: both report zero
|
|
// connections and stall at the "secured" activation stage. These separate
|
|
// the two. Counts only; hosts, credentials, and error text never leave the
|
|
// install. Successes are attempts minus failures.
|
|
NodeTestAttempts30d int `json:"node_test_attempts_30d"`
|
|
NodeTestFailures30d int `json:"node_test_failures_30d"`
|
|
|
|
// Workload history adoption over the install-ID rotation window. The local
|
|
// browser intake is session-deduplicated and contains only the closed
|
|
// milestone; guest/user identity, routes, coordinates, and timings never
|
|
// enter the tally or payload.
|
|
SuccessfulHistoryReadsOver7d30d int `json:"successful_history_reads_over_7d_30d"`
|
|
WorkloadHistoryPreviewSessions30d int `json:"workload_history_preview_sessions_30d"`
|
|
WorkloadHistoryScrubSessions30d int `json:"workload_history_scrub_sessions_30d"`
|
|
WorkloadHistoryRangeChangeSessions30d int `json:"workload_history_range_change_sessions_30d"`
|
|
WorkloadHistoryDetailsSelectionSessions30d int `json:"workload_history_details_selection_sessions_30d"`
|
|
|
|
// Core product outcomes. Alert history is retained locally for 30 days;
|
|
// notification delivery rows are locally retention-bounded to seven days.
|
|
AlertsFired30d int `json:"alerts_fired_30d"`
|
|
AlertsAcknowledged30d int `json:"alerts_acknowledged_30d"`
|
|
AlertsResolved30d int `json:"alerts_resolved_30d"`
|
|
// Alert quality aggregates. Severity totals reconcile to the existing
|
|
// lifecycle totals, while age and resolution-duration buckets have fixed
|
|
// inclusive lower and exclusive upper boundaries.
|
|
ActiveAlertsInfo int `json:"active_alerts_info"`
|
|
ActiveAlertsWarning int `json:"active_alerts_warning"`
|
|
ActiveAlertsCritical int `json:"active_alerts_critical"`
|
|
ActiveAlertsAgeUnder1h int `json:"active_alerts_age_under_1h"`
|
|
ActiveAlertsAge1h24h int `json:"active_alerts_age_1h_24h"`
|
|
ActiveAlertsAge1d7d int `json:"active_alerts_age_1d_7d"`
|
|
ActiveAlertsAge7dPlus int `json:"active_alerts_age_7d_plus"`
|
|
AlertsFiredInfo30d int `json:"alerts_fired_info_30d"`
|
|
AlertsFiredWarning30d int `json:"alerts_fired_warning_30d"`
|
|
AlertsFiredCritical30d int `json:"alerts_fired_critical_30d"`
|
|
AlertsResolvedInfo30d int `json:"alerts_resolved_info_30d"`
|
|
AlertsResolvedWarning30d int `json:"alerts_resolved_warning_30d"`
|
|
AlertsResolvedCritical30d int `json:"alerts_resolved_critical_30d"`
|
|
AlertsResolutionUnder15m30d int `json:"alerts_resolution_under_15m_30d"`
|
|
AlertsResolution15m1h30d int `json:"alerts_resolution_15m_1h_30d"`
|
|
AlertsResolution1h24h30d int `json:"alerts_resolution_1h_24h_30d"`
|
|
AlertsResolution1d7d30d int `json:"alerts_resolution_1d_7d_30d"`
|
|
AlertsResolution7dPlus30d int `json:"alerts_resolution_7d_plus_30d"`
|
|
AlertsRepeatOccurrences30d int `json:"alerts_repeat_occurrences_30d"`
|
|
AlertsSnoozedOccurrences30d int `json:"alerts_snoozed_occurrences_30d"`
|
|
AlertsResolvedWhileSnoozed30d int `json:"alerts_resolved_while_snoozed_30d"`
|
|
AlertManagerTenants int `json:"alert_manager_tenants"`
|
|
AlertDeliveryActiveTenants int `json:"alert_delivery_active_tenants"`
|
|
AlertFlappingEnabledTenants int `json:"alert_flapping_enabled_tenants"`
|
|
AlertIntentPolicyConfiguredTenants int `json:"alert_intent_policy_configured_tenants"`
|
|
AlertEventHistoryAuthoritativeTenants int `json:"alert_event_history_authoritative_tenants"`
|
|
AlertActiveStateAuthoritativeTenants int `json:"alert_active_state_authoritative_tenants"`
|
|
AlertActiveStatePersistenceDegradedTenants int `json:"alert_active_state_persistence_degraded_tenants"`
|
|
NotificationAttempts7d int `json:"notification_attempts_7d"`
|
|
NotificationDeliveries7d int `json:"notification_deliveries_7d"`
|
|
// NotificationFailures7d counts only terminal failed/dead-letter outcomes.
|
|
// Retry-attempt failures remain represented in NotificationAttempts7d.
|
|
NotificationFailures7d int `json:"notification_failures_7d"`
|
|
NotificationFailuresAuthentication7d int `json:"notification_failures_authentication_7d"`
|
|
NotificationFailuresRateLimited7d int `json:"notification_failures_rate_limited_7d"`
|
|
NotificationFailuresConnectivity7d int `json:"notification_failures_connectivity_7d"`
|
|
NotificationFailuresTLS7d int `json:"notification_failures_tls_7d"`
|
|
NotificationFailuresConfiguration7d int `json:"notification_failures_configuration_7d"`
|
|
NotificationFailuresRejected7d int `json:"notification_failures_rejected_7d"`
|
|
NotificationFailuresServerError7d int `json:"notification_failures_server_error_7d"`
|
|
NotificationFailuresUnknown7d int `json:"notification_failures_unknown_7d"`
|
|
|
|
// Pulse Intelligence usage (30-day counts/booleans — no prompts, commands, outputs, resource IDs, or token values)
|
|
PulseIntelligenceLoopConfigured bool `json:"pulse_intelligence_loop_configured"`
|
|
PulseIntelligenceLoopActive30d bool `json:"pulse_intelligence_loop_active_30d"`
|
|
PulseIntelligenceCompleteOperationsLoop30d bool `json:"pulse_intelligence_complete_operations_loop_30d"`
|
|
PulseIntelligenceApprovedExecutionLoop30d bool `json:"pulse_intelligence_approved_execution_loop_30d"`
|
|
PulseIntelligenceResolvedOperationsLoop30d bool `json:"pulse_intelligence_resolved_operations_loop_30d"`
|
|
PulseIntelligencePatrolControlCompletedOperationsLoop30d bool `json:"pulse_intelligence_patrol_control_completed_operations_loop_30d"`
|
|
PulseIntelligencePatrolControlResolvedOperationsLoop30d bool `json:"pulse_intelligence_patrol_control_resolved_operations_loop_30d"`
|
|
PulseIntelligencePatrolControlPaidCompletedOperationsLoop30d bool `json:"pulse_intelligence_patrol_control_paid_completed_operations_loop_30d"`
|
|
PulseIntelligencePatrolControlPaidResolvedOperationsLoop30d bool `json:"pulse_intelligence_patrol_control_paid_resolved_operations_loop_30d"`
|
|
PulseIntelligenceProActivationCompletedOperationsLoop30d bool `json:"pulse_intelligence_pro_activation_completed_operations_loop_30d"`
|
|
PulseIntelligenceProActivationResolvedOperationsLoop30d bool `json:"pulse_intelligence_pro_activation_resolved_operations_loop_30d"`
|
|
PulseIntelligenceProActivationPaidCompletedOperationsLoop30d bool `json:"pulse_intelligence_pro_activation_paid_completed_operations_loop_30d"`
|
|
PulseIntelligenceProActivationPaidResolvedOperationsLoop30d bool `json:"pulse_intelligence_pro_activation_paid_resolved_operations_loop_30d"`
|
|
PulseIntelligenceGovernedActionActive30d bool `json:"pulse_intelligence_governed_action_active_30d"`
|
|
PulseIntelligenceAssistantOperationsLoop30d bool `json:"pulse_intelligence_assistant_operations_loop_30d"`
|
|
PulseIntelligenceAssistantApprovedExecutionLoop30d bool `json:"pulse_intelligence_assistant_approved_execution_loop_30d"`
|
|
PulseIntelligenceAssistantApprovedActionSuccessLoop30d bool `json:"pulse_intelligence_assistant_approved_action_success_loop_30d"`
|
|
PulseIntelligenceAssistantResolvedOperationsLoop30d bool `json:"pulse_intelligence_assistant_resolved_operations_loop_30d"`
|
|
PulseIntelligenceExternalAgentOperationsLoop30d bool `json:"pulse_intelligence_external_agent_operations_loop_30d"`
|
|
PulseIntelligenceExternalAgentApprovedExecutionLoop30d bool `json:"pulse_intelligence_external_agent_approved_execution_loop_30d"`
|
|
PulseIntelligenceExternalAgentApprovedActionSuccessLoop30d bool `json:"pulse_intelligence_external_agent_approved_action_success_loop_30d"`
|
|
PulseIntelligenceExternalAgentResolvedOperationsLoop30d bool `json:"pulse_intelligence_external_agent_resolved_operations_loop_30d"`
|
|
PulseIntelligenceMCPAdapterOperationsLoop30d bool `json:"pulse_intelligence_mcp_adapter_operations_loop_30d"`
|
|
PulseIntelligenceMCPAdapterApprovedExecutionLoop30d bool `json:"pulse_intelligence_mcp_adapter_approved_execution_loop_30d"`
|
|
PulseIntelligenceMCPAdapterApprovedActionSuccessLoop30d bool `json:"pulse_intelligence_mcp_adapter_approved_action_success_loop_30d"`
|
|
PulseIntelligenceMCPAdapterResolvedOperationsLoop30d bool `json:"pulse_intelligence_mcp_adapter_resolved_operations_loop_30d"`
|
|
PulseIntelligenceOperationsLoopStarterRequests30d int `json:"pulse_intelligence_operations_loop_starter_requests_30d"`
|
|
PulseIntelligenceAssistantOperationsLoopStarterRequests30d int `json:"pulse_intelligence_assistant_operations_loop_starter_requests_30d"`
|
|
PulseIntelligencePatrolOperationsLoopStarterRequests30d int `json:"pulse_intelligence_patrol_operations_loop_starter_requests_30d"`
|
|
PulseIntelligencePatrolControlOperationsLoopStarterRequests30d int `json:"pulse_intelligence_patrol_control_operations_loop_starter_requests_30d"`
|
|
PulseIntelligenceProActivationOperationsLoopStarterRequests30d int `json:"pulse_intelligence_pro_activation_operations_loop_starter_requests_30d"`
|
|
PulseIntelligenceMCPOperationsLoopStarterRequests30d int `json:"pulse_intelligence_mcp_operations_loop_starter_requests_30d"`
|
|
PulseIntelligenceAssistantAICalls30d int `json:"pulse_intelligence_assistant_ai_calls_30d"`
|
|
PulseIntelligenceAssistantContextAICalls30d int `json:"pulse_intelligence_assistant_context_ai_calls_30d"`
|
|
PulseIntelligenceAssistantToolCalls30d int `json:"pulse_intelligence_assistant_tool_calls_30d"`
|
|
PulseIntelligencePatrolAICalls30d int `json:"pulse_intelligence_patrol_ai_calls_30d"`
|
|
PulseIntelligencePatrolRuns30d int `json:"pulse_intelligence_patrol_runs_30d"`
|
|
PulseIntelligencePatrolNewFindings30d int `json:"pulse_intelligence_patrol_new_findings_30d"`
|
|
PulseIntelligencePatrolInvestigations30d int `json:"pulse_intelligence_patrol_investigations_30d"`
|
|
PulseIntelligencePatrolResolvedFindings30d int `json:"pulse_intelligence_patrol_resolved_findings_30d"`
|
|
PulseIntelligencePatrolBlockedCause string `json:"pulse_intelligence_patrol_blocked_cause,omitempty"`
|
|
// Effective Patrol autonomy level after licence and Autopilot
|
|
// acknowledgement gating: monitor, approval, assisted, or full.
|
|
PulseIntelligencePatrolAutonomyLevel string `json:"pulse_intelligence_patrol_autonomy_level"`
|
|
// Coarse 30-day Patrol token volume from the local usage ledger. Closed
|
|
// buckets only; exact totals, prices, providers, and models stay local.
|
|
PulseIntelligencePatrolInputTokensBucket30d string `json:"pulse_intelligence_patrol_input_tokens_bucket_30d"`
|
|
PulseIntelligencePatrolOutputTokensBucket30d string `json:"pulse_intelligence_patrol_output_tokens_bucket_30d"`
|
|
// Investigation outcome partition of the findings counted by
|
|
// pulse_intelligence_patrol_investigations_30d. Each investigated finding
|
|
// is in exactly one bucket; the buckets sum to the investigation count.
|
|
PulseIntelligencePatrolInvestigationOutcomeFixVerified30d int `json:"pulse_intelligence_patrol_investigation_outcome_fix_verified_30d"`
|
|
PulseIntelligencePatrolInvestigationOutcomeFixQueued30d int `json:"pulse_intelligence_patrol_investigation_outcome_fix_queued_30d"`
|
|
PulseIntelligencePatrolInvestigationOutcomeFixExecuted30d int `json:"pulse_intelligence_patrol_investigation_outcome_fix_executed_30d"`
|
|
PulseIntelligencePatrolInvestigationOutcomeFixRejected30d int `json:"pulse_intelligence_patrol_investigation_outcome_fix_rejected_30d"`
|
|
PulseIntelligencePatrolInvestigationOutcomeFixFailed30d int `json:"pulse_intelligence_patrol_investigation_outcome_fix_failed_30d"`
|
|
PulseIntelligencePatrolInvestigationOutcomeFixVerificationUnknown30d int `json:"pulse_intelligence_patrol_investigation_outcome_fix_verification_unknown_30d"`
|
|
PulseIntelligencePatrolInvestigationOutcomeResolved30d int `json:"pulse_intelligence_patrol_investigation_outcome_resolved_30d"`
|
|
PulseIntelligencePatrolInvestigationOutcomeNeedsAttention30d int `json:"pulse_intelligence_patrol_investigation_outcome_needs_attention_30d"`
|
|
PulseIntelligencePatrolInvestigationOutcomeCannotFix30d int `json:"pulse_intelligence_patrol_investigation_outcome_cannot_fix_30d"`
|
|
PulseIntelligencePatrolInvestigationOutcomeTimedOut30d int `json:"pulse_intelligence_patrol_investigation_outcome_timed_out_30d"`
|
|
PulseIntelligencePatrolInvestigationOutcomeInProgress30d int `json:"pulse_intelligence_patrol_investigation_outcome_in_progress_30d"`
|
|
PulseIntelligencePatrolInvestigationOutcomeFailed30d int `json:"pulse_intelligence_patrol_investigation_outcome_failed_30d"`
|
|
PulseIntelligencePatrolInvestigationOutcomeOther30d int `json:"pulse_intelligence_patrol_investigation_outcome_other_30d"`
|
|
PulseIntelligenceExternalAgentEnabled bool `json:"pulse_intelligence_external_agent_enabled"`
|
|
PulseIntelligenceExternalAgentUsed30d bool `json:"pulse_intelligence_external_agent_used_30d"`
|
|
PulseIntelligenceMCPAdapterUsed30d bool `json:"pulse_intelligence_mcp_adapter_used_30d"`
|
|
PulseIntelligenceExternalAgentContextRequests30d int `json:"pulse_intelligence_external_agent_context_requests_30d"`
|
|
PulseIntelligenceExternalAgentEventStreamRequests30d int `json:"pulse_intelligence_external_agent_event_stream_requests_30d"`
|
|
PulseIntelligenceExternalAgentProvisioningRequests30d int `json:"pulse_intelligence_external_agent_provisioning_requests_30d"`
|
|
PulseIntelligenceExternalAgentOperatorStateRequests30d int `json:"pulse_intelligence_external_agent_operator_state_requests_30d"`
|
|
PulseIntelligenceExternalAgentFindingRequests30d int `json:"pulse_intelligence_external_agent_finding_requests_30d"`
|
|
PulseIntelligenceExternalAgentActionRequests30d int `json:"pulse_intelligence_external_agent_action_requests_30d"`
|
|
PulseIntelligenceActionPlans30d int `json:"pulse_intelligence_action_plans_30d"`
|
|
PulseIntelligenceApprovalRequests30d int `json:"pulse_intelligence_approval_requests_30d"`
|
|
PulseIntelligenceRejectedActionDecisions30d int `json:"pulse_intelligence_rejected_action_decisions_30d"`
|
|
PulseIntelligenceApprovedActionDecisions30d int `json:"pulse_intelligence_approved_action_decisions_30d"`
|
|
PulseIntelligenceApprovedActionAttempts30d int `json:"pulse_intelligence_approved_action_attempts_30d"`
|
|
PulseIntelligenceApprovedActionSuccesses30d int `json:"pulse_intelligence_approved_action_successes_30d"`
|
|
PulseIntelligencePatrolActionPlans30d int `json:"pulse_intelligence_patrol_action_plans_30d"`
|
|
PulseIntelligencePatrolApprovalRequests30d int `json:"pulse_intelligence_patrol_approval_requests_30d"`
|
|
PulseIntelligencePatrolRejectedActionDecisions30d int `json:"pulse_intelligence_patrol_rejected_action_decisions_30d"`
|
|
PulseIntelligencePatrolApprovedActionDecisions30d int `json:"pulse_intelligence_patrol_approved_action_decisions_30d"`
|
|
PulseIntelligencePatrolApprovedActionAttempts30d int `json:"pulse_intelligence_patrol_approved_action_attempts_30d"`
|
|
PulseIntelligencePatrolApprovedActionSuccesses30d int `json:"pulse_intelligence_patrol_approved_action_successes_30d"`
|
|
|
|
// Cause-coded approved-action failure counters. Together with successes
|
|
// and still-in-flight attempts these partition the attempt count, so the
|
|
// attempt/success gap is attributable without exporting action content.
|
|
PulseIntelligenceApprovedActionFailuresPreDispatch30d int `json:"pulse_intelligence_approved_action_failures_pre_dispatch_30d"`
|
|
PulseIntelligenceApprovedActionFailuresExecution30d int `json:"pulse_intelligence_approved_action_failures_execution_30d"`
|
|
PulseIntelligenceApprovedActionFailuresUnverified30d int `json:"pulse_intelligence_approved_action_failures_unverified_30d"`
|
|
PulseIntelligenceApprovedActionStuckExecuting30d int `json:"pulse_intelligence_approved_action_stuck_executing_30d"`
|
|
PulseIntelligenceApprovedActionInFlight30d int `json:"pulse_intelligence_approved_action_in_flight_30d"`
|
|
PulseIntelligenceApprovedActionUnclassified30d int `json:"pulse_intelligence_approved_action_unclassified_30d"`
|
|
PulseIntelligenceApprovedActionRefusalsPlanStale30d int `json:"pulse_intelligence_approved_action_refusals_plan_stale_30d"`
|
|
PulseIntelligenceApprovedActionRefusalsPolicy30d int `json:"pulse_intelligence_approved_action_refusals_policy_30d"`
|
|
PulseIntelligenceApprovedActionRefusalsCapability30d int `json:"pulse_intelligence_approved_action_refusals_capability_30d"`
|
|
PulseIntelligenceApprovedActionRefusalsTargetChanged30d int `json:"pulse_intelligence_approved_action_refusals_target_changed_30d"`
|
|
PulseIntelligenceApprovedActionRefusalsPrerequisite30d int `json:"pulse_intelligence_approved_action_refusals_prerequisite_30d"`
|
|
PulseIntelligenceApprovedActionRefusalsContract30d int `json:"pulse_intelligence_approved_action_refusals_contract_30d"`
|
|
PulseIntelligenceApprovedActionRefusalsUncoded30d int `json:"pulse_intelligence_approved_action_refusals_uncoded_30d"`
|
|
PulseIntelligenceApprovedActionRefusalsOther30d int `json:"pulse_intelligence_approved_action_refusals_other_30d"`
|
|
PulseIntelligenceVerifiedFindingResolutions30d int `json:"pulse_intelligence_verified_finding_resolutions_30d"`
|
|
PulseIntelligenceApprovedActionLastFailureReason30d string `json:"pulse_intelligence_approved_action_last_failure_reason_30d,omitempty"`
|
|
|
|
// Content-free reporting outcomes and measurement coverage.
|
|
ReportingGeneratedResources30d int `json:"reporting_generated_resources_30d"`
|
|
ReportingGeneratedVMInventory30d int `json:"reporting_generated_vm_inventory_30d"`
|
|
ReportingGeneratedBackupHealth30d int `json:"reporting_generated_backup_health_30d"`
|
|
ReportingGeneratedPatrolDigest30d int `json:"reporting_generated_patrol_digest_30d"`
|
|
ReportingSuccessfulResourcesPeriods30d int `json:"reporting_successful_resources_periods_30d"`
|
|
ReportingSuccessfulVMInventoryPeriods30d int `json:"reporting_successful_vm_inventory_periods_30d"`
|
|
ReportingSuccessfulBackupHealthPeriods30d int `json:"reporting_successful_backup_health_periods_30d"`
|
|
ReportingSuccessfulPatrolDigestPeriods30d int `json:"reporting_successful_patrol_digest_periods_30d"`
|
|
ReportingRepeatResourcesPeriods30d int `json:"reporting_repeat_resources_periods_30d"`
|
|
ReportingRepeatVMInventoryPeriods30d int `json:"reporting_repeat_vm_inventory_periods_30d"`
|
|
ReportingRepeatBackupHealthPeriods30d int `json:"reporting_repeat_backup_health_periods_30d"`
|
|
ReportingRepeatPatrolDigestPeriods30d int `json:"reporting_repeat_patrol_digest_periods_30d"`
|
|
ReportingSaved30d int `json:"reporting_saved_30d"`
|
|
ReportingEmailAccepted30d int `json:"reporting_email_accepted_30d"`
|
|
ReportingDeliveryFailed30d int `json:"reporting_delivery_failed_30d"`
|
|
ReportingDeliveryUnconfirmed30d int `json:"reporting_delivery_unconfirmed_30d"`
|
|
ReportingDataComplete30d int `json:"reporting_data_complete_30d"`
|
|
ReportingDataIncomplete30d int `json:"reporting_data_incomplete_30d"`
|
|
ReportingDataUnavailable30d int `json:"reporting_data_unavailable_30d"`
|
|
ReportingTelemetryAvailable bool `json:"reporting_telemetry_available"`
|
|
ReportingObservationDays int `json:"reporting_observation_days"`
|
|
ReportingResourcesActiveDays30d int `json:"reporting_resources_active_days_30d"`
|
|
ReportingVMInventoryActiveDays30d int `json:"reporting_vm_inventory_active_days_30d"`
|
|
ReportingBackupHealthActiveDays30d int `json:"reporting_backup_health_active_days_30d"`
|
|
ReportingPatrolDigestActiveDays30d int `json:"reporting_patrol_digest_active_days_30d"`
|
|
}
|
|
|
|
// Snapshot holds the dynamic state gathered at ping time.
|
|
// The telemetry package calls a user-provided SnapshotFunc to populate this,
|
|
// keeping the package decoupled from monitor/config internals.
|
|
type Snapshot struct {
|
|
PVENodes int
|
|
PBSInstances int
|
|
PMGInstances int
|
|
VMs int
|
|
Containers int
|
|
AgentHosts int
|
|
DockerHosts int
|
|
DockerContainers int
|
|
KubernetesClusters int
|
|
KubernetesNodes int
|
|
KubernetesPods int
|
|
KubernetesDeployments int
|
|
StoragePools int
|
|
PhysicalDisks int
|
|
CephClusters int
|
|
NetworkShares int
|
|
TrueNASSystems int
|
|
TrueNASVMs int
|
|
TrueNASApps int
|
|
VMwareHosts int
|
|
VMwareVMs int
|
|
VMwareDatastores int
|
|
AvailabilityTargets int
|
|
AvailabilityProbeTargets int
|
|
AvailabilityProbeAgents int
|
|
AIEnabled bool
|
|
PatrolEnabled bool
|
|
DiscoveryEnabled bool
|
|
NotificationsEnabled bool
|
|
AIActionsEnabled bool
|
|
AlertAIEnabled bool
|
|
AIProviderClass string
|
|
ActiveAlerts int
|
|
RelayEnabled bool
|
|
SSOEnabled bool
|
|
MultiTenant bool
|
|
PaidLicense bool
|
|
HasAPITokens bool
|
|
RBACCustomRoles int
|
|
RBACUserAssignments int
|
|
AuditReads30d int
|
|
ReportSchedules int
|
|
ReportSchedulesEnabled int
|
|
ReportSchedulesRun30d int
|
|
AgentProfiles int
|
|
UpdateAttempts30d int
|
|
UpdateSuccesses30d int
|
|
UpdateFailures30d int
|
|
UpdateLastFailureCategory string
|
|
UpdateChannel string
|
|
UpdateCheckOutcome string
|
|
UpdateAvailable bool
|
|
NodeTestAttempts30d int
|
|
NodeTestFailures30d int
|
|
SuccessfulHistoryReadsOver7d30d int
|
|
WorkloadHistoryPreviewSessions30d int
|
|
WorkloadHistoryScrubSessions30d int
|
|
WorkloadHistoryRangeChangeSessions30d int
|
|
WorkloadHistoryDetailsSelectionSessions30d int
|
|
AuthConfigured bool
|
|
ConfiguredConnections int
|
|
AlertsFired30d int
|
|
AlertsAcknowledged30d int
|
|
AlertsResolved30d int
|
|
ActiveAlertsInfo int
|
|
ActiveAlertsWarning int
|
|
ActiveAlertsCritical int
|
|
ActiveAlertsAgeUnder1h int
|
|
ActiveAlertsAge1h24h int
|
|
ActiveAlertsAge1d7d int
|
|
ActiveAlertsAge7dPlus int
|
|
AlertsFiredInfo30d int
|
|
AlertsFiredWarning30d int
|
|
AlertsFiredCritical30d int
|
|
AlertsResolvedInfo30d int
|
|
AlertsResolvedWarning30d int
|
|
AlertsResolvedCritical30d int
|
|
AlertsResolutionUnder15m30d int
|
|
AlertsResolution15m1h30d int
|
|
AlertsResolution1h24h30d int
|
|
AlertsResolution1d7d30d int
|
|
AlertsResolution7dPlus30d int
|
|
AlertsRepeatOccurrences30d int
|
|
AlertsSnoozedOccurrences30d int
|
|
AlertsResolvedWhileSnoozed30d int
|
|
AlertManagerTenants int
|
|
AlertDeliveryActiveTenants int
|
|
AlertFlappingEnabledTenants int
|
|
AlertIntentPolicyConfiguredTenants int
|
|
AlertEventHistoryAuthoritativeTenants int
|
|
AlertActiveStateAuthoritativeTenants int
|
|
AlertActiveStatePersistenceDegradedTenants int
|
|
NotificationAttempts7d int
|
|
NotificationDeliveries7d int
|
|
NotificationFailures7d int
|
|
NotificationFailuresAuthentication7d int
|
|
NotificationFailuresRateLimited7d int
|
|
NotificationFailuresConnectivity7d int
|
|
NotificationFailuresTLS7d int
|
|
NotificationFailuresConfiguration7d int
|
|
NotificationFailuresRejected7d int
|
|
NotificationFailuresServerError7d int
|
|
NotificationFailuresUnknown7d int
|
|
PulseIntelligenceLoopConfigured bool
|
|
PulseIntelligenceLoopActive30d bool
|
|
PulseIntelligenceCompleteOperationsLoop30d bool
|
|
PulseIntelligenceApprovedExecutionLoop30d bool
|
|
PulseIntelligenceResolvedOperationsLoop30d bool
|
|
PulseIntelligencePatrolControlCompletedOperationsLoop30d bool
|
|
PulseIntelligencePatrolControlResolvedOperationsLoop30d bool
|
|
PulseIntelligencePatrolControlPaidCompletedOperationsLoop30d bool
|
|
PulseIntelligencePatrolControlPaidResolvedOperationsLoop30d bool
|
|
PulseIntelligenceProActivationCompletedOperationsLoop30d bool
|
|
PulseIntelligenceProActivationResolvedOperationsLoop30d bool
|
|
PulseIntelligenceProActivationPaidCompletedOperationsLoop30d bool
|
|
PulseIntelligenceProActivationPaidResolvedOperationsLoop30d bool
|
|
PulseIntelligenceGovernedActionActive30d bool
|
|
PulseIntelligenceAssistantOperationsLoop30d bool
|
|
PulseIntelligenceAssistantApprovedExecutionLoop30d bool
|
|
PulseIntelligenceAssistantApprovedActionSuccessLoop30d bool
|
|
PulseIntelligenceAssistantResolvedOperationsLoop30d bool
|
|
PulseIntelligenceExternalAgentOperationsLoop30d bool
|
|
PulseIntelligenceExternalAgentApprovedExecutionLoop30d bool
|
|
PulseIntelligenceExternalAgentApprovedActionSuccessLoop30d bool
|
|
PulseIntelligenceExternalAgentResolvedOperationsLoop30d bool
|
|
PulseIntelligenceMCPAdapterOperationsLoop30d bool
|
|
PulseIntelligenceMCPAdapterApprovedExecutionLoop30d bool
|
|
PulseIntelligenceMCPAdapterApprovedActionSuccessLoop30d bool
|
|
PulseIntelligenceMCPAdapterResolvedOperationsLoop30d bool
|
|
PulseIntelligenceOperationsLoopStarterRequests30d int
|
|
PulseIntelligenceAssistantOperationsLoopStarterRequests30d int
|
|
PulseIntelligencePatrolOperationsLoopStarterRequests30d int
|
|
PulseIntelligencePatrolControlOperationsLoopStarterRequests30d int
|
|
PulseIntelligenceProActivationOperationsLoopStarterRequests30d int
|
|
PulseIntelligenceMCPOperationsLoopStarterRequests30d int
|
|
PulseIntelligenceAssistantAICalls30d int
|
|
PulseIntelligenceAssistantContextAICalls30d int
|
|
PulseIntelligenceAssistantToolCalls30d int
|
|
PulseIntelligencePatrolAICalls30d int
|
|
PulseIntelligencePatrolRuns30d int
|
|
PulseIntelligencePatrolNewFindings30d int
|
|
PulseIntelligencePatrolInvestigations30d int
|
|
PulseIntelligencePatrolResolvedFindings30d int
|
|
PulseIntelligencePatrolBlockedCause string
|
|
PulseIntelligencePatrolAutonomyLevel string
|
|
PulseIntelligencePatrolInputTokensBucket30d string
|
|
PulseIntelligencePatrolOutputTokensBucket30d string
|
|
PulseIntelligencePatrolInvestigationOutcomeFixVerified30d int
|
|
PulseIntelligencePatrolInvestigationOutcomeFixQueued30d int
|
|
PulseIntelligencePatrolInvestigationOutcomeFixExecuted30d int
|
|
PulseIntelligencePatrolInvestigationOutcomeFixRejected30d int
|
|
PulseIntelligencePatrolInvestigationOutcomeFixFailed30d int
|
|
PulseIntelligencePatrolInvestigationOutcomeFixVerificationUnknown30d int
|
|
PulseIntelligencePatrolInvestigationOutcomeResolved30d int
|
|
PulseIntelligencePatrolInvestigationOutcomeNeedsAttention30d int
|
|
PulseIntelligencePatrolInvestigationOutcomeCannotFix30d int
|
|
PulseIntelligencePatrolInvestigationOutcomeTimedOut30d int
|
|
PulseIntelligencePatrolInvestigationOutcomeInProgress30d int
|
|
PulseIntelligencePatrolInvestigationOutcomeFailed30d int
|
|
PulseIntelligencePatrolInvestigationOutcomeOther30d int
|
|
PulseIntelligenceExternalAgentEnabled bool
|
|
PulseIntelligenceExternalAgentOperationsLoopReady bool
|
|
PulseIntelligenceExternalAgentUsed30d bool
|
|
PulseIntelligenceMCPAdapterUsed30d bool
|
|
PulseIntelligenceExternalAgentContextRequests30d int
|
|
PulseIntelligenceExternalAgentEventStreamRequests30d int
|
|
PulseIntelligenceExternalAgentProvisioningRequests30d int
|
|
PulseIntelligenceExternalAgentOperatorStateRequests30d int
|
|
PulseIntelligenceExternalAgentFindingRequests30d int
|
|
PulseIntelligenceExternalAgentActionRequests30d int
|
|
PulseIntelligenceActionPlans30d int
|
|
PulseIntelligenceApprovalRequests30d int
|
|
PulseIntelligenceRejectedActionDecisions30d int
|
|
PulseIntelligenceApprovedActionDecisions30d int
|
|
PulseIntelligenceApprovedActionAttempts30d int
|
|
PulseIntelligenceApprovedActionSuccesses30d int
|
|
PulseIntelligencePatrolActionPlans30d int
|
|
PulseIntelligencePatrolApprovalRequests30d int
|
|
PulseIntelligencePatrolRejectedActionDecisions30d int
|
|
PulseIntelligencePatrolApprovedActionDecisions30d int
|
|
PulseIntelligencePatrolApprovedActionAttempts30d int
|
|
PulseIntelligencePatrolApprovedActionSuccesses30d int
|
|
PulseIntelligenceApprovedActionFailuresPreDispatch30d int
|
|
PulseIntelligenceApprovedActionFailuresExecution30d int
|
|
PulseIntelligenceApprovedActionFailuresUnverified30d int
|
|
PulseIntelligenceApprovedActionStuckExecuting30d int
|
|
PulseIntelligenceApprovedActionInFlight30d int
|
|
PulseIntelligenceApprovedActionUnclassified30d int
|
|
PulseIntelligenceApprovedActionRefusalsPlanStale30d int
|
|
PulseIntelligenceApprovedActionRefusalsPolicy30d int
|
|
PulseIntelligenceApprovedActionRefusalsCapability30d int
|
|
PulseIntelligenceApprovedActionRefusalsTargetChanged30d int
|
|
PulseIntelligenceApprovedActionRefusalsPrerequisite30d int
|
|
PulseIntelligenceApprovedActionRefusalsContract30d int
|
|
PulseIntelligenceApprovedActionRefusalsUncoded30d int
|
|
PulseIntelligenceApprovedActionRefusalsOther30d int
|
|
PulseIntelligenceVerifiedFindingResolutions30d int
|
|
PulseIntelligenceApprovedActionLastFailureReason30d string
|
|
|
|
// Content-free reporting outcomes and measurement coverage.
|
|
ReportingGeneratedResources30d int
|
|
ReportingGeneratedVMInventory30d int
|
|
ReportingGeneratedBackupHealth30d int
|
|
ReportingGeneratedPatrolDigest30d int
|
|
ReportingSuccessfulResourcesPeriods30d int
|
|
ReportingSuccessfulVMInventoryPeriods30d int
|
|
ReportingSuccessfulBackupHealthPeriods30d int
|
|
ReportingSuccessfulPatrolDigestPeriods30d int
|
|
ReportingRepeatResourcesPeriods30d int
|
|
ReportingRepeatVMInventoryPeriods30d int
|
|
ReportingRepeatBackupHealthPeriods30d int
|
|
ReportingRepeatPatrolDigestPeriods30d int
|
|
ReportingSaved30d int
|
|
ReportingEmailAccepted30d int
|
|
ReportingDeliveryFailed30d int
|
|
ReportingDeliveryUnconfirmed30d int
|
|
ReportingDataComplete30d int
|
|
ReportingDataIncomplete30d int
|
|
ReportingDataUnavailable30d int
|
|
ReportingTelemetryAvailable bool
|
|
ReportingObservationDays int
|
|
ReportingResourcesActiveDays30d int
|
|
ReportingVMInventoryActiveDays30d int
|
|
ReportingBackupHealthActiveDays30d int
|
|
ReportingPatrolDigestActiveDays30d int
|
|
}
|
|
|
|
// PulseIntelligenceActionSnapshot is the action-governance portion of the
|
|
// Pulse Intelligence telemetry loop. It is intentionally count-only so callers
|
|
// can aggregate local audit records without exporting action details. The
|
|
// failure-cause fields carry only closed machine reason codes, never command
|
|
// text, resource identifiers, or output.
|
|
type PulseIntelligenceActionSnapshot struct {
|
|
ActionPlans30d int
|
|
ApprovalRequests30d int
|
|
RejectedActionDecisions30d int
|
|
ApprovedActionDecisions30d int
|
|
ApprovedActionAttempts30d int
|
|
ApprovedActionSuccesses30d int
|
|
PatrolActionPlans30d int
|
|
PatrolApprovalRequests30d int
|
|
PatrolRejectedActionDecisions30d int
|
|
PatrolApprovedActionDecisions30d int
|
|
PatrolApprovedActionAttempts30d int
|
|
PatrolApprovedActionSuccesses30d int
|
|
|
|
// ApprovedActionFailuresPreDispatch30d counts approved attempts refused
|
|
// terminally before dispatch (plan drift, expiry, emergency stop, policy
|
|
// authorization).
|
|
ApprovedActionFailuresPreDispatch30d int
|
|
// ApprovedActionFailuresExecution30d counts approved attempts whose
|
|
// dispatched execution failed or ended inconclusive.
|
|
ApprovedActionFailuresExecution30d int
|
|
// ApprovedActionFailuresUnverified30d counts approved attempts that
|
|
// completed execution but whose outcome verification was not confirmed.
|
|
ApprovedActionFailuresUnverified30d int
|
|
// ApprovedActionStuckExecuting30d counts approved attempts still in the
|
|
// executing state well past any legitimate dispatch window.
|
|
ApprovedActionStuckExecuting30d int
|
|
// ApprovedActionInFlight30d counts approved attempts still inside the
|
|
// legitimate dispatch window.
|
|
ApprovedActionInFlight30d int
|
|
// ApprovedActionUnclassified30d counts approved attempts whose
|
|
// authoritative audit is missing or cannot be mapped to a terminal class.
|
|
ApprovedActionUnclassified30d int
|
|
// The refusal category counters partition ApprovedActionFailuresPreDispatch30d
|
|
// into stable, content-free operator diagnostics.
|
|
ApprovedActionRefusalsPlanStale30d int
|
|
ApprovedActionRefusalsPolicy30d int
|
|
ApprovedActionRefusalsCapability30d int
|
|
ApprovedActionRefusalsTargetChanged30d int
|
|
ApprovedActionRefusalsPrerequisite30d int
|
|
ApprovedActionRefusalsContract30d int
|
|
// ApprovedActionRefusalsUncoded30d counts pre-dispatch refusals that
|
|
// arrived without any machine reason code, which is what an agent older
|
|
// than the typed refusal contract reports.
|
|
ApprovedActionRefusalsUncoded30d int
|
|
ApprovedActionRefusalsOther30d int
|
|
// VerifiedFindingResolutions30d counts completed, approved Patrol-origin
|
|
// actions whose postcondition was independently confirmed. No finding or
|
|
// action identity leaves the runtime.
|
|
VerifiedFindingResolutions30d int
|
|
// ApprovedActionLastFailureReason30d is the machine reason code of the
|
|
// most recent approved-action failure, sanitized to a closed code shape.
|
|
ApprovedActionLastFailureReason30d string
|
|
// PatrolBlockedCause is the fixed machine cause code exported only while
|
|
// an enabled Patrol is in the blocked runtime state. Blocked-reason text,
|
|
// provider endpoints, model names, and configuration never leave the
|
|
// runtime.
|
|
PatrolBlockedCause string
|
|
// PatrolAutonomyLevel is the effective Patrol autonomy level after
|
|
// licence and Autopilot acknowledgement gating: monitor, approval,
|
|
// assisted, or full. Empty means the AI service was unavailable and the
|
|
// sender reports monitor.
|
|
PatrolAutonomyLevel string
|
|
}
|
|
|
|
// ApplyUpdateTelemetrySnapshot adds content-free update funnel counters from
|
|
// local update history. It reports only aggregate counts and one coarse failure
|
|
// category over the install-ID rotation window.
|
|
func ApplyUpdateTelemetrySnapshot(s *Snapshot, history *updates.UpdateHistory, now time.Time) {
|
|
if s == nil || history == nil {
|
|
return
|
|
}
|
|
if now.IsZero() {
|
|
now = time.Now()
|
|
}
|
|
since := now.UTC().Add(-installIDRotationWindow)
|
|
entries := history.ListEntries(updates.HistoryFilter{Action: "update"})
|
|
var lastFailure *updates.UpdateHistoryEntry
|
|
for i := range entries {
|
|
entry := entries[i]
|
|
if entry.Timestamp.IsZero() || entry.Timestamp.UTC().Before(since) {
|
|
continue
|
|
}
|
|
s.UpdateAttempts30d++
|
|
switch entry.Status {
|
|
case updates.StatusSuccess:
|
|
s.UpdateSuccesses30d++
|
|
case updates.StatusFailed, updates.StatusRolledBack:
|
|
s.UpdateFailures30d++
|
|
if lastFailure == nil || entry.Timestamp.After(lastFailure.Timestamp) {
|
|
candidate := entry
|
|
lastFailure = &candidate
|
|
}
|
|
}
|
|
}
|
|
if lastFailure != nil {
|
|
s.UpdateLastFailureCategory = classifyUpdateFailureCategory(*lastFailure)
|
|
}
|
|
}
|
|
|
|
// ApplyUpdateCheckTelemetrySnapshot adds the effective update channel and the
|
|
// closed outcome of the last check on it. Only fixed categories leave the
|
|
// instance; versions, URLs, and error text never do.
|
|
func ApplyUpdateCheckTelemetrySnapshot(s *Snapshot, check updates.UpdateCheckObservation) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
s.UpdateChannel = normalizeUpdateChannelForTelemetry(check.Channel)
|
|
s.UpdateCheckOutcome = normalizeUpdateCheckOutcomeForTelemetry(check.Outcome)
|
|
s.UpdateAvailable = check.Available && s.UpdateCheckOutcome == updates.UpdateCheckOutcomeAvailable
|
|
}
|
|
|
|
func normalizeUpdateChannelForTelemetry(channel string) string {
|
|
switch strings.ToLower(strings.TrimSpace(channel)) {
|
|
case "stable":
|
|
return "stable"
|
|
case "rc":
|
|
return "rc"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
func normalizeUpdateCheckOutcomeForTelemetry(outcome string) string {
|
|
switch outcome = strings.ToLower(strings.TrimSpace(outcome)); outcome {
|
|
case updates.UpdateCheckOutcomeNotChecked,
|
|
updates.UpdateCheckOutcomeUpToDate,
|
|
updates.UpdateCheckOutcomeAvailable,
|
|
updates.UpdateCheckOutcomeNoRelease,
|
|
updates.UpdateCheckOutcomeRateLimited,
|
|
updates.UpdateCheckOutcomeNetworkError,
|
|
updates.UpdateCheckOutcomeMetadataError,
|
|
updates.UpdateCheckOutcomeSkipped,
|
|
updates.UpdateCheckOutcomeError:
|
|
return outcome
|
|
case "":
|
|
return updates.UpdateCheckOutcomeNotChecked
|
|
default:
|
|
return updates.UpdateCheckOutcomeError
|
|
}
|
|
}
|
|
|
|
func classifyUpdateFailureCategory(entry updates.UpdateHistoryEntry) string {
|
|
switch entry.Status {
|
|
case updates.StatusRolledBack:
|
|
return "rolled_back"
|
|
case updates.StatusCancelled:
|
|
return "cancelled"
|
|
}
|
|
text := ""
|
|
if entry.Error != nil {
|
|
text = strings.ToLower(strings.TrimSpace(entry.Error.Code + " " + entry.Error.Message + " " + entry.Error.Details))
|
|
}
|
|
switch {
|
|
case strings.Contains(text, "signature"):
|
|
return "signature"
|
|
case strings.Contains(text, "checksum"):
|
|
return "checksum"
|
|
case strings.Contains(text, "download"):
|
|
return "download"
|
|
case strings.Contains(text, "disk space") || strings.Contains(text, "insufficient disk"):
|
|
return "disk_space"
|
|
case strings.Contains(text, "extract") || strings.Contains(text, "archive"):
|
|
return "extract"
|
|
case strings.Contains(text, "backup"):
|
|
return "backup"
|
|
case strings.Contains(text, "apply"):
|
|
return "apply"
|
|
case strings.Contains(text, "restart"):
|
|
return "restart"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
const (
|
|
PulseIntelligenceProActivationValueProofNotStarted = "not_started"
|
|
PulseIntelligenceProActivationValueProofInProgress = "in_progress"
|
|
PulseIntelligenceProActivationValueProofGovernedDecisionRecorded = "governed_decision_recorded"
|
|
// PulseIntelligenceProActivationValueProofVerifiedNeedsExternalMCP remains a
|
|
// tolerated legacy value for API consumers, but MCP readiness is no longer a
|
|
// required Patrol control value gate.
|
|
PulseIntelligenceProActivationValueProofVerifiedNeedsExternalMCP = "verified_needs_mcp"
|
|
PulseIntelligenceProActivationValueProofVerified = "verified"
|
|
)
|
|
|
|
// PulseIntelligencePatrolControlProofInput is the count-only evidence needed
|
|
// to classify whether the first-party Patrol control loop reached governed
|
|
// operations value.
|
|
type PulseIntelligencePatrolControlProofInput struct {
|
|
PatrolControlStarterCount int
|
|
PatrolIssueEvidenceCount int
|
|
ContextualCollaborationCount int
|
|
ApprovedDecisionCount int
|
|
RejectedDecisionCount int
|
|
VerifiedOutcomeCount int
|
|
// ExternalAgentReady is retained for callers that also report optional MCP
|
|
// readiness; it is not part of the Patrol control value classifier.
|
|
ExternalAgentReady bool
|
|
}
|
|
|
|
// PulseIntelligencePatrolControlProof is the shared classification used by
|
|
// the native status endpoint and outbound usage telemetry.
|
|
type PulseIntelligencePatrolControlProof struct {
|
|
Completed bool
|
|
Resolved bool
|
|
ValueProofState string
|
|
}
|
|
|
|
// PulseIntelligencePatrolAutonomyProofInput is the legacy count-only evidence
|
|
// shape for the same Patrol control value proof. Keep this type stable for
|
|
// existing callers and persisted event surfaces.
|
|
type PulseIntelligencePatrolAutonomyProofInput struct {
|
|
PatrolAutonomyStarterCount int
|
|
PatrolIssueEvidenceCount int
|
|
ContextualCollaborationCount int
|
|
ApprovedDecisionCount int
|
|
RejectedDecisionCount int
|
|
VerifiedOutcomeCount int
|
|
// ExternalAgentReady is retained for callers that also report optional MCP
|
|
// readiness; it is not part of the Patrol control value classifier.
|
|
ExternalAgentReady bool
|
|
}
|
|
|
|
// PulseIntelligencePatrolAutonomyProof is the legacy name for
|
|
// PulseIntelligencePatrolControlProof.
|
|
type PulseIntelligencePatrolAutonomyProof = PulseIntelligencePatrolControlProof
|
|
|
|
// PulseIntelligenceProActivationProofInput is the legacy metric/storage shape
|
|
// for the same Patrol control value proof. Keep this type stable for existing
|
|
// telemetry callers and persisted event surfaces.
|
|
type PulseIntelligenceProActivationProofInput struct {
|
|
ProActivationStarterCount int
|
|
PatrolIssueEvidenceCount int
|
|
ContextualCollaborationCount int
|
|
ApprovedDecisionCount int
|
|
RejectedDecisionCount int
|
|
VerifiedOutcomeCount int
|
|
// ExternalAgentReady is retained for callers that also report optional MCP
|
|
// readiness; it is not part of the Patrol control value classifier.
|
|
ExternalAgentReady bool
|
|
}
|
|
|
|
// PulseIntelligenceProActivationProof is the legacy name for
|
|
// PulseIntelligencePatrolControlProof.
|
|
type PulseIntelligenceProActivationProof = PulseIntelligencePatrolControlProof
|
|
|
|
// ClassifyPulseIntelligencePatrolControlProof classifies the Patrol control
|
|
// loop from counters only. It intentionally mirrors the product proof contract:
|
|
// a completed loop can end in an approved verified outcome or a rejected
|
|
// governed decision, while resolved value requires approved verification.
|
|
func ClassifyPulseIntelligencePatrolControlProof(input PulseIntelligencePatrolControlProofInput) PulseIntelligencePatrolControlProof {
|
|
return classifyPulseIntelligenceOperationsValueProof(
|
|
input.PatrolControlStarterCount,
|
|
input.PatrolIssueEvidenceCount,
|
|
input.ContextualCollaborationCount,
|
|
input.ApprovedDecisionCount,
|
|
input.RejectedDecisionCount,
|
|
input.VerifiedOutcomeCount,
|
|
)
|
|
}
|
|
|
|
// ClassifyPulseIntelligencePatrolAutonomyProof classifies the legacy Patrol
|
|
// autonomy proof shape. New callers should use
|
|
// ClassifyPulseIntelligencePatrolControlProof.
|
|
func ClassifyPulseIntelligencePatrolAutonomyProof(input PulseIntelligencePatrolAutonomyProofInput) PulseIntelligencePatrolAutonomyProof {
|
|
proof := ClassifyPulseIntelligencePatrolControlProof(PulseIntelligencePatrolControlProofInput{
|
|
PatrolControlStarterCount: input.PatrolAutonomyStarterCount,
|
|
PatrolIssueEvidenceCount: input.PatrolIssueEvidenceCount,
|
|
ContextualCollaborationCount: input.ContextualCollaborationCount,
|
|
ApprovedDecisionCount: input.ApprovedDecisionCount,
|
|
RejectedDecisionCount: input.RejectedDecisionCount,
|
|
VerifiedOutcomeCount: input.VerifiedOutcomeCount,
|
|
ExternalAgentReady: input.ExternalAgentReady,
|
|
})
|
|
return PulseIntelligencePatrolAutonomyProof(proof)
|
|
}
|
|
|
|
// ClassifyPulseIntelligenceProActivationProof classifies the legacy Pro
|
|
// activation proof shape. New callers should use
|
|
// ClassifyPulseIntelligencePatrolControlProof.
|
|
func ClassifyPulseIntelligenceProActivationProof(input PulseIntelligenceProActivationProofInput) PulseIntelligenceProActivationProof {
|
|
proof := ClassifyPulseIntelligencePatrolControlProof(PulseIntelligencePatrolControlProofInput{
|
|
PatrolControlStarterCount: input.ProActivationStarterCount,
|
|
PatrolIssueEvidenceCount: input.PatrolIssueEvidenceCount,
|
|
ContextualCollaborationCount: input.ContextualCollaborationCount,
|
|
ApprovedDecisionCount: input.ApprovedDecisionCount,
|
|
RejectedDecisionCount: input.RejectedDecisionCount,
|
|
VerifiedOutcomeCount: input.VerifiedOutcomeCount,
|
|
ExternalAgentReady: input.ExternalAgentReady,
|
|
})
|
|
return PulseIntelligenceProActivationProof(proof)
|
|
}
|
|
|
|
func classifyPulseIntelligenceOperationsValueProof(starterCount int, patrolIssueEvidenceCount int, contextualCollaborationCount int, approvedDecisionCount int, rejectedDecisionCount int, verifiedOutcomeCount int) PulseIntelligencePatrolControlProof {
|
|
starterActive := starterCount > 0
|
|
patrolIssueEvidenceActive := patrolIssueEvidenceCount > 0
|
|
contextualCollaborationActive := contextualCollaborationCount > 0
|
|
approvedVerifiedOutcomeActive := approvedDecisionCount > 0 && verifiedOutcomeCount > 0
|
|
rejectedDecisionWithoutApproval := rejectedDecisionCount > 0 && approvedDecisionCount == 0
|
|
|
|
resolved := starterActive &&
|
|
patrolIssueEvidenceActive &&
|
|
contextualCollaborationActive &&
|
|
approvedVerifiedOutcomeActive
|
|
completed := starterActive &&
|
|
patrolIssueEvidenceActive &&
|
|
contextualCollaborationActive &&
|
|
(rejectedDecisionCount > 0 || approvedVerifiedOutcomeActive)
|
|
|
|
valueProofState := PulseIntelligenceProActivationValueProofInProgress
|
|
switch {
|
|
case !starterActive:
|
|
valueProofState = PulseIntelligenceProActivationValueProofNotStarted
|
|
case resolved:
|
|
valueProofState = PulseIntelligenceProActivationValueProofVerified
|
|
case rejectedDecisionWithoutApproval:
|
|
valueProofState = PulseIntelligenceProActivationValueProofGovernedDecisionRecorded
|
|
case completed:
|
|
valueProofState = PulseIntelligenceProActivationValueProofGovernedDecisionRecorded
|
|
}
|
|
|
|
return PulseIntelligencePatrolControlProof{
|
|
Completed: completed,
|
|
Resolved: resolved,
|
|
ValueProofState: valueProofState,
|
|
}
|
|
}
|
|
|
|
// SnapshotFunc returns the current state snapshot for telemetry.
|
|
// It is called on each heartbeat to gather fresh data.
|
|
type SnapshotFunc func() Snapshot
|
|
|
|
// Config holds the static configuration for the telemetry runner.
|
|
type Config struct {
|
|
Version string
|
|
DataDir string
|
|
IsDocker bool
|
|
// DeploymentMethod may be one of docker_compose, docker_run,
|
|
// container_other, systemd, binary_other, or other. Empty/invalid values
|
|
// fall back to container_other or binary_other without exporting raw input.
|
|
DeploymentMethod string
|
|
Enabled bool // From cfg.TelemetryEnabled (system settings or env var)
|
|
GetSnapshot SnapshotFunc
|
|
GetServiceHealth func() ServiceHealthObservation
|
|
}
|
|
|
|
// runner holds the state for the background heartbeat goroutine.
|
|
type runner struct {
|
|
cancel context.CancelFunc
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
var (
|
|
mu sync.Mutex
|
|
current *runner
|
|
)
|
|
|
|
// Start begins outbound usage telemetry if enabled.
|
|
// It reads or creates a rotating install ID in dataDir, waits for the monitor
|
|
// to populate state, sends a startup ping, and schedules a daily heartbeat.
|
|
// Call Stop() on shutdown.
|
|
//
|
|
// This is a no-op when outbound usage telemetry is disabled.
|
|
func Start(ctx context.Context, cfg Config) {
|
|
startWithDelay(ctx, cfg, startupDelay)
|
|
}
|
|
|
|
// Keep the production settling delay constant while allowing deterministic
|
|
// tests to exercise the same background runner without a two-minute sleep.
|
|
func startWithDelay(ctx context.Context, cfg Config, initialDelay time.Duration) {
|
|
if !cfg.Enabled {
|
|
log.Info().Msg("Outbound usage telemetry is disabled (enable via PULSE_TELEMETRY=true or Settings → System)")
|
|
return
|
|
}
|
|
|
|
if getOrCreateInstallID(cfg.DataDir) == "" {
|
|
log.Warn().Msg("Could not determine install ID; telemetry will not run")
|
|
return
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
r := &runner{cancel: cancel}
|
|
|
|
mu.Lock()
|
|
if current != nil {
|
|
current.cancel()
|
|
}
|
|
current = r
|
|
mu.Unlock()
|
|
|
|
log.Info().
|
|
Str("platform", platformName(cfg.IsDocker)).
|
|
Msg("Outbound usage telemetry enabled: sends a rotating pseudonymous install ID, version identity, coarse lifecycle and local service-health buckets, aggregate resource/outcome counts, feature flags, content-free Patrol, Assistant, capability-API, and session-deduplicated workload-history adoption counters")
|
|
|
|
r.wg.Add(1)
|
|
go func() {
|
|
defer r.wg.Done()
|
|
|
|
// Wait for the monitor to connect and populate state before the first ping.
|
|
startTimer := time.NewTimer(initialDelay)
|
|
select {
|
|
case <-ctx.Done():
|
|
startTimer.Stop()
|
|
return
|
|
case <-startTimer.C:
|
|
}
|
|
|
|
// Send startup ping with current snapshot.
|
|
sendEvent(ctx, cfg, "startup")
|
|
|
|
// Daily heartbeat with jitter.
|
|
for {
|
|
timer := time.NewTimer(jitteredHeartbeat())
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
return
|
|
case <-timer.C:
|
|
sendEvent(ctx, cfg, "heartbeat")
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Stop shuts down the telemetry background goroutine.
|
|
func Stop() {
|
|
mu.Lock()
|
|
r := current
|
|
current = nil
|
|
mu.Unlock()
|
|
|
|
if r != nil {
|
|
r.cancel()
|
|
r.wg.Wait()
|
|
}
|
|
}
|
|
|
|
// BuildPreview returns the current heartbeat payload without sending it.
|
|
func BuildPreview(cfg Config) (Ping, error) {
|
|
return buildPingAt(cfg, "heartbeat", time.Now().UTC())
|
|
}
|
|
|
|
// SendServiceHealthEvent sends one immediate bounded service-health ping. It
|
|
// is used for listener/startup/runtime failures that would otherwise exit
|
|
// before the normal delayed startup heartbeat. Disabled telemetry and mock
|
|
// mode remain fully suppressed.
|
|
func SendServiceHealthEvent(ctx context.Context, cfg Config, event string, observation ServiceHealthObservation) error {
|
|
if !cfg.Enabled || mock.IsMockEnabled() {
|
|
return nil
|
|
}
|
|
cfg.GetServiceHealth = func() ServiceHealthObservation { return observation }
|
|
ping, err := buildPingAt(cfg, event, time.Now().UTC())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return send(ctx, ping)
|
|
}
|
|
|
|
// ResetInstallID rotates the locally stored telemetry install ID immediately
|
|
// and returns the new pseudonymous identifier.
|
|
func ResetInstallID(dataDir string) (string, error) {
|
|
return resetInstallIDAt(dataDir, time.Now().UTC())
|
|
}
|
|
|
|
// IsEnabled reports whether telemetry is enabled.
|
|
// Telemetry is on by default; set PULSE_TELEMETRY=false to disable.
|
|
func IsEnabled() bool {
|
|
v := os.Getenv("PULSE_TELEMETRY")
|
|
if v == "" {
|
|
return true // enabled by default
|
|
}
|
|
return v == "true" || v == "1"
|
|
}
|
|
|
|
// jitteredHeartbeat returns heartbeatInterval ± a random offset up to maxHeartbeatJitter.
|
|
func jitteredHeartbeat() time.Duration {
|
|
jitter := time.Duration(rand.Int63n(int64(2*maxHeartbeatJitter)+1)) - maxHeartbeatJitter
|
|
return heartbeatInterval + jitter
|
|
}
|
|
|
|
func basePing(cfg Config, installID string) Ping {
|
|
versionIdentity := updates.DescribeUsageDataVersion(cfg.Version)
|
|
return Ping{
|
|
SchemaVersion: TelemetrySchemaVersion,
|
|
InstallID: installID,
|
|
Version: versionIdentity.Version,
|
|
VersionRaw: versionIdentity.RawVersion,
|
|
VersionChannel: versionIdentity.Channel,
|
|
VersionBuild: versionIdentity.Build,
|
|
VersionDevelopment: versionIdentity.IsDevelopment,
|
|
VersionPublished: versionIdentity.IsPublishedRelease,
|
|
Platform: platformName(cfg.IsDocker),
|
|
OS: runtime.GOOS,
|
|
Arch: runtime.GOARCH,
|
|
DeploymentMethod: deploymentMethod(cfg),
|
|
}
|
|
}
|
|
|
|
func deploymentMethod(cfg Config) string {
|
|
raw := strings.ToLower(strings.TrimSpace(cfg.DeploymentMethod))
|
|
if raw == "" {
|
|
raw = strings.ToLower(strings.TrimSpace(os.Getenv("PULSE_DEPLOYMENT_METHOD")))
|
|
}
|
|
switch raw {
|
|
case "docker_compose", "docker_run", "container_other", "systemd", "binary_other", "other":
|
|
return raw
|
|
}
|
|
if cfg.IsDocker {
|
|
return "container_other"
|
|
}
|
|
return "binary_other"
|
|
}
|
|
|
|
func platformName(isDocker bool) string {
|
|
if isDocker {
|
|
return "docker"
|
|
}
|
|
return "binary"
|
|
}
|
|
|
|
// BuildPingForSnapshot projects a Snapshot onto an otherwise-zero Ping using
|
|
// the same merge the outbound path uses.
|
|
//
|
|
// It exists so the licensed-feature adoption guard can assert against the real
|
|
// serialized payload instead of a hand-built stand-in. Asserting on a
|
|
// hand-built ping is what lets a field look correct in a test while shipping
|
|
// something else, which is how audit_logging_persistent reached production
|
|
// reading true on every install.
|
|
func BuildPingForSnapshot(s Snapshot) Ping {
|
|
return applySnapshot(Ping{}, func() Snapshot { return s })
|
|
}
|
|
|
|
// applySnapshot merges dynamic state into the base ping.
|
|
// applyReportingActivitySnapshot maps the reporting count family without
|
|
// adding identifiers or changing the heartbeat's existing opt-out boundary.
|
|
func applyReportingActivitySnapshot(ping *Ping, snapshot *Snapshot) {
|
|
ping.ReportingGeneratedResources30d = snapshot.ReportingGeneratedResources30d
|
|
ping.ReportingGeneratedVMInventory30d = snapshot.ReportingGeneratedVMInventory30d
|
|
ping.ReportingGeneratedBackupHealth30d = snapshot.ReportingGeneratedBackupHealth30d
|
|
ping.ReportingGeneratedPatrolDigest30d = snapshot.ReportingGeneratedPatrolDigest30d
|
|
ping.ReportingSuccessfulResourcesPeriods30d = snapshot.ReportingSuccessfulResourcesPeriods30d
|
|
ping.ReportingSuccessfulVMInventoryPeriods30d = snapshot.ReportingSuccessfulVMInventoryPeriods30d
|
|
ping.ReportingSuccessfulBackupHealthPeriods30d = snapshot.ReportingSuccessfulBackupHealthPeriods30d
|
|
ping.ReportingSuccessfulPatrolDigestPeriods30d = snapshot.ReportingSuccessfulPatrolDigestPeriods30d
|
|
ping.ReportingRepeatResourcesPeriods30d = snapshot.ReportingRepeatResourcesPeriods30d
|
|
ping.ReportingRepeatVMInventoryPeriods30d = snapshot.ReportingRepeatVMInventoryPeriods30d
|
|
ping.ReportingRepeatBackupHealthPeriods30d = snapshot.ReportingRepeatBackupHealthPeriods30d
|
|
ping.ReportingRepeatPatrolDigestPeriods30d = snapshot.ReportingRepeatPatrolDigestPeriods30d
|
|
ping.ReportingSaved30d = snapshot.ReportingSaved30d
|
|
ping.ReportingEmailAccepted30d = snapshot.ReportingEmailAccepted30d
|
|
ping.ReportingDeliveryFailed30d = snapshot.ReportingDeliveryFailed30d
|
|
ping.ReportingDeliveryUnconfirmed30d = snapshot.ReportingDeliveryUnconfirmed30d
|
|
ping.ReportingDataComplete30d = snapshot.ReportingDataComplete30d
|
|
ping.ReportingDataIncomplete30d = snapshot.ReportingDataIncomplete30d
|
|
ping.ReportingDataUnavailable30d = snapshot.ReportingDataUnavailable30d
|
|
ping.ReportingTelemetryAvailable = snapshot.ReportingTelemetryAvailable
|
|
ping.ReportingObservationDays = snapshot.ReportingObservationDays
|
|
ping.ReportingResourcesActiveDays30d = snapshot.ReportingResourcesActiveDays30d
|
|
ping.ReportingVMInventoryActiveDays30d = snapshot.ReportingVMInventoryActiveDays30d
|
|
ping.ReportingBackupHealthActiveDays30d = snapshot.ReportingBackupHealthActiveDays30d
|
|
ping.ReportingPatrolDigestActiveDays30d = snapshot.ReportingPatrolDigestActiveDays30d
|
|
}
|
|
|
|
func applySnapshot(base Ping, fn SnapshotFunc) Ping {
|
|
ping := base
|
|
if fn == nil {
|
|
return ping
|
|
}
|
|
s := fn()
|
|
ping.PVENodes = s.PVENodes
|
|
ping.PBSInstances = s.PBSInstances
|
|
ping.PMGInstances = s.PMGInstances
|
|
ping.VMs = s.VMs
|
|
ping.Containers = s.Containers
|
|
ping.AgentHosts = s.AgentHosts
|
|
ping.DockerHosts = s.DockerHosts
|
|
ping.DockerContainers = s.DockerContainers
|
|
ping.KubernetesClusters = s.KubernetesClusters
|
|
ping.KubernetesNodes = s.KubernetesNodes
|
|
ping.KubernetesPods = s.KubernetesPods
|
|
ping.KubernetesDeployments = s.KubernetesDeployments
|
|
ping.StoragePools = s.StoragePools
|
|
ping.PhysicalDisks = s.PhysicalDisks
|
|
ping.CephClusters = s.CephClusters
|
|
ping.NetworkShares = s.NetworkShares
|
|
ping.TrueNASSystems = s.TrueNASSystems
|
|
ping.TrueNASVMs = s.TrueNASVMs
|
|
ping.TrueNASApps = s.TrueNASApps
|
|
ping.VMwareHosts = s.VMwareHosts
|
|
ping.VMwareVMs = s.VMwareVMs
|
|
ping.VMwareDatastores = s.VMwareDatastores
|
|
ping.AvailabilityTargets = s.AvailabilityTargets
|
|
ping.AvailabilityProbeTargets = s.AvailabilityProbeTargets
|
|
ping.AvailabilityProbeAgents = s.AvailabilityProbeAgents
|
|
ping.AIEnabled = s.AIEnabled
|
|
ping.PatrolEnabled = s.PatrolEnabled
|
|
ping.DiscoveryEnabled = s.DiscoveryEnabled
|
|
ping.NotificationsEnabled = s.NotificationsEnabled
|
|
ping.AIActionsEnabled = s.AIActionsEnabled
|
|
ping.AlertAIEnabled = s.AlertAIEnabled
|
|
ping.AIProviderClass = s.AIProviderClass
|
|
ping.ActiveAlerts = s.ActiveAlerts
|
|
ping.RelayEnabled = s.RelayEnabled
|
|
ping.SSOEnabled = s.SSOEnabled
|
|
ping.MultiTenant = s.MultiTenant
|
|
ping.PaidLicense = s.PaidLicense
|
|
ping.HasAPITokens = s.HasAPITokens
|
|
ping.RBACCustomRoles = s.RBACCustomRoles
|
|
ping.RBACUserAssignments = s.RBACUserAssignments
|
|
ping.AuditReads30d = s.AuditReads30d
|
|
ping.ReportSchedules = s.ReportSchedules
|
|
ping.ReportSchedulesEnabled = s.ReportSchedulesEnabled
|
|
ping.ReportSchedulesRun30d = s.ReportSchedulesRun30d
|
|
applyReportingActivitySnapshot(&ping, &s)
|
|
|
|
ping.AgentProfiles = s.AgentProfiles
|
|
ping.UpdateAttempts30d = s.UpdateAttempts30d
|
|
ping.UpdateSuccesses30d = s.UpdateSuccesses30d
|
|
ping.UpdateFailures30d = s.UpdateFailures30d
|
|
ping.UpdateLastFailureCategory = s.UpdateLastFailureCategory
|
|
ping.UpdateChannel = normalizeUpdateChannelForTelemetry(s.UpdateChannel)
|
|
ping.UpdateCheckOutcome = normalizeUpdateCheckOutcomeForTelemetry(s.UpdateCheckOutcome)
|
|
ping.UpdateAvailable = s.UpdateAvailable && ping.UpdateCheckOutcome == updates.UpdateCheckOutcomeAvailable
|
|
ping.NodeTestAttempts30d = s.NodeTestAttempts30d
|
|
ping.NodeTestFailures30d = s.NodeTestFailures30d
|
|
ping.SuccessfulHistoryReadsOver7d30d = s.SuccessfulHistoryReadsOver7d30d
|
|
ping.WorkloadHistoryPreviewSessions30d = s.WorkloadHistoryPreviewSessions30d
|
|
ping.WorkloadHistoryScrubSessions30d = s.WorkloadHistoryScrubSessions30d
|
|
ping.WorkloadHistoryRangeChangeSessions30d = s.WorkloadHistoryRangeChangeSessions30d
|
|
ping.WorkloadHistoryDetailsSelectionSessions30d = s.WorkloadHistoryDetailsSelectionSessions30d
|
|
ping.AuthConfigured = s.AuthConfigured
|
|
ping.ConfiguredConnections = s.ConfiguredConnections
|
|
ping.AlertsFired30d = s.AlertsFired30d
|
|
ping.AlertsAcknowledged30d = s.AlertsAcknowledged30d
|
|
ping.AlertsResolved30d = s.AlertsResolved30d
|
|
ping.ActiveAlertsInfo = s.ActiveAlertsInfo
|
|
ping.ActiveAlertsWarning = s.ActiveAlertsWarning
|
|
ping.ActiveAlertsCritical = s.ActiveAlertsCritical
|
|
ping.ActiveAlertsAgeUnder1h = s.ActiveAlertsAgeUnder1h
|
|
ping.ActiveAlertsAge1h24h = s.ActiveAlertsAge1h24h
|
|
ping.ActiveAlertsAge1d7d = s.ActiveAlertsAge1d7d
|
|
ping.ActiveAlertsAge7dPlus = s.ActiveAlertsAge7dPlus
|
|
ping.AlertsFiredInfo30d = s.AlertsFiredInfo30d
|
|
ping.AlertsFiredWarning30d = s.AlertsFiredWarning30d
|
|
ping.AlertsFiredCritical30d = s.AlertsFiredCritical30d
|
|
ping.AlertsResolvedInfo30d = s.AlertsResolvedInfo30d
|
|
ping.AlertsResolvedWarning30d = s.AlertsResolvedWarning30d
|
|
ping.AlertsResolvedCritical30d = s.AlertsResolvedCritical30d
|
|
ping.AlertsResolutionUnder15m30d = s.AlertsResolutionUnder15m30d
|
|
ping.AlertsResolution15m1h30d = s.AlertsResolution15m1h30d
|
|
ping.AlertsResolution1h24h30d = s.AlertsResolution1h24h30d
|
|
ping.AlertsResolution1d7d30d = s.AlertsResolution1d7d30d
|
|
ping.AlertsResolution7dPlus30d = s.AlertsResolution7dPlus30d
|
|
ping.AlertsRepeatOccurrences30d = s.AlertsRepeatOccurrences30d
|
|
ping.AlertsSnoozedOccurrences30d = s.AlertsSnoozedOccurrences30d
|
|
ping.AlertsResolvedWhileSnoozed30d = s.AlertsResolvedWhileSnoozed30d
|
|
ping.AlertManagerTenants = s.AlertManagerTenants
|
|
ping.AlertDeliveryActiveTenants = s.AlertDeliveryActiveTenants
|
|
ping.AlertFlappingEnabledTenants = s.AlertFlappingEnabledTenants
|
|
ping.AlertIntentPolicyConfiguredTenants = s.AlertIntentPolicyConfiguredTenants
|
|
ping.AlertEventHistoryAuthoritativeTenants = s.AlertEventHistoryAuthoritativeTenants
|
|
ping.AlertActiveStateAuthoritativeTenants = s.AlertActiveStateAuthoritativeTenants
|
|
ping.AlertActiveStatePersistenceDegradedTenants = s.AlertActiveStatePersistenceDegradedTenants
|
|
ping.NotificationAttempts7d = s.NotificationAttempts7d
|
|
ping.NotificationDeliveries7d = s.NotificationDeliveries7d // gitleaks:allow -- schema field name, not a credential
|
|
ping.NotificationFailures7d = s.NotificationFailures7d
|
|
ping.NotificationFailuresAuthentication7d = s.NotificationFailuresAuthentication7d
|
|
ping.NotificationFailuresRateLimited7d = s.NotificationFailuresRateLimited7d
|
|
ping.NotificationFailuresConnectivity7d = s.NotificationFailuresConnectivity7d
|
|
ping.NotificationFailuresTLS7d = s.NotificationFailuresTLS7d
|
|
ping.NotificationFailuresConfiguration7d = s.NotificationFailuresConfiguration7d
|
|
ping.NotificationFailuresRejected7d = s.NotificationFailuresRejected7d
|
|
ping.NotificationFailuresServerError7d = s.NotificationFailuresServerError7d
|
|
ping.NotificationFailuresUnknown7d = s.NotificationFailuresUnknown7d
|
|
ping.PulseIntelligenceLoopConfigured = s.PulseIntelligenceLoopConfigured
|
|
ping.PulseIntelligenceLoopActive30d = s.PulseIntelligenceLoopActive30d
|
|
ping.PulseIntelligenceCompleteOperationsLoop30d = s.PulseIntelligenceCompleteOperationsLoop30d
|
|
ping.PulseIntelligenceApprovedExecutionLoop30d = s.PulseIntelligenceApprovedExecutionLoop30d
|
|
ping.PulseIntelligenceResolvedOperationsLoop30d = s.PulseIntelligenceResolvedOperationsLoop30d
|
|
ping.PulseIntelligencePatrolControlCompletedOperationsLoop30d = s.PulseIntelligencePatrolControlCompletedOperationsLoop30d
|
|
ping.PulseIntelligencePatrolControlResolvedOperationsLoop30d = s.PulseIntelligencePatrolControlResolvedOperationsLoop30d
|
|
ping.PulseIntelligencePatrolControlPaidCompletedOperationsLoop30d = s.PulseIntelligencePatrolControlPaidCompletedOperationsLoop30d
|
|
ping.PulseIntelligencePatrolControlPaidResolvedOperationsLoop30d = s.PulseIntelligencePatrolControlPaidResolvedOperationsLoop30d
|
|
ping.PulseIntelligenceProActivationCompletedOperationsLoop30d = s.PulseIntelligenceProActivationCompletedOperationsLoop30d
|
|
ping.PulseIntelligenceProActivationResolvedOperationsLoop30d = s.PulseIntelligenceProActivationResolvedOperationsLoop30d
|
|
ping.PulseIntelligenceProActivationPaidCompletedOperationsLoop30d = s.PulseIntelligenceProActivationPaidCompletedOperationsLoop30d
|
|
ping.PulseIntelligenceProActivationPaidResolvedOperationsLoop30d = s.PulseIntelligenceProActivationPaidResolvedOperationsLoop30d
|
|
ping.PulseIntelligenceGovernedActionActive30d = s.PulseIntelligenceGovernedActionActive30d
|
|
ping.PulseIntelligenceAssistantOperationsLoop30d = s.PulseIntelligenceAssistantOperationsLoop30d
|
|
ping.PulseIntelligenceAssistantApprovedExecutionLoop30d = s.PulseIntelligenceAssistantApprovedExecutionLoop30d
|
|
ping.PulseIntelligenceAssistantApprovedActionSuccessLoop30d = s.PulseIntelligenceAssistantApprovedActionSuccessLoop30d
|
|
ping.PulseIntelligenceAssistantResolvedOperationsLoop30d = s.PulseIntelligenceAssistantResolvedOperationsLoop30d
|
|
ping.PulseIntelligenceExternalAgentOperationsLoop30d = s.PulseIntelligenceExternalAgentOperationsLoop30d
|
|
ping.PulseIntelligenceExternalAgentApprovedExecutionLoop30d = s.PulseIntelligenceExternalAgentApprovedExecutionLoop30d
|
|
ping.PulseIntelligenceExternalAgentApprovedActionSuccessLoop30d = s.PulseIntelligenceExternalAgentApprovedActionSuccessLoop30d
|
|
ping.PulseIntelligenceExternalAgentResolvedOperationsLoop30d = s.PulseIntelligenceExternalAgentResolvedOperationsLoop30d
|
|
ping.PulseIntelligenceMCPAdapterOperationsLoop30d = s.PulseIntelligenceMCPAdapterOperationsLoop30d
|
|
ping.PulseIntelligenceMCPAdapterApprovedExecutionLoop30d = s.PulseIntelligenceMCPAdapterApprovedExecutionLoop30d
|
|
ping.PulseIntelligenceMCPAdapterApprovedActionSuccessLoop30d = s.PulseIntelligenceMCPAdapterApprovedActionSuccessLoop30d
|
|
ping.PulseIntelligenceMCPAdapterResolvedOperationsLoop30d = s.PulseIntelligenceMCPAdapterResolvedOperationsLoop30d
|
|
ping.PulseIntelligenceOperationsLoopStarterRequests30d = s.PulseIntelligenceOperationsLoopStarterRequests30d
|
|
ping.PulseIntelligenceAssistantOperationsLoopStarterRequests30d = s.PulseIntelligenceAssistantOperationsLoopStarterRequests30d
|
|
ping.PulseIntelligencePatrolOperationsLoopStarterRequests30d = s.PulseIntelligencePatrolOperationsLoopStarterRequests30d
|
|
ping.PulseIntelligencePatrolControlOperationsLoopStarterRequests30d = s.PulseIntelligencePatrolControlOperationsLoopStarterRequests30d
|
|
ping.PulseIntelligenceProActivationOperationsLoopStarterRequests30d = s.PulseIntelligenceProActivationOperationsLoopStarterRequests30d
|
|
ping.PulseIntelligenceMCPOperationsLoopStarterRequests30d = s.PulseIntelligenceMCPOperationsLoopStarterRequests30d
|
|
ping.PulseIntelligenceAssistantAICalls30d = s.PulseIntelligenceAssistantAICalls30d
|
|
ping.PulseIntelligenceAssistantContextAICalls30d = s.PulseIntelligenceAssistantContextAICalls30d
|
|
ping.PulseIntelligenceAssistantToolCalls30d = s.PulseIntelligenceAssistantToolCalls30d
|
|
ping.PulseIntelligencePatrolAICalls30d = s.PulseIntelligencePatrolAICalls30d
|
|
ping.PulseIntelligencePatrolRuns30d = s.PulseIntelligencePatrolRuns30d
|
|
ping.PulseIntelligencePatrolNewFindings30d = s.PulseIntelligencePatrolNewFindings30d
|
|
ping.PulseIntelligencePatrolInvestigations30d = s.PulseIntelligencePatrolInvestigations30d
|
|
ping.PulseIntelligencePatrolResolvedFindings30d = s.PulseIntelligencePatrolResolvedFindings30d
|
|
ping.PulseIntelligencePatrolBlockedCause = s.PulseIntelligencePatrolBlockedCause
|
|
ping.PulseIntelligencePatrolAutonomyLevel = s.PulseIntelligencePatrolAutonomyLevel
|
|
ping.PulseIntelligencePatrolInputTokensBucket30d = s.PulseIntelligencePatrolInputTokensBucket30d
|
|
ping.PulseIntelligencePatrolOutputTokensBucket30d = s.PulseIntelligencePatrolOutputTokensBucket30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeFixVerified30d = s.PulseIntelligencePatrolInvestigationOutcomeFixVerified30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeFixQueued30d = s.PulseIntelligencePatrolInvestigationOutcomeFixQueued30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeFixExecuted30d = s.PulseIntelligencePatrolInvestigationOutcomeFixExecuted30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeFixRejected30d = s.PulseIntelligencePatrolInvestigationOutcomeFixRejected30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeFixFailed30d = s.PulseIntelligencePatrolInvestigationOutcomeFixFailed30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeFixVerificationUnknown30d = s.PulseIntelligencePatrolInvestigationOutcomeFixVerificationUnknown30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeResolved30d = s.PulseIntelligencePatrolInvestigationOutcomeResolved30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeNeedsAttention30d = s.PulseIntelligencePatrolInvestigationOutcomeNeedsAttention30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeCannotFix30d = s.PulseIntelligencePatrolInvestigationOutcomeCannotFix30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeTimedOut30d = s.PulseIntelligencePatrolInvestigationOutcomeTimedOut30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeInProgress30d = s.PulseIntelligencePatrolInvestigationOutcomeInProgress30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeFailed30d = s.PulseIntelligencePatrolInvestigationOutcomeFailed30d
|
|
ping.PulseIntelligencePatrolInvestigationOutcomeOther30d = s.PulseIntelligencePatrolInvestigationOutcomeOther30d
|
|
ping.PulseIntelligenceExternalAgentEnabled = s.PulseIntelligenceExternalAgentEnabled
|
|
ping.PulseIntelligenceExternalAgentUsed30d = s.PulseIntelligenceExternalAgentUsed30d
|
|
ping.PulseIntelligenceMCPAdapterUsed30d = s.PulseIntelligenceMCPAdapterUsed30d
|
|
ping.PulseIntelligenceExternalAgentContextRequests30d = s.PulseIntelligenceExternalAgentContextRequests30d
|
|
ping.PulseIntelligenceExternalAgentEventStreamRequests30d = s.PulseIntelligenceExternalAgentEventStreamRequests30d
|
|
ping.PulseIntelligenceExternalAgentProvisioningRequests30d = s.PulseIntelligenceExternalAgentProvisioningRequests30d
|
|
ping.PulseIntelligenceExternalAgentOperatorStateRequests30d = s.PulseIntelligenceExternalAgentOperatorStateRequests30d
|
|
ping.PulseIntelligenceExternalAgentFindingRequests30d = s.PulseIntelligenceExternalAgentFindingRequests30d
|
|
ping.PulseIntelligenceExternalAgentActionRequests30d = s.PulseIntelligenceExternalAgentActionRequests30d
|
|
ping.PulseIntelligenceActionPlans30d = s.PulseIntelligenceActionPlans30d
|
|
ping.PulseIntelligenceApprovalRequests30d = s.PulseIntelligenceApprovalRequests30d
|
|
ping.PulseIntelligenceRejectedActionDecisions30d = s.PulseIntelligenceRejectedActionDecisions30d
|
|
ping.PulseIntelligenceApprovedActionDecisions30d = s.PulseIntelligenceApprovedActionDecisions30d
|
|
ping.PulseIntelligenceApprovedActionAttempts30d = s.PulseIntelligenceApprovedActionAttempts30d
|
|
ping.PulseIntelligenceApprovedActionSuccesses30d = s.PulseIntelligenceApprovedActionSuccesses30d
|
|
ping.PulseIntelligencePatrolActionPlans30d = s.PulseIntelligencePatrolActionPlans30d
|
|
ping.PulseIntelligencePatrolApprovalRequests30d = s.PulseIntelligencePatrolApprovalRequests30d
|
|
ping.PulseIntelligencePatrolRejectedActionDecisions30d = s.PulseIntelligencePatrolRejectedActionDecisions30d
|
|
ping.PulseIntelligencePatrolApprovedActionDecisions30d = s.PulseIntelligencePatrolApprovedActionDecisions30d
|
|
ping.PulseIntelligencePatrolApprovedActionAttempts30d = s.PulseIntelligencePatrolApprovedActionAttempts30d
|
|
ping.PulseIntelligencePatrolApprovedActionSuccesses30d = s.PulseIntelligencePatrolApprovedActionSuccesses30d
|
|
ping.PulseIntelligenceApprovedActionFailuresPreDispatch30d = s.PulseIntelligenceApprovedActionFailuresPreDispatch30d
|
|
ping.PulseIntelligenceApprovedActionFailuresExecution30d = s.PulseIntelligenceApprovedActionFailuresExecution30d
|
|
ping.PulseIntelligenceApprovedActionFailuresUnverified30d = s.PulseIntelligenceApprovedActionFailuresUnverified30d
|
|
ping.PulseIntelligenceApprovedActionStuckExecuting30d = s.PulseIntelligenceApprovedActionStuckExecuting30d
|
|
ping.PulseIntelligenceApprovedActionInFlight30d = s.PulseIntelligenceApprovedActionInFlight30d
|
|
ping.PulseIntelligenceApprovedActionUnclassified30d = s.PulseIntelligenceApprovedActionUnclassified30d
|
|
ping.PulseIntelligenceApprovedActionRefusalsPlanStale30d = s.PulseIntelligenceApprovedActionRefusalsPlanStale30d
|
|
ping.PulseIntelligenceApprovedActionRefusalsPolicy30d = s.PulseIntelligenceApprovedActionRefusalsPolicy30d
|
|
ping.PulseIntelligenceApprovedActionRefusalsCapability30d = s.PulseIntelligenceApprovedActionRefusalsCapability30d
|
|
ping.PulseIntelligenceApprovedActionRefusalsTargetChanged30d = s.PulseIntelligenceApprovedActionRefusalsTargetChanged30d
|
|
ping.PulseIntelligenceApprovedActionRefusalsPrerequisite30d = s.PulseIntelligenceApprovedActionRefusalsPrerequisite30d
|
|
ping.PulseIntelligenceApprovedActionRefusalsContract30d = s.PulseIntelligenceApprovedActionRefusalsContract30d
|
|
ping.PulseIntelligenceApprovedActionRefusalsUncoded30d = s.PulseIntelligenceApprovedActionRefusalsUncoded30d
|
|
ping.PulseIntelligenceApprovedActionRefusalsOther30d = s.PulseIntelligenceApprovedActionRefusalsOther30d
|
|
ping.PulseIntelligenceVerifiedFindingResolutions30d = s.PulseIntelligenceVerifiedFindingResolutions30d
|
|
ping.PulseIntelligenceApprovedActionLastFailureReason30d = s.PulseIntelligenceApprovedActionLastFailureReason30d
|
|
return ping
|
|
}
|
|
|
|
// getOrCreateInstallID reads or generates a rotating install ID in dataDir.
|
|
func getOrCreateInstallID(dataDir string) string {
|
|
return getOrCreateInstallIDAt(dataDir, time.Now().UTC())
|
|
}
|
|
|
|
func getOrCreateInstallIDAt(dataDir string, now time.Time) string {
|
|
p := filepath.Join(dataDir, installIDFile)
|
|
now = now.UTC()
|
|
|
|
data, err := os.ReadFile(p)
|
|
if err == nil {
|
|
record, ok := parseInstallIDRecord(data)
|
|
if ok && shouldKeepInstallIDRecord(record, now) {
|
|
return record.InstallID
|
|
}
|
|
}
|
|
|
|
record := installIDRecord{
|
|
InstallID: uuid.New().String(),
|
|
IssuedAt: now,
|
|
}
|
|
if err := writeInstallIDRecordAt(dataDir, record); err != nil {
|
|
log.Warn().Err(err).Str("path", p).Msg("Failed to persist install ID")
|
|
// Still use the generated ID for this session.
|
|
}
|
|
return record.InstallID
|
|
}
|
|
|
|
func resetInstallIDAt(dataDir string, now time.Time) (string, error) {
|
|
record := installIDRecord{
|
|
InstallID: uuid.New().String(),
|
|
IssuedAt: now.UTC(),
|
|
}
|
|
if err := writeInstallIDRecordAt(dataDir, record); err != nil {
|
|
return "", err
|
|
}
|
|
return record.InstallID, nil
|
|
}
|
|
|
|
func writeInstallIDRecordAt(dataDir string, record installIDRecord) error {
|
|
if err := os.MkdirAll(dataDir, 0700); err != nil {
|
|
return err
|
|
}
|
|
encoded, err := json.Marshal(record)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(filepath.Join(dataDir, installIDFile), append(encoded, '\n'), 0600)
|
|
}
|
|
|
|
func parseInstallIDRecord(data []byte) (installIDRecord, bool) {
|
|
trimmed := bytes.TrimSpace(data)
|
|
if len(trimmed) == 0 {
|
|
return installIDRecord{}, false
|
|
}
|
|
|
|
var record installIDRecord
|
|
if err := json.Unmarshal(trimmed, &record); err == nil {
|
|
record.InstallID = string(bytes.TrimSpace([]byte(record.InstallID)))
|
|
if _, err := uuid.Parse(record.InstallID); err == nil && !record.IssuedAt.IsZero() {
|
|
return record, true
|
|
}
|
|
return installIDRecord{}, false
|
|
}
|
|
|
|
legacyID := string(trimmed)
|
|
if _, err := uuid.Parse(legacyID); err == nil {
|
|
// Legacy plaintext IDs are accepted as migration input only. Rotate to a
|
|
// new record immediately instead of preserving an unbounded stable ID.
|
|
return installIDRecord{}, false
|
|
}
|
|
return installIDRecord{}, false
|
|
}
|
|
|
|
func shouldKeepInstallIDRecord(record installIDRecord, now time.Time) bool {
|
|
if _, err := uuid.Parse(record.InstallID); err != nil {
|
|
return false
|
|
}
|
|
issuedAt := record.IssuedAt.UTC()
|
|
if issuedAt.IsZero() || issuedAt.After(now) {
|
|
return false
|
|
}
|
|
return now.Sub(issuedAt) < installIDRotationWindow
|
|
}
|
|
|
|
var lifecycleMu sync.Mutex
|
|
|
|
func applyLifecycle(ping *Ping, dataDir string, now time.Time) {
|
|
if ping == nil {
|
|
return
|
|
}
|
|
now = now.UTC()
|
|
lifecycleMu.Lock()
|
|
defer lifecycleMu.Unlock()
|
|
|
|
record := readLifecycleRecord(dataDir)
|
|
if record.FirstObservedAt.IsZero() || record.FirstObservedAt.After(now) {
|
|
record.FirstObservedAt = now
|
|
}
|
|
|
|
currentStage := activationStage(*ping)
|
|
if activationStageRank(currentStage) > activationStageRank(record.HighestObservedActivation) {
|
|
record.HighestObservedActivation = currentStage
|
|
}
|
|
if record.HighestObservedActivation == "" {
|
|
record.HighestObservedActivation = "started"
|
|
}
|
|
|
|
ping.MonitoringActive = monitoredResourceCount(*ping) > 0
|
|
ping.OutcomeObserved30d = ping.ActiveAlerts > 0 ||
|
|
ping.AlertsFired30d > 0 ||
|
|
ping.AlertsAcknowledged30d > 0 ||
|
|
ping.AlertsResolved30d > 0 ||
|
|
ping.NotificationDeliveries7d > 0
|
|
if ping.MonitoringActive && record.FirstMonitoredResourceAt == nil {
|
|
observedAt := now
|
|
record.FirstMonitoredResourceAt = &observedAt
|
|
}
|
|
|
|
ping.KnownInstallAgeBucket = durationBucket(now.Sub(record.FirstObservedAt), []durationBoundary{
|
|
{24 * time.Hour, "under_1d"},
|
|
{7 * 24 * time.Hour, "1_7d"},
|
|
{30 * 24 * time.Hour, "8_30d"},
|
|
{90 * 24 * time.Hour, "31_90d"},
|
|
{365 * 24 * time.Hour, "91_365d"},
|
|
}, "over_365d")
|
|
ping.ActivationStage = record.HighestObservedActivation
|
|
ping.EstateSizeBucket = estateSizeBucket(monitoredResourceCount(*ping))
|
|
ping.TimeToFirstMonitoredResourceBucket = "not_observed"
|
|
if record.FirstMonitoredResourceAt != nil {
|
|
if record.FirstMonitoredResourceAt.Equal(record.FirstObservedAt) {
|
|
ping.TimeToFirstMonitoredResourceBucket = "present_at_first_observation"
|
|
} else {
|
|
elapsed := record.FirstMonitoredResourceAt.Sub(record.FirstObservedAt)
|
|
if elapsed < 0 {
|
|
elapsed = 0
|
|
}
|
|
ping.TimeToFirstMonitoredResourceBucket = durationBucket(elapsed, []durationBoundary{
|
|
{15 * time.Minute, "under_15m"},
|
|
{time.Hour, "15m_1h"},
|
|
{6 * time.Hour, "1_6h"},
|
|
{24 * time.Hour, "6_24h"},
|
|
{3 * 24 * time.Hour, "1_3d"},
|
|
{7 * 24 * time.Hour, "4_7d"},
|
|
{30 * 24 * time.Hour, "8_30d"},
|
|
}, "over_30d")
|
|
}
|
|
}
|
|
|
|
if err := writeLifecycleRecord(dataDir, record); err != nil {
|
|
log.Debug().Err(err).Msg("Could not persist coarse telemetry lifecycle milestones")
|
|
}
|
|
}
|
|
|
|
type durationBoundary struct {
|
|
upper time.Duration
|
|
label string
|
|
}
|
|
|
|
func durationBucket(value time.Duration, boundaries []durationBoundary, overflow string) string {
|
|
for _, boundary := range boundaries {
|
|
if value < boundary.upper {
|
|
return boundary.label
|
|
}
|
|
}
|
|
return overflow
|
|
}
|
|
|
|
func activationStage(ping Ping) string {
|
|
switch {
|
|
case ping.ActiveAlerts > 0 || ping.AlertsFired30d > 0 || ping.AlertsResolved30d > 0 || ping.NotificationDeliveries7d > 0:
|
|
return "outcome_observed"
|
|
case monitoredResourceCount(ping) > 0:
|
|
return "monitoring"
|
|
case ping.ConfiguredConnections > 0:
|
|
return "connected"
|
|
case ping.AuthConfigured:
|
|
return "secured"
|
|
default:
|
|
return "started"
|
|
}
|
|
}
|
|
|
|
func activationStageRank(stage string) int {
|
|
switch stage {
|
|
case "secured":
|
|
return 2
|
|
case "connected":
|
|
return 3
|
|
case "monitoring":
|
|
return 4
|
|
case "outcome_observed":
|
|
return 5
|
|
default:
|
|
return 1
|
|
}
|
|
}
|
|
|
|
func validActivationStage(stage string) bool {
|
|
switch stage {
|
|
case "started", "secured", "connected", "monitoring", "outcome_observed":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func monitoredResourceCount(ping Ping) int {
|
|
return ping.PVENodes + ping.PBSInstances + ping.PMGInstances + ping.VMs +
|
|
ping.Containers + ping.AgentHosts + ping.DockerHosts + ping.DockerContainers +
|
|
ping.KubernetesClusters + ping.KubernetesNodes + ping.KubernetesPods +
|
|
ping.KubernetesDeployments + ping.StoragePools + ping.PhysicalDisks +
|
|
ping.CephClusters + ping.NetworkShares + ping.TrueNASSystems + ping.TrueNASVMs +
|
|
ping.TrueNASApps + ping.VMwareHosts + ping.VMwareVMs + ping.VMwareDatastores +
|
|
ping.AvailabilityTargets
|
|
}
|
|
|
|
func estateSizeBucket(resources int) string {
|
|
switch {
|
|
case resources <= 0:
|
|
return "empty"
|
|
case resources <= 10:
|
|
return "1_10"
|
|
case resources <= 50:
|
|
return "11_50"
|
|
case resources <= 200:
|
|
return "51_200"
|
|
case resources <= 1000:
|
|
return "201_1000"
|
|
default:
|
|
return "over_1000"
|
|
}
|
|
}
|
|
|
|
func readLifecycleRecord(dataDir string) lifecycleRecord {
|
|
data, err := os.ReadFile(filepath.Join(dataDir, lifecycleStateFile))
|
|
if err != nil {
|
|
return lifecycleRecord{}
|
|
}
|
|
var record lifecycleRecord
|
|
if err := json.Unmarshal(data, &record); err != nil {
|
|
return lifecycleRecord{}
|
|
}
|
|
if !validActivationStage(record.HighestObservedActivation) {
|
|
record.HighestObservedActivation = ""
|
|
}
|
|
return record
|
|
}
|
|
|
|
func writeLifecycleRecord(dataDir string, record lifecycleRecord) error {
|
|
if err := os.MkdirAll(dataDir, 0700); err != nil {
|
|
return err
|
|
}
|
|
encoded, err := json.Marshal(record)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tmp, err := os.CreateTemp(dataDir, ".telemetry_lifecycle-*")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tmpPath := tmp.Name()
|
|
defer os.Remove(tmpPath)
|
|
if err := tmp.Chmod(0600); err != nil {
|
|
_ = tmp.Close()
|
|
return err
|
|
}
|
|
if _, err := tmp.Write(append(encoded, '\n')); err != nil {
|
|
_ = tmp.Close()
|
|
return err
|
|
}
|
|
if err := tmp.Close(); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(tmpPath, filepath.Join(dataDir, lifecycleStateFile))
|
|
}
|
|
|
|
// sendEvent builds and sends one ping for the given event unless mock mode is
|
|
// active. A mock-mode snapshot describes the synthetic fixture fleet, not a
|
|
// real installation, so it must never reach the telemetry endpoint. The check
|
|
// runs per event (not once at Start) because mock mode can be toggled at
|
|
// runtime.
|
|
func sendEvent(ctx context.Context, cfg Config, event string) {
|
|
if mock.IsMockEnabled() {
|
|
log.Debug().Str("event", event).Msg("Suppressing outbound telemetry ping while mock mode is enabled")
|
|
return
|
|
}
|
|
ping, err := buildPingAt(cfg, event, time.Now().UTC())
|
|
if err != nil {
|
|
log.Debug().Err(err).Str("event", event).Msg("Telemetry ping could not be built")
|
|
return
|
|
}
|
|
if err := send(ctx, ping); err != nil {
|
|
log.Debug().Err(err).Msg("Telemetry ping failed (will retry at next heartbeat)")
|
|
}
|
|
}
|
|
|
|
func buildPingAt(cfg Config, event string, now time.Time) (Ping, error) {
|
|
now = now.UTC()
|
|
installID := getOrCreateInstallIDAt(cfg.DataDir, now)
|
|
if installID == "" {
|
|
return Ping{}, errInstallIDUnavailable
|
|
}
|
|
ping := applySnapshot(basePing(cfg, installID), cfg.GetSnapshot)
|
|
ping.Event = event
|
|
ping.SentAt = now.Format(time.RFC3339)
|
|
applyLifecycle(&ping, cfg.DataDir, now)
|
|
applyServiceHealth(&ping, cfg.DataDir, cfg.GetServiceHealth)
|
|
return ping, nil
|
|
}
|
|
|
|
// send posts a ping to the telemetry endpoint. Errors are observable in debug
|
|
// logs but never affect normal Pulse operation.
|
|
func send(ctx context.Context, ping Ping) error {
|
|
// A test binary is not a real installation. Any test that boots the real
|
|
// server (pkg/server.Run and anything like it) runs against a throwaway
|
|
// data directory, so it mints a fresh install ID per run and would be
|
|
// counted as a distinct live install. Telemetry's own tests redirect
|
|
// pingEndpoint at a local server and are unaffected by this guard.
|
|
if testing.Testing() && pingEndpoint == productionPingEndpoint {
|
|
return errProductionEndpointUnderTest
|
|
}
|
|
|
|
body, err := json.Marshal(ping)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
reqCtx, cancel := context.WithTimeout(ctx, httpTimeout)
|
|
defer cancel()
|
|
|
|
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, pingEndpoint, bytes.NewReader(body))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
_, _ = io.CopyN(io.Discard, resp.Body, 4096)
|
|
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
|
return fmt.Errorf("telemetry endpoint returned HTTP %d", resp.StatusCode)
|
|
}
|
|
return nil
|
|
}
|