/* SPDX-License-Identifier: GPL-2.0 */
/* scx_cake BPF/userspace interface - shared data structures and constants */

#ifndef __CAKE_INTF_H
#define __CAKE_INTF_H

#include <limits.h>

/* Type defs for BPF/userspace compat - defined when vmlinux.h is not included */
#ifndef __VMLINUX_H__
typedef unsigned char  u8;
typedef unsigned short u16;
typedef unsigned int   u32;
typedef unsigned long  u64;

typedef signed char    s8;
typedef signed short   s16;
typedef signed int     s32;
typedef signed long    s64;
#endif

/* CAKE TIER SYSTEM — legacy 4-tier latency buckets
 *
 * These enum values are retained for telemetry and older UI paths.
 * The current scheduler does not classify hot-path work into these tiers. */
enum cake_tier {
	CAKE_TIER_CRITICAL = 0, /* Lowest-latency legacy bucket */
	CAKE_TIER_INTERACT = 1, /* Interactive legacy bucket */
	CAKE_TIER_FRAME	   = 2, /* Medium-latency legacy bucket */
	CAKE_TIER_BULK	   = 3, /* Highest-latency legacy bucket */
	CAKE_TIER_MAX	   = 4,
};

/* ═══ TASK CLASS — legacy/debug classification labels ═══
 * The hot path no longer promotes game families. These enums remain for
 * telemetry, iter output, and compatibility with older debug tooling. */
enum cake_class {
	CAKE_CLASS_NORMAL = 0, /* Default */
	CAKE_CLASS_GAME	  = 1, /* Reserved legacy class */
	CAKE_CLASS_HOG	  = 2, /* Reserved legacy class */
	CAKE_CLASS_BG	  = 3, /* Reserved legacy class */
	CAKE_CLASS_MAX	  = 4,
};

enum cake_startup_phase {
	CAKE_STARTUP_PHASE_NONE	   = 0,
	CAKE_STARTUP_PHASE_ENQUEUE = 1,
	CAKE_STARTUP_PHASE_SELECT  = 2,
	CAKE_STARTUP_PHASE_RUNNING = 3,
};

#define CAKE_STARTUP_MASK_ENQUEUE (1u << 0)
#define CAKE_STARTUP_MASK_SELECT (1u << 1)
#define CAKE_STARTUP_MASK_RUNNING (1u << 2)

/* ═══ Additive Fairness Model ═══
 * Hot path reads p->scx.weight directly from task_struct (L1-hot).
 * vtime_mult_cache[] BSS array DELETED. -4KB BSS, -64 cache lines.
 * VTIME_MULT_CACHE_SIZE kept for backward compat only (no consumers). */
#define VTIME_MULT_CACHE_SIZE 2048

/* brain_class_cache REMOVED: 131KB BSS array (16384×8B) was hydrated by Rust
 * every poll cycle but had zero BPF readers. */

/* ═══ Per-Task Context ═══
 * Scheduling state now lives in task_struct plus the arena-backed
 * cake_task_ctx below. Release hot paths avoid arena access where possible;
 * the arena copy exists primarily for iter output and debug telemetry. */

/* ═══ PER-CPU BSS ═══
 * Written by local CPU callbacks, read by cake_stopping and a small number of
 * same-CPU enqueue/dispatch helpers. Release remote wake placement reads the
 * owner-published cpu_status/cpu_frontier lanes instead of these private hints.
 *
 * Page aligned for cache/TLB isolation across CPUs.
 * Single-writer (local running), multi-reader (no atomics needed). */
struct cake_cpu_bss {
#ifndef CAKE_RELEASE
	u32 run_start; /* 4B: (u32)cake_clock when task started.
				 *     Written: cake_running (debug only).
				 *     Read: running_telemetry (debug only). */
	u8  tick_count; /* 1B: cake_tick throttle counter */
	u8  llc_id; /* 1B: per-CPU LLC ID cache.
				 *     Set once at init from cpu_llc_id RODATA.
				 *     Eliminates indexed RODATA load in dispatch/tick. */
	u16 _pad_6; /* 2B: natural alignment for tick_slice */
	#endif
	u64 tick_slice; /* 8B: p->scx.slice ?: quantum_ns */
	u32 last_pid; /* 4B: Fast path — skip task-change work if same pid */
	u32 frame_victim_pid; /* 4B: diag-only frame-preemptor attribution latch;
				 *     written only in CAKE_GAME_DIAG builds,
				 *     unconditional here so intf.h layout never
				 *     depends on the diag define. */
#ifndef CAKE_RELEASE
	u8 idle_hint; /* 1B: debug-only private mirror of idle state */
#endif
#ifndef CAKE_RELEASE
	u8 cpu_pressure; /* 1B: debug/non-lean raw pressure; release publishes
				 *     pressure bucket in cpu_status. */
#endif
#ifndef CAKE_RELEASE
	u8 busy_wakeup_pending; /* 1B: reserved for future wake handoff policy */
	u8 last_wake_class; /* 1B: last running task's shadow wake class */
#endif
	u32 owner_avg_runtime_ns; /* 4B: release policy EWMA for current CPU owner */
	u16 owner_run_count; /* 2B: samples behind owner_avg_runtime_ns */
	u8 owner_service_kind; /* 1B: service token for current CPU owner.
				*     Written on running() task change, then read
				*     by stopping() so owner SAT/cache residency can
				*     stay service-gated without parsing comm every
				*     stop callback. */
	u8 service_reset_kind; /* 1B: last explicit service that scrubbed stale
				*     per-CPU/global residue for this target CPU.
				*     This is intentionally separate from
				*     owner_service_kind: transitions need a one-shot
				*     cleanup before the new service has run. */
	u64 throughput_decision; /* Packed cache/mem throughput owner + dispatch state */
	u64 decision_confidence; /* Packed confidence for accelerator paths */
	u64 route_prediction_last;		       /* CPU-owned keep-run outcome predictor */
	u64 run_start_ns;			       /* Wall-clock start for preemption grace period */
#ifndef CAKE_RELEASE
	u8 last_strict_wake_class; /* Strict dry-run class for busy-preempt experiments */
	u8  _pad_strict[7];
	u64 smt_run_start_ns; /* Wall-clock start for SMT overlap accounting */
	u64 smt_last_stop_ns; /* Last observed stop for sibling overlap accounting */
	u8 smt_sibling_active_start; /* 1 if sibling lane was active at run start */
	u8  _pad_smt[7];
	u64 cake_clock; /* 8B: BPF-native monotonic clock (ns).
				 *     Debug-only accumulator. Advanced by consumed
				 *     slice in cake_stopping. Resynced from
				 *     scx_bpf_now() only on task-change (25%). */
#endif
	/* Compiler pads to 4096B via aligned attribute.
	 * Hardware page boundary isolation ensures L2 prefetchers
	 * cannot speculatively load adjacent CPU states. */
} __attribute__((aligned(4096)));
/* 4096B alignment: V-Cache Telescoping (Frontier Phase 1).
 * Hardware page boundary isolation ensures L2 hardware prefetchers cannot
 * speculatively load adjacent CPU states, absolutely eradicating MESI
 * snoop invalidation storms across dual-CCD topologies. */

/* Owner-written, remotely-readable CPU publication lanes.
 *
 * cpu_bss is private scheduling bookkeeping: tick baselines, last_pid, owner
 * runtime, and local vtime integration. Wake placement only needs a compact
 * hint surface, so release paths publish status/frontier separately and keep
 * cross-CPU reads away from the private BSS line.
 */
#define CAKE_CPU_STATUS_IDLE (1ULL << 0)
#define CAKE_CPU_STATUS_ACCEPT_WAKE (1ULL << 1)
#define CAKE_CPU_STATUS_SAT_CACHE_MEM (1ULL << 2)
#define CAKE_CPU_STATUS_PREEMPT_WAKE (1ULL << 3)

#define CAKE_CPU_STATUS_OWNER_SHIFT 8U
#define CAKE_CPU_STATUS_OWNER_MASK 0x7ULL
#define CAKE_CPU_STATUS_PRESS_SHIFT 12U
#define CAKE_CPU_STATUS_PRESS_MASK 0x3ULL
#define CAKE_CPU_STATUS_EPOCH_SHIFT 16U
#define CAKE_CPU_STATUS_EPOCH_MASK 0xfULL
#define CAKE_CPU_STATUS_LATENCY_SHIFT 20U
#define CAKE_CPU_STATUS_LATENCY_MASK 0x3ULL

#define CAKE_CPU_OWNER_UNKNOWN 0U
#define CAKE_CPU_OWNER_SHORT 1U
#define CAKE_CPU_OWNER_INTERACTIVE 2U
#define CAKE_CPU_OWNER_FRAME 3U
#define CAKE_CPU_OWNER_BULK 4U

#define CAKE_CPU_PRESSURE_LOW 0U
#define CAKE_CPU_PRESSURE_MED 1U
#define CAKE_CPU_PRESSURE_HIGH 2U
#define CAKE_CPU_PRESSURE_FULL 3U

#define CAKE_CPU_LATENCY_UNKNOWN 0U
#define CAKE_CPU_LATENCY_LATENCY 1U
#define CAKE_CPU_LATENCY_FRAME 2U
#define CAKE_CPU_LATENCY_BULK 3U

struct cake_cpu_status {
	u64 flags; /* CAKE_CPU_STATUS_* plus owner/pressure buckets */
	u64 frontier_vtime; /* Published vtime frontier for wake clamp/seeding */
	u64 local_waiter_debt; /* Remote wake admitted a local waiter for owner quench */
	u64 core_steal_pending; /* Conservative per-primary-CPU steal DSQ has-work hint */
} __attribute__((aligned(64)));
_Static_assert(sizeof(struct cake_cpu_status) == 64,
	       "cake_cpu_status must stay one cache line");

#if CAKE_FUTEX_TRACE
#define CAKE_FUTEX_TASK_TRACE_SLOTS 32U
struct cake_futex_trace {
	u64 select_enter;
	u64 select_futex;
	u64 idle_found;
	u64 idle_scoreboard;
	u64 idle_core_spread;
	u64 idle_native;
	u64 native_noidle;
	u64 direct_clean_enter;
	u64 direct_clean_futex;
	u64 direct_clean_lane_active;
	u64 direct_clean_first;
	u64 tunnel_enter;
	u64 tunnel_futex;
	u64 tunnel_futex_insert;
	u64 enqueue_futex;
	u64 local_waiter_futex_insert;
	u64 local_waiter_futex_reject;
	u64 running_futex;
	u64 running_futex_changed;
	u64 running_futex_same;
	u64 stopping_futex;
	u64 stopping_futex_runnable;
	u64 stopping_futex_blocked;
	u64 schbench_direct_reset;
	u64 schbench_enqueue_reset;
	u64 schbench_stopping_reset;
	u64 schbench_stopping_runnable;
	u64 schbench_stopping_blocked;
	u64 latency_reset_enter;
	u64 latency_reset_decision;
	u64 latency_reset_owner_avg;
	u64 latency_reset_owner_runs;
	u64 latency_reset_cache_simple;
	u64 latency_reset_stream_pending;
	u64 latency_reset_status;
	u64 first_select_pid;
	u64 first_select_order;
	u64 first_idle_pid;
	u64 first_idle_order;
	u64 first_tunnel_pid;
	u64 first_tunnel_order;
	u64 first_enqueue_pid;
	u64 first_enqueue_order;
	u64 first_run_pid;
	u64 first_run_order;
} __attribute__((aligned(256)));

struct cake_futex_task_trace {
	u64 pid;
	u64 first_order;
	u64 first_cpu;
	u64 select_count;
	u64 idle_count;
	u64 tunnel_count;
	u64 enqueue_count;
	u64 run_count;
	u64 select_cpu_mask;
	u64 idle_cpu_mask;
	u64 tunnel_cpu_mask;
	u64 enqueue_cpu_mask;
	u64 run_cpu_mask;
} __attribute__((aligned(128)));
#endif

struct cake_scoreboard_summary {
	u64 idle_clean_mask; /* bit cpu = published idle and scoreboard-clean */
} __attribute__((aligned(64)));
_Static_assert(sizeof(struct cake_scoreboard_summary) == 64,
	       "cake_scoreboard_summary must stay one cache line");

struct cake_cpu_frontier {
	u64 vtime; /* Published vtime frontier for wake clamp/seeding */
} __attribute__((aligned(64)));
_Static_assert(sizeof(struct cake_cpu_frontier) == 64,
	       "cake_cpu_frontier must stay one cache line");

struct cake_throughput_lane {
	u64 pending; /* Advisory per-owner cache-throughput DSQ has work */
	u64 stream; /* Pending item is known stress-ng-memcpy service work */
} __attribute__((aligned(64)));
_Static_assert(sizeof(struct cake_throughput_lane) == 64,
	       "cake_throughput_lane must stay one cache line");

#ifndef CAKE_RELEASE_LOCAL_WAITER
#define CAKE_RELEASE_LOCAL_WAITER 1
#endif
#if CAKE_RELEASE_LOCAL_WAITER
#define CAKE_HAS_LOCAL_WAITER 1
#else
#define CAKE_HAS_LOCAL_WAITER 0
#endif

#if CAKE_HAS_LOCAL_WAITER
struct cake_local_waiter {
	u64 debt; /* Remote wake admitted a local waiter for owner quench */
} __attribute__((aligned(64)));
_Static_assert(sizeof(struct cake_local_waiter) == 64,
	       "cake_local_waiter must stay one cache line");
#endif

#ifndef CAKE_RELEASE_DOMAIN_DRR
#define CAKE_RELEASE_DOMAIN_DRR 1
#endif
#if CAKE_RELEASE_DOMAIN_DRR
#define CAKE_HAS_DOMAIN_DRR 1
#else
#define CAKE_HAS_DOMAIN_DRR 0
#endif

#if CAKE_HAS_DOMAIN_DRR
struct cake_domain_drr {
	u64 cache_pending;  /* cache-service DSQ probably has work */
	u64 stream_pending; /* stream/memcpy-service DSQ probably has work */
	u64 cache_burst;    /* cache pulls since last stream service */
} __attribute__((aligned(64)));
_Static_assert(sizeof(struct cake_domain_drr) == 64,
	       "cake_domain_drr must stay one cache line");
#endif

#ifndef CAKE_RELEASE_LLC_PENDING
#define CAKE_RELEASE_LLC_PENDING 0
#endif
#if defined(CAKE_RELEASE) && CAKE_RELEASE_LLC_PENDING
#define CAKE_HAS_LLC_PENDING 1
#else
#define CAKE_HAS_LLC_PENDING 0
#endif

#if CAKE_HAS_LLC_PENDING
struct cake_llc_pending {
	u64 pending; /* Conservative per-LLC shared DSQ has-work hint */
} __attribute__((aligned(64)));
_Static_assert(sizeof(struct cake_llc_pending) == 64,
	       "cake_llc_pending must stay one cache line");
#endif

struct cake_core_steal_pending {
	u64 pending; /* Conservative per-primary-CPU steal DSQ has-work hint */
} __attribute__((aligned(64)));
_Static_assert(sizeof(struct cake_core_steal_pending) == 64,
	       "cake_core_steal_pending must stay one cache line");

/* Userspace confidence governor.
 *
 * cake_trust_user is written by userspace at low cadence after the packed
 * confidence lanes prove a CPU is predictable. cake_trust_bpf is written by
 * BPF only on rare hard misses. Keeping ownership split avoids atomics in the
 * scheduler hot path: BPF reads a tiny policy word, and if the trusted claim
 * misses it blocks that generation until userspace cools down and promotes a
 * fresh one.
 */
#define CAKE_TRUST_FLAG_PREV_DIRECT (1U << 0)

enum cake_trust_demotion_reason {
	CAKE_TRUST_DEMOTE_NONE		  = 0,
	CAKE_TRUST_DEMOTE_PREV_CLAIM_MISS = 1,
	CAKE_TRUST_DEMOTE_MAX		  = 2,
};

struct cake_trust_user {
	u32 policy; /* CAKE_TRUST_FLAG_* enabled by userspace */
	u32 generation; /* Incremented by userspace on promotion */
} __attribute__((aligned(64)));
_Static_assert(sizeof(struct cake_trust_user) == 64,
	       "cake_trust_user must stay one cache line");

struct cake_trust_bpf {
	u32 blocked; /* CAKE_TRUST_FLAG_* blocked for generation */
	u32 generation; /* User generation observed when blocked */
	u32 reason; /* enum cake_trust_demotion_reason */
	u32 demotion_count; /* Rare hard misses, cumulative */
	u64 cooldown_until; /* BPF clock (ns) when cooldown ends */
	u64 cooldown_ns; /* Current cooldown duration (ns) */
	u64 last_miss_ns; /* Timestamp of the last demotion */
} __attribute__((aligned(64)));
_Static_assert(sizeof(struct cake_trust_bpf) == 64,
	       "cake_trust_bpf must stay one cache line");

enum cake_storm_guard_mode {
	CAKE_STORM_GUARD_OFF	= 0,
	CAKE_STORM_GUARD_SHADOW = 1,
	CAKE_STORM_GUARD_SHIELD = 2,
	CAKE_STORM_GUARD_FULL	= 3,
	CAKE_STORM_GUARD_MAX	= 4,
};

enum cake_storm_guard_decision {
	CAKE_STORM_GUARD_CANDIDATE	 = 0,
	CAKE_STORM_GUARD_BASE_ALLOW	 = 1,
	CAKE_STORM_GUARD_SHADOW_ONLY	 = 2,
	CAKE_STORM_GUARD_SHIELD_ALLOW	 = 3,
	CAKE_STORM_GUARD_FULL_ALLOW	 = 4,
	CAKE_STORM_GUARD_SMT_BLOCKED	 = 5,
	CAKE_STORM_GUARD_UNKNOWN_OWNER	 = 6,
	CAKE_STORM_GUARD_MODE_DISABLED	 = 7,
	CAKE_STORM_GUARD_POLICY_REJECTED = 8,
	CAKE_STORM_GUARD_DECISION_MAX	 = 9,
};

/* ═══ COMPILE-TIME HARDWARE SCALING ═══
 * build.rs detects host CPU/LLC count from sysfs at compile time,
 * rounds to next power-of-2, and passes -DCAKE_MAX_CPUS=N -DCAKE_MAX_LLCS=N.
 * All arrays, loops, BSS, and RODATA compile to exactly the hardware size.
 *
 * Power-of-2 required: & (CAKE_MAX_CPUS - 1) bitmask at 50+ indexing sites.
 * Range [16, 512] covers 4-core old CPUs (clamped up) to dual EPYC.
 *
 * Fallback values used when -D not provided (generic/CI builds). */
#ifndef CAKE_MAX_CPUS
#define CAKE_MAX_CPUS 64
#endif
/* Compile-time validation (BPF compilation only — skipped during Rust bindgen) */
#ifdef __BPF__
_Static_assert((CAKE_MAX_CPUS & (CAKE_MAX_CPUS - 1)) == 0,
	       "CAKE_MAX_CPUS must be power of 2");
_Static_assert(CAKE_MAX_CPUS >= 16 && CAKE_MAX_CPUS <= 512,
	       "CAKE_MAX_CPUS out of range [16, 512]");
#endif

#ifndef CAKE_MAX_LLCS
#define CAKE_MAX_LLCS 8
#endif

#ifdef __BPF__
_Static_assert((CAKE_MAX_LLCS & (CAKE_MAX_LLCS - 1)) == 0,
	       "CAKE_MAX_LLCS must be power of 2");
#endif

/* CPU mask word count: ceiling division for <64 CPU systems.
 * At 16 CPUs: (16+63)/64 = 1 word.  At 64: 1.  At 512: 8. */
#define CAKE_CPU_MASK_WORDS ((CAKE_MAX_CPUS + 63) / 64)

/* Auto-size CPU ID type: u8 for <256 CPUs (consumer), u16 for ≥256 (EPYC).
 * Strict <256: CPU ID 255 must NOT collide with sentinel 0xFF.
 * Saves 1 byte per RODATA entry on consumer hardware vs always-u16.
 * Uses standard C types for bindgen compatibility. */
#if CAKE_MAX_CPUS < 256
typedef unsigned char cake_cpu_id_t;
#define CAKE_CPU_SENTINEL 0xFF
#else
typedef unsigned short cake_cpu_id_t;
#define CAKE_CPU_SENTINEL 0xFFFF
#endif

/* Telemetry histogram: scales with hardware for accurate per-CPU run counts.
 * At 16 CPUs: 32B/task (16×u16).  At 64: 128B/task.  Compiled out in release. */
#define CAKE_TELEM_MAX_CPUS CAKE_MAX_CPUS

/* Physical core array sizing: MAX/2 assumes SMT=2 (AMD/Intel consumer).
 * _Static_assert fires if a non-SMT build exceeds this (future EPYC). */
#define CAKE_MAX_CORES (CAKE_MAX_CPUS / 2)
#ifdef __BPF__
#if CAKE_NEEDS_ARENA
#ifndef __arena
#define __arena __attribute__((address_space(1)))
#endif

#define LFDEQ_CAPACITY 128
#define LFDEQ_CAPACITY_SHIFT 7
#if (1U << LFDEQ_CAPACITY_SHIFT) != LFDEQ_CAPACITY
#error "LFDEQ_CAPACITY must match LFDEQ_CAPACITY_SHIFT"
#endif

struct lfdeq_slot {
	u64 pid;
	u64 pad[7];
} __attribute__((aligned(64)));

struct lfdeq_ring {
	u64 head __attribute__((aligned(64)));
	u64 tail __attribute__((aligned(64)));
	struct lfdeq_slot tasks[LFDEQ_CAPACITY] __attribute__((aligned(64)));
} __attribute__((aligned(64)));

struct scx_lfdeq {
	// Local Chase-Lev deque (single producer, multi consumer)
	u64 head __attribute__((aligned(64)));
	u64 tail __attribute__((aligned(64)));
	struct lfdeq_slot tasks[LFDEQ_CAPACITY] __attribute__((aligned(64)));

	// Enqueue (write) buffer (multi producer, single consumer)
	struct lfdeq_ring eq_buf __attribute__((aligned(64)));
} __attribute__((aligned(64)));

extern struct scx_lfdeq __arena *cpu_lfdeqs[CAKE_MAX_CPUS];

struct scx_dhq;
extern struct scx_dhq __arena *core_dhqs[CAKE_MAX_CORES];
#endif
#endif
/* Queue policy selected by loader through RODATA. */
#define CAKE_QUEUE_POLICY_LOCAL 0U
#define CAKE_QUEUE_POLICY_VTIME_LOCAL 1U
/* Backward-compatible name for old scripts/history.  The release build treats
 * this value as vtime-guided local placement, not shared LLC-DSQ custody. */
#define CAKE_QUEUE_POLICY_LLC_VTIME CAKE_QUEUE_POLICY_VTIME_LOCAL

/* Shared-DSQ base kept for debug/experimental service paths. Release
 * vtime-local does not store normal hot-path work here. */
#define LLC_DSQ_BASE 200
#define CAKE_THROUGHPUT_DSQ_BASE 1024
#define CAKE_DOMAIN_DRR_DSQ_BASE 2048
/* Per-primary-CPU user DSQs used for work-stealable busy-wake fallback. */
#define CAKE_CORE_STEAL_DSQ_BASE 4096
#define CAKE_DOMAIN_DRR_CLASS_CACHE 0U
#define CAKE_DOMAIN_DRR_CLASS_STREAM 1U
#define CAKE_DOMAIN_DRR_CLASS_MAX 2U

/* STAGED_BIT_* defines REMOVED: staged_vtime_bits field was dead.
 * Ordering state now lives in p->scx.dsq_vtime; legacy class/tier metadata
 * remains only in packed_info/telemetry paths. */

/* Select cascade:
 *   1. Cake route prediction from packed per-CPU confidence
 *   2. Cake scoreboard probes over prev/fast-probe candidates
 *   3. Native kernel idle helper fallback for correctness and broad search
 *   4. Hybrid perf-ordered backup when compiled in
 *   5. Tunnel: all CPUs busy -> return prev_cpu for local enqueue */

/* Flow state flags (packed_info bits 24-27).
 * packed_info is now primarily an iter/debug transport field.
 * Only NEW/YIELDER and KCRITICAL have live readers; the rest are
 * retained for telemetry compatibility. */
enum cake_flow_flags {
	CAKE_FLOW_NEW	  = 1 << 0, /* Task is newly created */
	CAKE_FLOW_YIELDER = 1
			    << 1, /* Task voluntarily yielded since last stop */
	CAKE_FLOW_WAKER_BOOST = 1 << 2, /* Reserved legacy flag */
	CAKE_FLOW_HOG	      = 1 << 3, /* Reserved legacy flag */
};

enum cake_wake_reason {
	CAKE_WAKE_REASON_NONE	= 0,
	CAKE_WAKE_REASON_DIRECT = 1,
	CAKE_WAKE_REASON_BUSY	= 2,
	CAKE_WAKE_REASON_QUEUED = 3,
	CAKE_WAKE_REASON_MAX	= 4,
};

enum cake_wake_bucket_idx {
	CAKE_WAKE_BUCKET_LT50US	 = 0,
	CAKE_WAKE_BUCKET_LT200US = 1,
	CAKE_WAKE_BUCKET_LT1MS	 = 2,
	CAKE_WAKE_BUCKET_LT5MS	 = 3,
	CAKE_WAKE_BUCKET_GE5MS	 = 4,
	CAKE_WAKE_BUCKET_MAX	 = 5,
};

enum cake_kick_kind {
	CAKE_KICK_KIND_NONE    = 0,
	CAKE_KICK_KIND_IDLE    = 1,
	CAKE_KICK_KIND_PREEMPT = 2,
	CAKE_KICK_KIND_MAX     = 3,
};

enum cake_ready_bit {
	CAKE_READY_URGENT_SERVICE	= 1U << 0,
	CAKE_READY_FAIRNESS_OK		= 1U << 1,
	CAKE_READY_OWNER_PREEMPT_OK	= 1U << 2,
	CAKE_READY_ROUTE_KNOWN		= 1U << 3,
	CAKE_READY_CACHE_DAMAGE_OK	= 1U << 4,
	CAKE_READY_KICK_BUYS_SERVICE	= 1U << 5,
};

enum cake_ready_action {
	CAKE_READY_ACT_NONE		= 0,
	CAKE_READY_ACT_LOCAL_INSERT	= 1,
	CAKE_READY_ACT_IDLE_KICK	= 2,
	CAKE_READY_ACT_PREEMPT_KICK	= 3,
	CAKE_READY_ACT_KEEP_OWNER	= 4,
	CAKE_READY_ACT_THROUGHPUT_LANE	= 5,
	CAKE_READY_ACT_SHARED_ESCAPE	= 6,
	CAKE_READY_ACT_CORE_STEAL_PROBE	= 7,
	CAKE_READY_ACT_MAX		= 8,
};

enum cake_wake_class {
	CAKE_WAKE_CLASS_NONE	= 0,
	CAKE_WAKE_CLASS_NORMAL	= 1,
	CAKE_WAKE_CLASS_SHIELD	= 2,
	CAKE_WAKE_CLASS_CONTAIN = 3,
	CAKE_WAKE_CLASS_MAX	= 4,
};

enum cake_wake_class_reason {
	CAKE_WAKE_CLASS_REASON_LOW_UTIL	     = 0,
	CAKE_WAKE_CLASS_REASON_SHORT_RUN     = 1,
	CAKE_WAKE_CLASS_REASON_WAKE_DENSE    = 2,
	CAKE_WAKE_CLASS_REASON_LATENCY_PRIO  = 3,
	CAKE_WAKE_CLASS_REASON_RUNTIME_HEAVY = 4,
	CAKE_WAKE_CLASS_REASON_PREEMPT_HEAVY = 5,
	CAKE_WAKE_CLASS_REASON_PRESSURE_HIGH = 6,
	CAKE_WAKE_CLASS_REASON_YIELD_HEAVY   = 7,
	CAKE_WAKE_CLASS_REASON_WAIT_TAIL     = 8,
	CAKE_WAKE_CLASS_REASON_MAX	     = 9,
};

enum cake_busy_preempt_shadow {
	CAKE_BUSY_PREEMPT_SHADOW_ALLOW = 0,
	CAKE_BUSY_PREEMPT_SHADOW_SKIP  = 1,
	CAKE_BUSY_PREEMPT_SHADOW_MAX   = 2,
};

enum cake_place_class {
	CAKE_PLACE_HOME_CPU  = 0,
	CAKE_PLACE_HOME_CORE = 1,
	CAKE_PLACE_HOME_LLC  = 2,
	CAKE_PLACE_REMOTE    = 3,
	CAKE_PLACE_CLASS_MAX = 4,
};

enum cake_select_path {
	CAKE_SELECT_PATH_NONE	   = 0,
	CAKE_SELECT_PATH_HOME_CPU  = 1,
	CAKE_SELECT_PATH_HOME_CORE = 2,
	CAKE_SELECT_PATH_PRIMARY   = 3,
	CAKE_SELECT_PATH_IDLE	   = 4,
	CAKE_SELECT_PATH_TUNNEL	   = 5,
	CAKE_SELECT_PATH_MAX	   = 6,
};

/* Debug-only select_cpu reason tracking.
 * CAKE_SELECT_PATH keeps the historical high-level buckets used by the dump.
 * These reasons split the coarse "idle" and "primary" paths into the exact
 * winner that led to the CPU choice so debug builds can explain hot-core
 * stickiness precisely. */
enum cake_select_reason {
	CAKE_SELECT_REASON_NONE		   = 0,
	CAKE_SELECT_REASON_HOME_CPU	   = 1,
	CAKE_SELECT_REASON_HOME_CORE	   = 2,
	CAKE_SELECT_REASON_PREV_PRIMARY	   = 3,
	CAKE_SELECT_REASON_PRIMARY_SCAN	   = 4,
	CAKE_SELECT_REASON_HYBRID_SCAN	   = 5,
	CAKE_SELECT_REASON_KERNEL_PREV	   = 6,
	CAKE_SELECT_REASON_KERNEL_IDLE	   = 7,
	CAKE_SELECT_REASON_TUNNEL	   = 8,
	CAKE_SELECT_REASON_PRESSURE_CORE   = 9,
	CAKE_SELECT_REASON_SCOREBOARD_PREV = 10,
	CAKE_SELECT_REASON_SCOREBOARD_SCAN = 11,
	CAKE_SELECT_REASON_CORE_SPREAD	   = 12,
	CAKE_SELECT_REASON_MAX		   = 13,
};

/* Debug-only accelerator attribution. These counters are userspace-facing
 * telemetry for the confidence/prediction path; release builds compile their
 * writers out. */
enum cake_accel_route_stat {
	CAKE_ACCEL_ROUTE_MAX = 7,
};

enum cake_accel_route_block {
	CAKE_ACCEL_BLOCK_INVALID_PREV  = 0,
	CAKE_ACCEL_BLOCK_AFFINITY      = 1,
	CAKE_ACCEL_BLOCK_KTHREAD       = 2,
	CAKE_ACCEL_BLOCK_ROUTE_LOW     = 3,
	CAKE_ACCEL_BLOCK_SELECT_LOW    = 4,
	CAKE_ACCEL_BLOCK_TRUST_LOW     = 5,
	CAKE_ACCEL_BLOCK_LOAD_SHOCK    = 6,
	CAKE_ACCEL_BLOCK_FLOOR_LOW     = 7,
	CAKE_ACCEL_BLOCK_OWNER_LOW     = 8,
	CAKE_ACCEL_BLOCK_PULL_LOW      = 9,
	CAKE_ACCEL_BLOCK_AUDIT	       = 10,
	CAKE_ACCEL_BLOCK_LATENCY_GATE  = 11,
	CAKE_ACCEL_BLOCK_UNKNOWN_ROUTE = 12,
	CAKE_ACCEL_ROUTE_BLOCK_MAX     = 13,
};

enum cake_accel_probe_outcome {
	CAKE_ACCEL_PROBE_HIT	     = 0,
	CAKE_ACCEL_PROBE_BUSY	     = 1,
	CAKE_ACCEL_PROBE_DIRTY	     = 2,
	CAKE_ACCEL_PROBE_SMT_BUSY    = 3,
	CAKE_ACCEL_PROBE_CLAIM_FAIL  = 4,
	CAKE_ACCEL_PROBE_CLAIM_SKIP  = 5,
	CAKE_ACCEL_PROBE_INVALID     = 6,
	CAKE_ACCEL_PROBE_OUTCOME_MAX = 7,
};

enum cake_accel_pull_mode {
	CAKE_ACCEL_PULL_MODE_PULL  = 0,
	CAKE_ACCEL_PULL_MODE_PROBE = 1,
	CAKE_ACCEL_PULL_MODE_SKIP  = 2,
	CAKE_ACCEL_PULL_MODE_MAX   = 3,
};

enum cake_accel_pull_probe {
	CAKE_ACCEL_PULL_PROBE_EMPTY = 0,
	CAKE_ACCEL_PULL_PROBE_WORK  = 1,
	CAKE_ACCEL_PULL_PROBE_MAX   = 2,
};

enum cake_accel_native_fallback {
	CAKE_ACCEL_NATIVE_ENTRY = 0,
	CAKE_ACCEL_NATIVE_DFL	= 1,
	CAKE_ACCEL_NATIVE_AND	= 2,
	CAKE_ACCEL_NATIVE_MAX	= 3,
};

enum cake_frontier_mode_stat {
	CAKE_FRONTIER_MODE_NONE  = 0,
	CAKE_FRONTIER_MODE_CACHE = 1,
	CAKE_FRONTIER_MODE_FAIR  = 2,
	CAKE_FRONTIER_MODE_MIXED = 3,
	CAKE_FRONTIER_MODE_MAX   = 4,
};

/* Debug coverage uses compact ringbuf events as its BPF/userspace boundary.
 * BPF emits raw facts; userspace builds wake graphs, policy shadows, and
 * long-window histograms so debug coverage does not consume verifier budget
 * with in-BPF analytics tables. */
#ifndef CAKE_RELEASE
#define CAKE_DEBUG_EVENT_STREAM 1
#else
#define CAKE_DEBUG_EVENT_STREAM 0
#endif

#define CAKE_WAKE_EDGE_SAMPLE_NS 1000000000ULL
#define CAKE_WAKE_EDGE_SAMPLE_DENOM 64U

/* Debug-only pressure spill diagnostics.
 * "anchor" groups structural blockers where there is no usable same-core
 * spill candidate. The companion anchor-block enum breaks that aggregate
 * down so dumps can distinguish bad inputs from intentional topology gates. */
enum cake_pressure_probe_site {
	CAKE_PRESSURE_PROBE_SITE_HOME = 0,
	CAKE_PRESSURE_PROBE_SITE_PREV = 1,
	CAKE_PRESSURE_PROBE_SITE_MAX  = 2,
};

enum cake_pressure_probe_outcome {
	CAKE_PRESSURE_PROBE_EVALUATED		 = 0,
	CAKE_PRESSURE_PROBE_BLOCKED_ANCHOR	 = 1,
	CAKE_PRESSURE_PROBE_BLOCKED_SCORE	 = 2,
	CAKE_PRESSURE_PROBE_BLOCKED_DELTA	 = 3,
	CAKE_PRESSURE_PROBE_BLOCKED_SIBLING_BUSY = 4,
	CAKE_PRESSURE_PROBE_SUCCESS		 = 5,
	CAKE_PRESSURE_PROBE_OUTCOME_MAX		 = 6,
};

enum cake_pressure_anchor_block_reason {
	CAKE_PRESSURE_ANCHOR_INVALID	= 0,
	CAKE_PRESSURE_ANCHOR_SECONDARY	= 1,
	CAKE_PRESSURE_ANCHOR_NO_SIBLING = 2,
	CAKE_PRESSURE_ANCHOR_AFFINITY	= 3,
	CAKE_PRESSURE_ANCHOR_REASON_MAX = 4,
};

enum cake_cb_idx {
	CAKE_CB_SELECT	 = 0,
	CAKE_CB_ENQUEUE	 = 1,
	CAKE_CB_DISPATCH = 2,
	CAKE_CB_RUNNING	 = 3,
	CAKE_CB_STOPPING = 4,
	CAKE_CB_MAX	 = 5,
};

enum cake_cb_bucket_idx {
	CAKE_CB_BUCKET_LT250NS = 0,
	CAKE_CB_BUCKET_LT500NS = 1,
	CAKE_CB_BUCKET_LT1US   = 2,
	CAKE_CB_BUCKET_LT2US   = 3,
	CAKE_CB_BUCKET_LT5US   = 4,
	CAKE_CB_BUCKET_LT10US  = 5,
	CAKE_CB_BUCKET_GE10US  = 6,
	CAKE_CB_BUCKET_MAX     = 7,
};

enum cake_dbg_event_kind {
	CAKE_DBG_EVENT_CALLBACK		 = 1,
	CAKE_DBG_EVENT_WAKEWAIT		 = 2,
	CAKE_DBG_EVENT_WAKE_TARGET_MISS	 = 3,
	CAKE_DBG_EVENT_KICK_SLOW	 = 4,
	CAKE_DBG_EVENT_WAKE_FOLLOW_MIG	 = 5,
	CAKE_DBG_EVENT_DISPATCH_GAP	 = 6,
	CAKE_DBG_EVENT_PREEMPT_CHAIN	 = 7,
	CAKE_DBG_EVENT_WAKE_EDGE_ENQUEUE = 8,
	CAKE_DBG_EVENT_WAKE_EDGE_RUN	 = 9,
	CAKE_DBG_EVENT_WAKE_EDGE_FOLLOW	 = 10,
};

#define CAKE_WAKE_EDGE_EVENT_FLAG_HIT_OR_SAME 1U
#define CAKE_WAKE_EDGE_EVENT_FLAG_SAMPLED 2U
#define CAKE_WAKE_EDGE_EVENT_FLAG_IMPORTANT 4U

struct cake_debug_event {
	u64  ts_ns;
	u64  value_ns;
	u32  pid;
	u32  aux;
	u32  tgid;
	u32  peer_pid;
	u32  peer_tgid;
	u16  cpu;
	u16  target_cpu;
	u16  peer_cpu;
	u8   kind;
	u8   slot;
	u8   reason;
	u8   path;
	u8   home_place;
	u8   waker_place;
	u8   flags;
	u8   _pad[3];
	char comm[16];
};

/* Record emitted by SEC("iter/task") cake_task_iter program.
 * One record per managed task. Userspace reads fixed-size records from the
 * iter fd instead of walking the pid_to_tctx hash map.
 * Replaces: pid_to_tctx BPF_MAP_TYPE_HASH (65536 entries, global bucket lock).
 * Benefit: cake_init_task + cake_exit_task become fully lockless (no map ops). */
struct cake_iter_record {
	u32 pid; /* pid of this task */
	u32 ppid; /* parent pid (promoted from tctx CL0) */
	u32 packed_info; /* iter/debug packed flags — read in Rust telemetry loop */
	u16 pelt_util; /* PELT util_avg (0-1024) from p->se.avg */
	u16 allowed_cpus; /* p->nr_cpus_allowed, capped to u16 */
	u16 task_weight; /* Raw task weight mirrored for iter/TUI display (100=nice0) */
	u16 home_cpu; /* Current sticky home CPU, or CAKE_CPU_SENTINEL */
	/* telemetry block: everything the TUI currently reads from arena pointers */
	struct {
		u64 run_start_ns;
		u64 run_duration_ns;
		u64 total_runtime_ns;
		u64 enqueue_start_ns;
		u64 wait_duration_ns;
		u32 select_cpu_duration_ns;
		u32 enqueue_duration_ns;
		u32 dsq_insert_ns;
		u32 gate_1_hits;
		u32 gate_2_hits;
		u32 gate_1w_hits;
		u32 gate_3_hits;
		u32 gate_1p_hits;
		u32 gate_1c_hits;
		u32 gate_1cp_hits;
		u32 gate_1d_hits;
		u32 gate_1wc_hits;
		u32 gate_tun_hits;
		u32 _pad2; /* debug iter anatomy: task_flags */
		u64 jitter_accum_ns;
		u32 total_runs;
		u16 core_placement;
		u16 migration_count;
		u16 preempt_count;
		u16 yield_count;
		u16 direct_dispatch_count;
		u16 enqueue_count;
		u16 cpumask_change_count;
		u16 _pad3; /* debug iter anatomy: task_policy */
		u32 stopping_duration_ns;
		u32 running_duration_ns;
		u32 max_runtime_us;
		u32 _pad4; /* debug iter anatomy: prio/static/normal packed as 3 u8s */
		u64 dispatch_gap_ns;
		u64 max_dispatch_gap_ns;
		u32 wait_hist_lt10us;
		u32 wait_hist_lt100us;
		u32 wait_hist_lt1ms;
		u32 wait_hist_ge1ms;
		u16 slice_util_pct;
		u16 llc_id;
		u16 llc_run_mask;
		u16 same_cpu_streak;
		u16 _pad_recomp; /* debug iter anatomy: bit0 has_mm, bit1 kthread */
		u32  wakeup_source_pid;
		u64  nivcsw_snapshot;
		u32  nvcsw_delta;
		u32  nivcsw_delta;
		u32  pid_inner; /* matches telemetry.pid */
		u32  tgid;
		char comm[16];
		u32  gate_cascade_ns;
		u32  lifecycle_init_ms;
		u32  vtime_compute_ns;
		u32  mbox_staging_ns;
		u32  startup_latency_us;
		u32  startup_enqueue_us;
		u32  lifecycle_live_ms;
		u32  startup_select_us;
		u64  quantum_full_count;
		u64  quantum_yield_count;
		u64  quantum_preempt_count;
		u8   startup_first_phase;
		u8   startup_phase_mask;
		u16  waker_cpu;
		u16  _pad_waker;
		u32  waker_tgid;
		u64  wake_reason_wait_ns[CAKE_WAKE_REASON_MAX - 1];
		u32  wake_reason_count[CAKE_WAKE_REASON_MAX - 1];
		u32  wake_reason_max_us[CAKE_WAKE_REASON_MAX - 1];
		u8   last_select_reason;
		u8   last_select_path;
		u8   last_place_class;
		u8   last_waker_place_class;
		u32  wake_same_tgid_count;
		u32  wake_cross_tgid_count;
		u64  home_place_wait_ns[CAKE_PLACE_CLASS_MAX];
		u32  home_place_wait_count[CAKE_PLACE_CLASS_MAX];
		u32  home_place_wait_max_us[CAKE_PLACE_CLASS_MAX];
		u64  smt_solo_runtime_ns;
		u64  smt_contended_runtime_ns;
		u64  smt_overlap_runtime_ns;
		u32  smt_solo_run_count;
		u32  smt_contended_run_count;
		u16  cpu_run_count[CAKE_TELEM_MAX_CPUS];
	} telemetry;
};

/* ═══════════════════════════════════════════════════════════════════════════
 * CONDITIONAL SIZE MACROS — Release/Debug struct sizing pattern
 *
 * DESIGN PHILOSOPHY: In release (CAKE_RELEASE=1, CAKE_STATS_ENABLED=0),
 * Clang dead-code eliminates every telemetry access. Structs
 * shrink to their scheduling-essential footprint by #ifdef-gating
 * debug-only fields. This saves arena memory, TLB footprint, and
 * cache pollution in production builds.
 *
 * Structs using this pattern:
 *   cake_task_ctx:       64B release (1 CL) / 576B debug (9 CL)
 * ═══════════════════════════════════════════════════════════════════════════ */
#ifdef CAKE_RELEASE
#define CAKE_TCTX_SIZE 64
#define CAKE_TCTX_ALIGN 64
#else
#define CAKE_TCTX_SIZE 576
#define CAKE_TCTX_ALIGN 64
#endif

/* Per-task context — conditional sizing (Release/Debug pattern).
 *   RELEASE: 64B (1 CL) — iter-visible fields only, telemetry compiled out
 *   DEBUG:  576B (9 CL) — iter-visible fields plus TUI telemetry
 *
 * CACHE LINE SEGREGATION:
 *   CL0 (bytes 0-63): Fields needed for iter output and debug bookkeeping.
 *     aligned(CAKE_TCTX_ALIGN) handles CL0 padding automatically —
 *     no manual _pad arrays needed.
 *   CL1+ (bytes 64+): Debug-only telemetry for TUI analytics.
 *     Compiled out in release → struct shrinks from 448B to 64B. */
struct cake_task_ctx {
	/* ═══ CACHE LINE 0: RELEASE + ITER FIELDS ═══ */
	u32 packed_info; /* 4B: Bitfield flags (iter-visible) */
	u32 ppid; /* 4B: Parent PID (iter-visible) */
	u64 last_run_ns; /* 8B: timestamp when task stopped running */
	u16 task_weight; /* 2B: Raw task weight for iter/TUI display (100=nice0) */
	u16 home_cpu; /* 2B: Sticky preferred CPU (primary SMT lane) */
	u8  home_score; /* 1B: Hysteresis for updating home_cpu */
	u8  home_core; /* 1B: Sticky preferred physical core */

#ifdef CAKE_RELEASE
	u8 primary_scan_credit; /* 1B: Guarded primary scans periodically earn one probe */
#else
	/* ── DEBUG-ONLY CL0 FIELDS ── */
	u8 task_class; /* 1B: CAKE_CLASS_* enum (iter sync, debug stopping) */
	u8 primary_scan_credit; /* 1B: Guarded primary scans periodically earn one probe */
	u64 nvcsw_snapshot; /* 8B: voluntary ctx switch snapshot (debug stopping) */
#endif
	/* Compiler pads to CAKE_TCTX_ALIGN via aligned attribute. */

#ifndef CAKE_RELEASE
	/* ═══ CL1+ (bytes 64-447): DEBUG-ONLY TELEMETRY ═══
	 * Compiled out in release → struct shrinks from 576B to 64B.
	 * Zero-cost pointer access via BPF Arena. User-space sweeps memory
	 * asynchronously to build 1% Lows and average runtimes. */
	struct {
		/* Timing Metrics */
		u64 run_start_ns;
		u64 run_duration_ns; /* Last observed runtime (stop - run_start) */
		u64 total_runtime_ns; /* Cumulative runtime charged to this task */
		u64 enqueue_start_ns; /* Internal wake-to-run staging timestamp */
		u64 wait_duration_ns; /* Last observed wake-to-run wait */
		u32 select_cpu_duration_ns; /* Last observed select_cpu overhead */
		u32 enqueue_duration_ns; /* Last observed enqueue overhead */
		u32 dsq_insert_ns; /* Last observed DSQ insert/vtime overhead */

		/* Topographic / Cache Data */
		u32 gate_1_hits; /* Number of local cache hit wakeups */
		u32 gate_2_hits; /* SMT sibling hits */
		u32 gate_1w_hits; /* Waker affinity hits (SMT sibling or LLC) */
		u32 gate_3_hits; /* Kernel fallback hits */
		u32 gate_1p_hits; /* Yielder-preempts-bulk hits (Gate 1P) */
		u32 gate_1c_hits; /* Home CPU warm set hits (Gate 1c) */
		u32 gate_1cp_hits; /* Home CPU preempt-hog hits (Gate 1c-P) */
		u32 gate_1d_hits; /* Domestic: same-process cache affinity (Gate 1D) */
		u32 gate_1wc_hits; /* Waker-chain: producer-consumer locality (Gate 1WC) */
		u32 gate_tun_hits; /* Complete miss tunneling */
		u64 jitter_accum_ns; /* Sum of |observed runtime - expected runtime| */
		u32 total_runs; /* Total executions over task lifetime */
		u16 core_placement; /* Physical CPU task last executed on */

		/* State Change Counters */
		u16 migration_count; /* Inter-cpu bounces inside select_cpu */
		u16 preempt_count; /* Task kicked/preempted */
		u16 yield_count; /* Explicit sched_yield callbacks */

		/* Lifecycle Counters */
		u16 direct_dispatch_count; /* SCX_DSQ_LOCAL_ON bypasses (no DSQ) */
		u16 enqueue_count; /* Total enqueue calls */
		u16 cpumask_change_count; /* sched_setaffinity changes */

		/* Callback Overhead (last-write-wins, ns) */
		u32 stopping_duration_ns; /* Last observed cake_stopping BPF overhead */
		u32 running_duration_ns; /* Last observed cake_running BPF overhead */

		/* Worst-Case Tracking */
		u32 max_runtime_us; /* Worst observed runtime over task lifetime */

		/* Scheduling Period (inter-dispatch gap) */
		u64 dispatch_gap_ns; /* Last observed gap since previous run start */
		u64 max_dispatch_gap_ns; /* Worst observed gap over task lifetime */

		/* Wait Latency Histogram (bucket counts, lifetime) */
		u32 wait_hist_lt10us; /* wait < 10µs */
		u32 wait_hist_lt100us; /* 10µs <= wait < 100µs */
		u32 wait_hist_lt1ms; /* 100µs <= wait < 1ms */
		u32 wait_hist_ge1ms; /* wait >= 1ms */

		/* Slice Utilization Metrics */
		u16 slice_util_pct; /* Approximate slice occupancy score where 128 ~= full slice */
		u16 llc_id; /* LLC node this task last ran on */
		u16 llc_run_mask; /* Bitmask of LLCs the task has run on */
		u16 same_cpu_streak; /* Consecutive runs on same CPU */
		u16 _pad_recomp; /* (was _deprecated_recomp — removed) */
		u32 wakeup_source_pid; /* PID that woke this task */

		/* Voluntary/involuntary context switch tracking (GPU detection) */
		u64 nivcsw_snapshot; /* Last read of p->nivcsw (for delta) */
		u32 nvcsw_delta; /* Accumulated nvcsw delta over task lifetime */
		u32 nivcsw_delta; /* Accumulated nivcsw delta over task lifetime */

		u32  pid;
		u32  tgid; /* Thread group ID (process) for TUI grouping */
		char comm[16];

		/* Per-callback sub-function stopwatch (verbose health).
		 * Several slots are retained only so the iter/TUI layout stays stable. */
		u32 gate_cascade_ns; /* select_cpu: full gate cascade duration */
		u32 lifecycle_init_ms; /* task init timestamp in monotonic ms */
		u32 vtime_compute_ns; /* enqueue: vtime calc minus DSQ insert */
		u32 mbox_staging_ns; /* running: mailbox CL0 write burst */
		u32 startup_latency_us; /* task init to first observed run, us */
		u32 startup_enqueue_us; /* task init to first observed enqueue, us */
		u32 lifecycle_live_ms; /* iter snapshot age for live task, ms */
		u32 startup_select_us; /* task init to first observed select_cpu, us */

		/* Quantum completion tracking */
		u64 quantum_full_count; /* Task consumed the full slice */
		u64 quantum_yield_count; /* Task stopped with slice left and became non-runnable */
		u64 quantum_preempt_count; /* Task was kicked/preempted while still runnable */
		u8 startup_first_phase; /* enum cake_startup_phase: first observed scheduler phase */
		u8 startup_phase_mask; /* CAKE_STARTUP_MASK_* phases observed before first run */

		/* Wake chain tracking */
		u16 waker_cpu; /* CPU the waker was running on */
		u8 pending_ready_bits; /* HELIOS-style debug readiness bitset */
		u8 pending_ready_action; /* enum cake_ready_action at enqueue */
		u32 waker_tgid; /* TGID of the waker (process group) */
		u64 wake_reason_wait_ns[CAKE_WAKE_REASON_MAX -
					1]; /* Wait by wake path */
		u32 wake_reason_count[CAKE_WAKE_REASON_MAX -
				      1]; /* Samples by wake path */
		u32 wake_reason_max_us[CAKE_WAKE_REASON_MAX -
				       1]; /* Worst wait by wake path */
		u8 pending_wake_reason; /* Wake path recorded at enqueue, consumed at run */
		u8 pending_select_path; /* select_cpu path recorded before the run */
		u8 pending_kick_kind; /* Last wake kick type issued for this wake */
		u8 postwake_watch; /* Whether the next run should be tracked as a post-wake continuation */
		u16 pending_target_cpu; /* CPU chosen at enqueue for the current wake */
		u16 postwake_first_cpu; /* CPU used for the first run after the wake */
		u8 postwake_reason; /* Wake reason associated with postwake_first_cpu */
		u8 pending_select_reason; /* select_cpu reason that fed the next run */
		u64 pending_kick_ts_ns; /* Timestamp when the last wake kick was issued */
		u8 last_select_path; /* Last select path that actually led to a run */
		u8 last_select_reason; /* Last select_cpu reason that fed an observed run */
		u8 last_place_class; /* Last run vs sticky home: cpu/core/llc/remote */
		u8 last_waker_place_class; /* Last run vs waker: cpu/core/llc/remote */
		u32 wake_same_tgid_count; /* Wake chain stayed inside the same process */
		u32 wake_cross_tgid_count; /* Wake came from a different TGID */
		u64 home_place_wait_ns
			[CAKE_PLACE_CLASS_MAX]; /* Wait totals by home locality */
		u32 home_place_wait_count
			[CAKE_PLACE_CLASS_MAX]; /* Wait samples by home locality */
		u32 home_place_wait_max_us
			[CAKE_PLACE_CLASS_MAX]; /* Worst wait by home locality */
		u32 pending_blocker_pid; /* CPU owner observed when enqueue hit busy target */
		u16 pending_blocker_cpu; /* Busy target CPU for pending wake */
		u8 pending_strict_owner_class; /* Strict owner class on busy target */
		u8 pending_target_pressure; /* Pressure observed on busy target */

		/* Task-local SMT overlap attribution */
		u64 smt_solo_runtime_ns; /* Runtime completed without sibling overlap */
		u64 smt_contended_runtime_ns; /* Runtime completed with sibling overlap */
		u64 smt_overlap_runtime_ns; /* Estimated overlap charged to this task */
		u32 smt_solo_run_count; /* Runs without observed sibling overlap */
		u32 smt_contended_run_count; /* Runs with observed sibling overlap */

		/* CPU core distribution histogram */
		u16 cpu_run_count
			[CAKE_TELEM_MAX_CPUS]; /* 128 bytes: per-CPU run count */
	} telemetry;
#endif /* !CAKE_RELEASE */
} __attribute__((aligned(CAKE_TCTX_ALIGN)));
_Static_assert(sizeof(struct cake_task_ctx) == CAKE_TCTX_SIZE,
	       "cake_task_ctx size mismatch (64B release, 576B debug)");

/* packed_info bitfield layout (iter/debug transport field):
 * [Stable:2][Tier:2][Flags:4][KCR:1][BG:1][Rsvd:2][WCS:4][Rsvd:8]
 *  31-30     29-28   27-24    23     22    21-20  19-16  15-8
 * TIER+STABLE+BG are retained legacy fields.
 * KCR = cached critical kernel thread (set once in cake_init_task).
 * WCS = wake-chain locality score, capped to 4 bits.
 * The TUI no longer surfaces the legacy fields as live cake classes. */
#define SHIFT_FLAGS 24 /* 4 bits: flow flags */
#define SHIFT_TIER 28 /* 2 bits: tier 0-3 (coalesced with STABLE) */
#define SHIFT_STABLE 30 /* 2 bits: tier-stability counter (0-3) */
#define SHIFT_WAKE_CHAIN_SCORE 16 /* 4 bits: behavior locality score */
#define BIT_KCRITICAL \
	23 /* 1 bit: latency-critical kthread (set in init_task) */
#define BIT_BG_NOISE 22 /* 1 bit: reserved legacy flag */

/* Cached task properties in packed_info (bits 8-15) to avoid task_struct cache misses */
#define SHIFT_CACHED_SERVICE_KIND        8   /* bits 8-10 (3 bits) */
#define MASK_CACHED_SERVICE_KIND         0x07U
#define BIT_CACHED_DEFAULT_USER          11  /* bit 11 (1 bit) */
#define BIT_CACHED_FULL_AFFINITY         12  /* bit 12 (1 bit) */
#define BIT_CACHED_STATIC_LATENCY_BIASED 13  /* bit 13 (1 bit) */

#define MASK_TIER 0x03 /* 2 bits: 0-3 */
#define MASK_FLAGS 0x0F /* 4 bits */
#define MASK_WAKE_CHAIN_SCORE 0x0F /* 4 bits */

/* Legacy HOG detection threshold retained for debug/telemetry compatibility.
 * Current release code does not actively reclassify tasks with it. */

/* Statistics shared with userspace.
 * Some legacy counters remain in the layout for TUI compatibility
 * even when current release code no longer increments them. */
struct cake_game_diag {
	u64 nfw_entry; /* Native-fast-wake candidate entered */
	u64 nfw_hit; /* Native idle pick hit */
	u64 nfw_hit_prev_cpu; /* NFW hit kept the wakee on prev_cpu */
	u64 nfw_hit_other_cpu; /* NFW hit migrated away from prev_cpu */
	u64 nfw_hit_select_cpu; /* NFW hit targeted the select/waker CPU */
	u64 nfw_hit_prev_primary; /* NFW hit stayed on prev's primary core */
	u64 nfw_hit_other_primary; /* NFW hit moved to another primary core */
	u64 nfw_hit_game_thread; /* NFW hit for Unreal GameThread */
	u64 nfw_hit_render_thread; /* NFW hit for Unreal RenderThread */
	u64 nfw_hit_taskgraph_thread; /* NFW hit for TaskGraphThread workers */
	u64 nfw_hit_pool_thread; /* NFW hit for PoolThread workers */
	u64 nfw_hit_fpsaim_thread; /* NFW hit for FPSAimTrainer-W* threads */
	u64 nfw_hit_chrome_thread; /* NFW hit for Chromium/CEF helper threads */
	u64 nfw_hit_crgpu_thread; /* NFW hit for CEF CrGpuMain threads */
	u64 nfw_hit_dxvk_thread; /* NFW hit for dxvk worker threads */
	u64 nfw_hit_audio_thread; /* NFW hit for audio mixer/audio threads */
	u64 nfw_hit_other_thread; /* NFW hit for uncategorized comm */
	u64 nfw_hit_local_depth_sample; /* NFW hit sampled target local DSQ depth */
	u64 nfw_hit_local_depth_nonzero; /* NFW hit target local DSQ already had work */
	u64 nfw_hit_local_depth_gt1; /* NFW hit target local DSQ depth > 1 */
	u64 nfw_hit_local_depth_gt3; /* NFW hit target local DSQ depth > 3 */
	u64 nfw_prev_idle_attempt; /* NFW considered a cheap prev_cpu idle claim */
	u64 nfw_prev_idle_sibling_block; /* Prev claim blocked by SMT sibling gate */
	u64 nfw_prev_idle_claim; /* Prev claim won before native broad pick */
	u64 nfw_prev_idle_fallback_attempt; /* Broad native pick after prev path missed */
	u64 nfw_prev_idle_fallback_hit; /* Broad native fallback found an idle CPU */
	u64 nfw_prev_idle_fallback_prev; /* Fallback still picked prev_cpu */
	u64 nfw_prev_idle_fallback_other; /* Fallback picked another CPU */
	u64 nfw_miss; /* Native idle pick missed */
	u64 nfw_miss_shared; /* NFW miss routed to shared LLC/vtime */
	u64 nfw_miss_tunnel; /* NFW miss tunneled to prev CPU */
	u64 nfw_fallthrough; /* NFW miss fell through to Cake machinery */
	u64 nfw_direct_insert; /* NFW hit inserted directly from select_cpu */
	u64 select_tunnel; /* select_cpu returned prev_cpu tunnel */

	u64 enqueue_call; /* enqueue_body entered */
	u64 enqueue_wakeup; /* enqueue callback for SCX_ENQ_WAKEUP */
	u64 enqueue_initial; /* first runnable placement */
	u64 enqueue_requeue; /* SCX_ENQ_REENQ observed */
	u64 enqueue_preserve; /* SCX_ENQ_PREEMPT observed */
	u64 enqueue_non_wakeup; /* enqueue without wakeup flag */

	u64 enqueue_direct_local; /* direct local DSQ insert on lean path */
	u64 enqueue_wake_direct; /* wake inserted while target looked idle */
	u64 enqueue_wake_idle; /* wake sent SCX_KICK_IDLE */
	u64 enqueue_wake_busy; /* wake found busy target */
	u64 enqueue_wake_busy_local; /* enqueue CPU == target CPU on busy wake */
	u64 enqueue_wake_busy_remote; /* enqueue CPU != target CPU on busy wake */

	u64 wake_kick_idle; /* explicit idle kick */
	u64 wake_kick_preempt; /* explicit preempt kick */
	u64 kthread_direct_insert; /* kthread bypass local insert */
	u64 kthread_wake_preempt; /* kthread wake forced preempt */

	/* Frame-preemptor attribution (2026-06-10): GameThread eats ~425
	 * involuntary switches/s under cake vs ~54 under EEVDF and the source
	 * is NOT cake's wake kicks (disabling them made it worse). These
	 * counters name the thief: stopping(runnable) of a frame thread arms
	 * frame_victim_pid; the next running() on that CPU classifies who
	 * took over. by_self = the victim itself resumed, meaning a non-scx
	 * (RT/stop-class) task ran in the gap — scx never sees it. */
	u64 frame_stop_runnable; /* GameThread/RenderThread preempted while runnable */
	u64 frame_preempt_by_self; /* victim resumed next: RT/non-scx preemptor */
	u64 frame_preempt_by_kworker; /* kworker took the CPU */
	u64 frame_preempt_by_kthread; /* other kthread took the CPU */
	u64 frame_preempt_by_game; /* another frame-family thread took the CPU */
	u64 frame_preempt_by_user; /* other user thread took the CPU */

	u64 local_waiter_attempt; /* local-waiter considered */
	u64 local_waiter_insert; /* local-waiter inserted */
	u64 local_waiter_reject; /* local-waiter rejected */
	u64 local_waiter_quench; /* local-waiter shortened current slice */

	u64 shared_escape; /* guarded shared escape insert */
	u64 shared_vtime_insert; /* LLC/vtime insert */
	u64 dispatch_call; /* dispatch(): callback entered */
	u64 dispatch_idle_core_rescue_hit; /* dispatch(): idle core rescue found work */
	u64 dispatch_idle_llc_rescue_hit; /* dispatch(): idle LLC rescue found work */
	/* Watchdog-stall fix instrumentation (2026-06-17). llc_nonwake_insert =
	 * non-wakeup tail of cake_insert_llc_vtime reached (the path that used to
	 * place work on LLC_DSQ_BASE with no kick); llc_nonwake_kick_idle = of
	 * those, target was idle so the new SCX_KICK_IDLE fired. llc_rescue_enter
	 * = idle-LLC rescue ran its backstop past the queued-prev early-exit;
	 * llc_rescue_pending_lost_save = rescue drained a task the pending bit had
	 * lost (pending clear but DSQ non-empty) — the only metric that justifies
	 * keeping the re-armed rescue. */
	u64 llc_nonwake_insert;
	u64 llc_nonwake_kick_idle;
	u64 llc_rescue_enter;
	u64 llc_rescue_pending_lost_save;
	u64 dispatch_cache_hit; /* dispatch(): cache-simple/cache-sprint path hit */
	u64 dispatch_throughput_hit; /* dispatch(): throughput lane path hit */
	u64 dispatch_core_steal_hit; /* dispatch(): core-steal path hit */
	u64 dispatch_llc_pull_hit; /* dispatch(): shared LLC pull hit */
	u64 dispatch_keep_running; /* dispatch(): replenished prev queued task */
	u64 dispatch_idle; /* dispatch(): no work, published idle */
	u64 publish_idle_call; /* publish idle helper entered */
	u64 publish_idle_write; /* publish idle changed visible state */
	u64 publish_idle_noop; /* publish idle found already-idle state */
	u64 publish_running_call; /* publish running helper entered */
	u64 publish_running_write; /* publish running changed visible state */
	u64 publish_running_noop; /* publish running found same visible state */
	u64 publish_owner_call; /* publish owner helper entered */
	u64 publish_owner_write; /* publish owner changed visible state */
	u64 publish_owner_noop; /* publish owner found same visible state */
	u64 stopping_call; /* stopping(): callback entered */
	u64 stopping_runnable; /* stopping(): task remained runnable */
	u64 stopping_blocked; /* stopping(): task blocked / slept */
	u64 stopping_owner_update; /* stopping(): owner avg update executed */
	u64 stopping_route_observe; /* stopping(): route predictor observation executed */
	u64 stopping_route_pending; /* stopping(): route observe had pending pid */
	u64 stopping_route_no_pending; /* stopping(): route observe was early no-op */
	u64 stopping_account_relaxed; /* stopping(): confidence allowed scoreboard skip */
	u64 stopping_account_audit; /* stopping(): confidence forced scoreboard audit */
	u64 stopping_scoreboard_owner_result; /* stopping(): owner result updated confidence */
	u64 stopping_lean_return; /* stopping(): lean-accounting early return */
	u64 reserved0;
	u64 reserved1;
} __attribute__((aligned(256)));

struct cake_stats {
	u64 nr_prev_cpu_tunnels; /* select_cpu() fell back to returning prev_cpu */
	u64 nr_steer_eligible; /* select_cpu() saw a task eligible for warm steering */
	u64 nr_home_cpu_steers; /* select_cpu() placed a hot task back on its sticky home CPU */
	u64 nr_home_core_steers; /* select_cpu() kept a hot task on its sticky core */
	u64 nr_primary_cpu_steers; /* select_cpu() steered a hot task onto an idle primary SMT lane */
	u64 nr_home_cpu_busy_misses; /* home_cpu was valid but not idle */
	u64 nr_prev_primary_busy_misses; /* prev core primary lane rescue was valid but not idle */
	u64 nr_primary_scan_misses; /* primary-lane scan ran but found no idle candidate */
	u64 nr_primary_scan_guarded; /* primary-lane scan skipped for stable low-util sync wake chains */
	u64 nr_primary_scan_credit_used; /* guard-eligible primary scan allowed by periodic credit */
	u64 nr_primary_scan_hot_guarded; /* primary-lane scan skipped for hot same-TGID micro-workers */
	u64 nr_wake_chain_locality_guarded; /* wake-chain locality held placement on learned/previous CPU */
	u64 nr_wake_chain_locality_credit_used; /* wake-chain locality guard allowed a periodic wide probe */
	u64 nr_core_spread_attempt; /* select_cpu(): vtime/core route considered preferred full-core spread */
	u64 nr_core_spread_vtime_skip; /* select_cpu(): route skipped because task was too far ahead in vtime */
	u64 nr_core_spread_candidate; /* select_cpu(): checked a preferred full-core candidate */
	u64 nr_core_spread_full_idle_reject; /* select_cpu(): candidate CPU or SMT sibling was not clean idle */
	u64 nr_core_spread_hit; /* select_cpu(): preferred full-core spread claimed an idle primary */
	u64 nr_busy_handoff_dispatches; /* dispatch() later pulled DSQ work while a busy wakeup was pending */
	u64 nr_busy_keep_suppressed; /* dispatch() skipped keep_running because a busy wakeup was pending */
	u64 nr_wakeup_busy_local_target; /* Busy wakeup fallback where target CPU matched enqueue CPU */
	u64 nr_wakeup_busy_remote_target; /* Busy wakeup fallback where target CPU differed from enqueue CPU */
	u64 nr_tier_dispatches
		[CAKE_TIER_MAX]; /* Reserved legacy per-tier counters */
	u64 nr_starvation_preempts_tier
		[CAKE_TIER_MAX]; /* Reserved legacy per-tier counters */
	u64 total_gate1_latency_ns; /* Total time spent in Gate 1 */
	u64 total_gate2_latency_ns; /* Total time spent in Gate 2 */
	u64 total_enqueue_latency_ns; /* Total time spent in enqueue */
	u64 nr_dropped_allocations; /* Count of failed scx_task_alloc requests */
	u64 nr_local_dispatches; /* Dispatched from local LLC DSQ */
	u64 nr_stolen_dispatches; /* Dispatched from remote LLC DSQ (steal) */
	u64 nr_dispatch_misses; /* dispatch() found no residual shared-queue work */
	u64 nr_dispatch_hint_skip; /* dispatch() skipped kfunc via dsq_pending hint (counter==0) */
	u64 nr_direct_local_inserts; /* Total SCX_DSQ_LOCAL_ON inserts performed by cake */
	u64 nr_direct_affine_inserts; /* Direct local inserts used to keep restricted-affinity tasks off shared queues */
	u64 nr_direct_kthread_inserts; /* Direct inserts from the kthread bypass path */
	u64 nr_direct_other_inserts; /* Direct inserts that were neither wakeups nor kthread bypass */
	u64 nr_dsq_queued; /* Shared vtime DSQ enqueue count (for depth calc where used) */
	u64 nr_dsq_consumed; /* Shared DSQ consume count */
	u64 nr_shared_vtime_inserts; /* Total shared vtime DSQ inserts performed by cake */
	u64 nr_shared_wakeup_inserts; /* Shared vtime inserts from wakeups */
	u64 nr_shared_requeue_inserts; /* Shared vtime inserts from adaptive requeue path */
	u64 nr_shared_preserve_inserts; /* Shared vtime inserts preserving slice/deadline state */
	u64 nr_shared_other_inserts; /* Shared vtime inserts that were neither wakeup nor requeue/preserve */

	/* Callback aggregate timing (cumulative ns, system-wide) */
	u64 total_select_cpu_ns; /* Total time in cake_select_cpu */
	u64 total_stopping_ns; /* Total time in cake_stopping */
	u64 total_running_ns; /* Total time in cake_running */
	u64 task_runtime_ns; /* Total scheduled task runtime completed on this CPU */
	u64 task_run_count; /* Completed task run segments on this CPU */
	u64 smt_solo_runtime_ns; /* Runtime completed while sibling lane appeared idle */
	u64 smt_contended_runtime_ns; /* Runtime completed with observed sibling overlap */
	u64 smt_overlap_runtime_ns; /* Estimated sibling-overlap runtime charged to this CPU */
	u64 smt_solo_run_count; /* Run segments without observed sibling overlap */
	u64 smt_contended_run_count; /* Run segments with observed sibling overlap */
	u64 smt_sibling_active_start_count; /* Runs whose sibling was active at start */
	u64 smt_sibling_active_stop_count; /* Runs whose sibling was active at stop */
	u64 smt_wake_wait_ns
		[2]; /* Wake wait by sibling-active-at-run-start bucket */
	u64 smt_wake_wait_count[2]; /* Wake wait samples by SMT bucket */
	u64 smt_wake_wait_max_ns[2]; /* Worst wake wait by SMT bucket */

	/* Callback max tracking (worst single invocation, ns) */
	u64 max_select_cpu_ns; /* Worst single cake_select_cpu */
	u64 max_stopping_ns; /* Worst single cake_stopping */
	u64 max_running_ns; /* Worst single cake_running */

	/* Stopping path breakdown (invocation counts) */
	u64 nr_stop_deferred_skip; /* Stops that skipped full task telemetry */
	u64 nr_stop_deferred; /* Stops that recorded full task telemetry */
	u64 nr_stop_ramp; /* Reserved legacy counter */
	u64 nr_stop_miss; /* Reserved legacy counter */

	/* Dispatch callback timing */
	u64 total_dispatch_ns; /* Total time in cake_dispatch */
	u64 max_dispatch_ns; /* Worst single cake_dispatch */
	u64 nr_select_cpu_calls; /* Total cake_select_cpu invocations */
	u64 nr_enqueue_calls; /* Total cake_enqueue invocations */
	u64 nr_dispatch_calls; /* Total cake_dispatch invocations */
	u64 nr_running_calls; /* Total cake_running invocations */
	u64 nr_stopping_calls; /* Total cake_stopping invocations */
	u64 nr_running_same_task; /* running(): last_pid matched current task */
	u64 nr_running_task_change; /* running(): observed a new task on this CPU */
	u64 nr_stopping_runnable; /* stopping(): task remained runnable */
	u64 nr_stopping_blocked; /* stopping(): task blocked / slept */
	u64 nr_enqueue_path_kthread; /* enqueue(): high-priority kthread bypass */
	u64 nr_enqueue_path_initial; /* enqueue(): first vtime seed path */
	u64 nr_enqueue_path_preserve; /* enqueue(): preserve slice/vtime state */
	u64 nr_enqueue_path_requeue; /* enqueue(): non-wakeup requeue path */
	u64 nr_enqueue_path_wakeup; /* enqueue(): normal wakeup path */
	u64 nr_enqueue_path_affine_preserve; /* enqueue(): affinity-restricted preserve */
	u64 nr_enqueue_path_affine_requeue; /* enqueue(): affinity-restricted requeue */
	u64 nr_enqueue_path_affine_dispatch; /* enqueue(): affinity-restricted dispatch */
	u64 nr_llc_vtime_wake_idle_direct; /* llc-vtime: wake found target idle, local insert */
	u64 nr_llc_vtime_wake_busy_shared; /* llc-vtime: wake target busy, shared insert + kick */
	u64 nr_llc_vtime_nonwake_shared; /* llc-vtime: non-wakeup shared insert */
	u64 nr_dispatch_llc_local_hit; /* dispatch(): local LLC DSQ had work */
	u64 nr_dispatch_llc_local_miss; /* dispatch(): local LLC DSQ was empty */
	u64 nr_dispatch_llc_steal_hit; /* dispatch(): remote LLC steal succeeded */
	u64 nr_dispatch_keep_running; /* dispatch(): replenished prev queued task */
	u64 nr_cache_throughput_requeue; /* Requeued cache/mem-sat work with throughput slice */
	u64 nr_cache_throughput_keep_running; /* keep_running used throughput slice */
	u64 nr_cache_throughput_lane_insert; /* Enqueued into per-CPU throughput lane */
	u64 nr_cache_throughput_lane_spill; /* Throughput lane busy, spilled to LLC DSQ */
	u64 nr_cache_throughput_lane_local_hit; /* Dispatch pulled own throughput lane */
	u64 nr_cache_throughput_lane_steal_hit; /* Dispatch stole fast-probe throughput lane */
	u64 nr_cache_throughput_lane_stale; /* Pending lane had no movable task */
	u64 nr_initial_shared_escape; /* Initial local-policy task escaped to shared LLC queue */
	u64 nr_busy_wake_slice_shrink; /* Busy wake clamped the current task slice */
	u64 nr_busy_wake_shared_escape; /* Busy wake escaped local target to shared LLC queue */
	u64 nr_local_waiter_attempt; /* local-waiter: wake tried owner-local service contract */
	u64 nr_local_waiter_reject; /* local-waiter: admission rejected before insert */
	u64 nr_local_waiter_insert; /* local-waiter: wake inserted at target local DSQ head */
	u64 nr_local_waiter_quench_current; /* local-waiter: current owner slice was quenched */
	u64 nr_local_waiter_debt_seen; /* running(): saw outstanding local-waiter debt */
	u64 nr_local_waiter_debt_consume; /* running(): next task consumed waiter debt */
	u64 nr_local_waiter_same_task_quench; /* running(): same owner was slice-quenched */
	u64 nr_domain_drr_cache_insert; /* domain-drr: cache-class task inserted */
	u64 nr_domain_drr_stream_insert; /* domain-drr: stream-class task inserted */
	u64 nr_domain_drr_cache_pull; /* domain-drr: dispatch pulled cache-class task */
	u64 nr_domain_drr_stream_pull; /* domain-drr: dispatch pulled stream-class task */
	u64 nr_domain_drr_stale; /* domain-drr: pending hint was stale */
	u64 nr_domain_drr_stream_due; /* domain-drr: stream service cut cache burst */

	/* Task lifecycle timing (cumulative us, debug builds) */
	u64 lifecycle_init_enqueue_us; /* Sum of task init to first enqueue */
	u64 lifecycle_init_enqueue_count; /* Samples for init->enqueue */
	u64 lifecycle_init_select_us; /* Sum of task init to first select_cpu */
	u64 lifecycle_init_select_count; /* Samples for init->select_cpu */
	u64 lifecycle_init_run_us; /* Sum of task init to first running */
	u64 lifecycle_init_run_count; /* Samples for init->run */
	u64 lifecycle_init_exit_us; /* Sum of task init to exit_task */
	u64 lifecycle_init_exit_count; /* Samples for init->exit */

	/* Wakeup enqueue gate counters */
	u64 nr_wakeup_direct_dispatches; /* Wakeups that direct-dispatched to SCX_DSQ_LOCAL_ON */
	u64 nr_wakeup_dsq_fallback_busy; /* Wakeups that missed direct dispatch because target CPU did not look idle */
	u64 nr_wakeup_dsq_fallback_queued; /* Wakeups that missed direct dispatch because the shared DSQ was already non-empty */
	u64 callback_hist
		[CAKE_CB_MAX]
		[CAKE_CB_BUCKET_MAX]; /* Callback duration histogram buckets */
	u64 callback_slow[CAKE_CB_MAX]; /* Threshold breaches per callback */
	u64 callback_release_est_ns
		[CAKE_CB_MAX]; /* Stopwatch-estimated release-shaped work */
	u64 callback_debug_tax_ns
		[CAKE_CB_MAX]; /* Stopwatch-estimated debug-only work */
	u64 callback_release_est_max_ns
		[CAKE_CB_MAX]; /* Worst release-shaped stopwatch sample */
	u64 callback_debug_tax_max_ns
		[CAKE_CB_MAX]; /* Worst debug-only stopwatch sample */
	u64 callback_split_count
		[CAKE_CB_MAX]; /* Samples with release/debug split accounting */
	u64 wake_reason_wait_all_ns
		[CAKE_WAKE_REASON_MAX]; /* Full-pop wake-to-run wait sums by wake path (0 unused) */
	u64 wake_reason_wait_all_count
		[CAKE_WAKE_REASON_MAX]; /* Full-pop wake-to-run samples by wake path (0 unused) */
	u64 wake_reason_wait_all_max_ns
		[CAKE_WAKE_REASON_MAX]; /* Full-pop worst wait by wake path (0 unused) */
	u64 wake_reason_wait_ns
		[CAKE_WAKE_REASON_MAX]; /* Wait sums by wake path (0 unused) */
	u64 wake_reason_wait_count
		[CAKE_WAKE_REASON_MAX]; /* Wait samples by wake path (0 unused) */
	u64 wake_reason_wait_max_ns
		[CAKE_WAKE_REASON_MAX]; /* Worst wait by wake path (0 unused) */
	u64 wake_reason_bucket_count
		[CAKE_WAKE_REASON_MAX]
		[CAKE_WAKE_BUCKET_MAX]; /* Wake-to-run buckets by wake path (0 unused) */
	u64 select_path_count
		[CAKE_SELECT_PATH_MAX]; /* Path chosen in select_cpu (0 unused) */
	u64 select_path_migration_count
		[CAKE_SELECT_PATH_MAX]; /* Debug: select_cpu choices that changed CPU by path */
	u64 select_reason_migration_count
		[CAKE_SELECT_REASON_MAX]; /* Debug: select_cpu choices that changed CPU by reason */
	u64 select_reason_wait_ns
		[CAKE_SELECT_REASON_MAX]; /* Wake-to-run wait after each select_cpu decision reason */
	u64 select_reason_wait_count
		[CAKE_SELECT_REASON_MAX]; /* Wait samples after each select_cpu decision reason */
	u64 select_reason_wait_max_ns
		[CAKE_SELECT_REASON_MAX]; /* Worst wait after each select_cpu decision reason */
	u64 select_reason_bucket_count
		[CAKE_SELECT_REASON_MAX]
		[CAKE_WAKE_BUCKET_MAX]; /* Wait buckets after each select_cpu decision reason */
	u64 select_reason_select_ns
		[CAKE_SELECT_REASON_MAX]; /* select_cpu overhead by decision reason */
	u64 select_reason_select_count
		[CAKE_SELECT_REASON_MAX]; /* select_cpu samples by decision reason */
	u64 select_reason_select_max_ns
		[CAKE_SELECT_REASON_MAX]; /* Worst select_cpu overhead by decision reason */
	u64 accel_route_attempt_count
		[CAKE_ACCEL_ROUTE_MAX]; /* Route-token prediction claim attempts by route */
	u64 accel_route_hit_count
		[CAKE_ACCEL_ROUTE_MAX]; /* Route-token prediction claim hits by route */
	u64 accel_route_miss_count
		[CAKE_ACCEL_ROUTE_MAX]; /* Route-token prediction claim misses by route */
	u64 accel_fast_attempt_count
		[CAKE_ACCEL_ROUTE_MAX]; /* Scoreboard fast-scan claim attempts by route slot */
	u64 accel_fast_hit_count
		[CAKE_ACCEL_ROUTE_MAX]; /* Scoreboard fast-scan claim hits by route slot */
	u64 accel_fast_miss_count
		[CAKE_ACCEL_ROUTE_MAX]; /* Scoreboard fast-scan claim misses by route slot */
	u64 accel_route_block_count
		[CAKE_ACCEL_ROUTE_BLOCK_MAX]; /* Why route prediction could not use the floor path */
	u64 accel_scoreboard_probe_count
		[CAKE_ACCEL_ROUTE_MAX]
		[CAKE_ACCEL_PROBE_OUTCOME_MAX]; /* Scoreboard probe outcomes by route slot */
	u64 accel_pull_mode_count
		[CAKE_ACCEL_PULL_MODE_MAX]; /* Dispatch pull shaping decisions */
	u64 accel_pull_probe_count
		[CAKE_ACCEL_PULL_PROBE_MAX]; /* Dispatch queue probes that were empty or had work */
	u64 accel_native_fallback_count
		[CAKE_ACCEL_NATIVE_MAX]; /* Native helper fallback entry and ABI branch calls */
	u64 accel_accounting_relaxed; /* Stopping callbacks that skipped owner-scoreboard accounting */
	u64 accel_accounting_audit; /* Stopping callbacks that audited owner-scoreboard accounting */
	u64 accel_trust_prev_attempt; /* Trusted userspace-governed direct prev claims */
	u64 accel_trust_prev_hit; /* Trusted direct prev claims that got the CPU */
	u64 accel_trust_prev_miss; /* Trusted direct prev claims that self-demoted */
	u64 frontier_pending_set; /* Frontier token marked pending after a shortcut succeeded */
	u64 frontier_observe_count; /* Pending token observations consumed in stopping */
	u64 frontier_observe_good; /* Observations that promoted cache-hot confidence */
	u64 frontier_observe_bad; /* Observations that decayed or rejected confidence */
	u64 frontier_conf_promote; /* Observation raised confidence */
	u64 frontier_conf_decay; /* Observation or failed trusted action reduced confidence */
	u64 frontier_conf_high; /* Observation reached high confidence */
	u64 frontier_mode_count
		[CAKE_FRONTIER_MODE_MAX]; /* Post-observe predictor mode distribution */
	u64 frontier_dispatch_trusted_attempt; /* Dispatch tried a trusted cache-hot shortcut */
	u64 frontier_dispatch_trusted_hit; /* Trusted dispatch keep-running succeeded */
	u64 frontier_dispatch_trusted_fail; /* Trusted dispatch keep-running failed and decayed */
	u64 frontier_dispatch_audit_skip; /* Trusted dispatch skipped fairness due to audit cadence */
	u64 frontier_dispatch_audit_due; /* Trusted dispatch hit audit cadence and asked fairness */
	u64 frontier_select_prev_attempt; /* Select path evaluated token-approved prev CPU */
	u64 frontier_select_prev_hit; /* Token-approved prev CPU claim succeeded */
	u64 frontier_select_prev_miss; /* Token-approved prev CPU claim failed */
	u64 frontier_enqueue_fact_attempt; /* Enqueue path tried token fact reuse */
	u64 frontier_enqueue_fact_hit; /* Enqueue token facts avoided affinity re-check */
	u64 frontier_enqueue_fact_miss; /* Enqueue token facts were unavailable or stale */
	u64 frontier_token_clear; /* running() cleared stale CPU token on task change */
	u64 home_place_wait_ns
		[CAKE_PLACE_CLASS_MAX]; /* Wait sums by task-home locality */
	u64 home_place_wait_count
		[CAKE_PLACE_CLASS_MAX]; /* Wait samples by task-home locality */
	u64 home_place_wait_max_ns
		[CAKE_PLACE_CLASS_MAX]; /* Worst wait by task-home locality */
	u64 home_place_run_ns
		[CAKE_PLACE_CLASS_MAX]; /* First-run sums by task-home locality */
	u64 home_place_run_count
		[CAKE_PLACE_CLASS_MAX]; /* First-run samples by task-home locality */
	u64 home_place_run_max_ns
		[CAKE_PLACE_CLASS_MAX]; /* Worst first-run by task-home locality */
	u64 waker_place_wait_ns
		[CAKE_PLACE_CLASS_MAX]; /* Wait sums by waker locality */
	u64 waker_place_wait_count
		[CAKE_PLACE_CLASS_MAX]; /* Wait samples by waker locality */
	u64 waker_place_wait_max_ns
		[CAKE_PLACE_CLASS_MAX]; /* Worst wait by waker locality */
	u64 nr_wake_same_tgid; /* Wakes from the same TGID */
	u64 nr_wake_cross_tgid; /* Wakes from a different TGID */
	u64 nr_idle_hint_remote_reads; /* Cross-CPU idle_hint reads */
	u64 nr_idle_hint_remote_busy; /* Remote idle_hint reads observing busy */
	u64 nr_idle_hint_remote_idle; /* Remote idle_hint reads observing idle */
	u64 nr_busy_pending_remote_sets; /* Cross-CPU busy_wakeup_pending writes for shared-fallback mode */
	u64 nr_enqueue_requeue_fastpath; /* Non-wakeup requeues that bypassed direct-handoff logic */
	u64 nr_enqueue_busy_local_skip_depth; /* Same-CPU busy wakeups that skipped DSQ depth query */
	u64 nr_enqueue_busy_remote_skip_depth; /* Remote busy wakeups that skipped DSQ depth query */
	u64 nr_busy_pending_set_skips; /* busy_wakeup_pending already 1 in shared-fallback mode */
	u64 nr_idle_hint_set_writes; /* idle_hint 0->1 transitions */
	u64 nr_idle_hint_clear_writes; /* idle_hint 1->0 transitions */
	u64 nr_idle_hint_set_skips; /* idle_hint already 1 */
	u64 nr_idle_hint_clear_skips; /* idle_hint already 0 */
	u64 nr_wake_kick_idle; /* Wake-driven kicks that used SCX_KICK_IDLE */
	u64 nr_wake_kick_preempt; /* Wake-driven kicks that used SCX_KICK_PREEMPT */
	u64 nr_ready_eval; /* Debug readiness classifications at wake enqueue */
	u64 nr_ready_urgent; /* Readiness samples with urgent-service bit set */
	u64 nr_ready_promote; /* Readiness action would promote/run wakee */
	u64 nr_ready_defer; /* Readiness action would keep owner / defer wakee */
	u64 nr_ready_defer_fairness; /* Deferred because fairness/debt bit failed */
	u64 nr_ready_defer_owner; /* Deferred because owner-preempt bit failed */
	u64 nr_ready_defer_route; /* Deferred because route/custody was unknown */
	u64 nr_ready_defer_cache; /* Deferred because cache-damage bit failed */
	u64 nr_ready_preempt; /* Readiness action selected preempt kick */
	u64 nr_ready_idle_kick; /* Readiness action selected idle kick */
	u64 nr_ready_local_only; /* Readiness action selected local insert/no kick */
	u64 nr_ready_promote_wait_lt100us; /* Promoted wake observed under 100us */
	u64 nr_ready_promote_wait_ge1ms; /* Promoted wake observed at/above 1ms */
	u64 nr_ready_defer_wait_lt100us; /* Deferred wake observed under 100us */
	u64 nr_ready_defer_wait_ge1ms; /* Deferred wake observed at/above 1ms */
	u64 wake_class_sample_count
		[CAKE_WAKE_CLASS_MAX]; /* Shadow wake class samples */
	u64 wake_class_reason_count
		[CAKE_WAKE_CLASS_REASON_MAX]; /* Shadow class reason hits */
	u64 wake_class_transition_count
		[CAKE_WAKE_CLASS_MAX]
		[CAKE_WAKE_CLASS_MAX]; /* Per-CPU owner class transitions */
	u64 busy_preempt_shadow_count
		[CAKE_BUSY_PREEMPT_SHADOW_MAX]; /* Shadow busy-wake decision counts */
	u64 busy_preempt_shadow_wakee_class_count
		[CAKE_WAKE_CLASS_MAX]; /* Busy shadow by wakee class */
	u64 busy_preempt_shadow_owner_class_count
		[CAKE_WAKE_CLASS_MAX]; /* Busy shadow by owner class */
	u64 busy_preempt_shadow_local; /* Busy shadow decisions where waker CPU matched target CPU */
	u64 busy_preempt_shadow_remote; /* Busy shadow decisions where target CPU was remote */
	u64 strict_wake_class_sample_count
		[CAKE_WAKE_CLASS_MAX]; /* Stricter shadow wake class samples */
	u64 strict_wake_class_reason_count
		[CAKE_WAKE_CLASS_REASON_MAX]; /* Strict class reason hits */
	u64 strict_wake_class_transition_count
		[CAKE_WAKE_CLASS_MAX]
		[CAKE_WAKE_CLASS_MAX]; /* Strict owner transitions */
	u64 strict_busy_preempt_shadow_count
		[CAKE_BUSY_PREEMPT_SHADOW_MAX]; /* Strict busy-wake decision counts */
	u64 strict_busy_preempt_shadow_wakee_class_count
		[CAKE_WAKE_CLASS_MAX]; /* Strict busy shadow by wakee class */
	u64 strict_busy_preempt_shadow_owner_class_count
		[CAKE_WAKE_CLASS_MAX]; /* Strict busy shadow by owner class */
	u64 strict_busy_preempt_shadow_local; /* Strict busy shadow decisions where waker CPU matched target CPU */
	u64 strict_busy_preempt_shadow_remote; /* Strict busy shadow decisions where target CPU was remote */
	u64 strict_wake_class_wait_ns
		[CAKE_WAKE_CLASS_MAX]; /* Strict wake-class wait sums */
	u64 strict_wake_class_wait_count
		[CAKE_WAKE_CLASS_MAX]; /* Strict wake-class wait samples */
	u64 strict_wake_class_wait_max_ns
		[CAKE_WAKE_CLASS_MAX]; /* Strict wake-class worst wait */
	u64 strict_wake_class_bucket_count
		[CAKE_WAKE_CLASS_MAX]
		[CAKE_WAKE_BUCKET_MAX]; /* Strict wake wait buckets by class */
	u64 storm_guard_mode_count
		[CAKE_STORM_GUARD_MAX]; /* Runtime storm-guard mode samples */
	u64 storm_guard_decision_count
		[CAKE_STORM_GUARD_DECISION_MAX]; /* Storm-guard busy-wake decisions */
	u64 nr_affine_kick_idle; /* Affinity-change kicks that used SCX_KICK_IDLE */
	u64 nr_affine_kick_preempt; /* Affinity-change kicks that used SCX_KICK_PREEMPT */
	u64 nr_quantum_full; /* Stops that consumed the full slice */
	u64 nr_quantum_yield; /* Stops that kept slice left but became non-runnable */
	u64 nr_quantum_preempt; /* Stops that were preempted while still runnable */
	u64 nr_sched_yield_calls; /* Explicit sched_yield callbacks observed by cake_yield() */
	u64 wake_target_hit_count
		[CAKE_WAKE_REASON_MAX]; /* Post-wake subset: first run landed on the selected CPU */
	u64 wake_target_miss_count
		[CAKE_WAKE_REASON_MAX]; /* Post-wake subset: first run landed on a different CPU */
	u64 wake_followup_same_cpu_count
		[CAKE_WAKE_REASON_MAX]; /* Post-wake subset: first continuation stayed on the same CPU */
	u64 wake_followup_migrate_count
		[CAKE_WAKE_REASON_MAX]; /* Post-wake subset: first continuation migrated away */
	u64 nr_wake_kick_observed
		[CAKE_KICK_KIND_MAX]; /* Wake kicks followed by an observed run */
	u64 nr_wake_kick_quick
		[CAKE_KICK_KIND_MAX]; /* Wake kicks that led to a run within 200us */
	u64 total_wake_kick_to_run_ns
		[CAKE_KICK_KIND_MAX]; /* Kick-to-run total latency by kick kind */
	u64 max_wake_kick_to_run_ns
		[CAKE_KICK_KIND_MAX]; /* Worst kick-to-run latency by kick kind */
	u64 wake_kick_bucket_count
		[CAKE_KICK_KIND_MAX]
		[CAKE_WAKE_BUCKET_MAX]; /* Kick-to-run buckets by kick kind */
} __attribute__((aligned(64)));

/* Current default values */
#define CAKE_DEFAULT_QUANTUM_NS (2 * 1000 * 1000) /* 2ms */
#define CAKE_DEFAULT_NEW_FLOW_BONUS_NS \
	(3 * 1000 * 1000) /* Reserved legacy constant */

/* Legacy adaptive-quantum constants retained for compatibility with older
 * debug paths. Current release scheduling does not drive policy from these
 * values. */
#define AQ_BULK_HEADROOM 1 /* 1× PELT runtime for non-yielders */
#define AQ_MIN_NS (50 * 1000) /* 50µs legacy floor */
#define AQ_YIELDER_CEILING_NS (50 * 1000000) /* 50ms legacy yielder ceiling */
#define AQ_BULK_CEILING_NS (2 * 1000000) /* 2ms legacy bulk ceiling */
#define AQ_BULK_CEILING_COMPILE_NS \
	(8 * 1000000) /* 8ms legacy compile ceiling */

/* Legacy preemption thresholds retained for compatibility. */
#define CAKE_PREEMPT_YIELDER_THRESHOLD_NS \
	(100 * 1000) /* 100µs legacy threshold */
#define CAKE_PREEMPT_VIP_THRESHOLD_NS (50 * 1000) /*  50µs — reserved legacy */

#endif /* __CAKE_INTF_H */
