diff --git a/python/sglang/srt/managers/scheduler_runtime_checker_mixin.py b/python/sglang/srt/managers/scheduler_runtime_checker_mixin.py index ef8e1f3fd..d35efdb53 100644 --- a/python/sglang/srt/managers/scheduler_runtime_checker_mixin.py +++ b/python/sglang/srt/managers/scheduler_runtime_checker_mixin.py @@ -128,6 +128,9 @@ class PoolStats: stats.swa_token_usage = self.swa_token_usage if self.is_hybrid_ssm: stats.mamba_usage = self.mamba_usage + stats.kv_available_tokens = self.full_available_size + stats.kv_evictable_tokens = self.full_evictable_size + stats.kv_used_tokens = self.full_num_used class SchedulerRuntimeCheckerMixin: diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 3c3430216..f02ac70ee 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -94,6 +94,9 @@ class SchedulerStats: cache_hit_rate: float = 0.0 max_total_num_tokens: int = 0 + kv_available_tokens: int = 0 + kv_evictable_tokens: int = 0 + kv_used_tokens: int = 0 # Speculative decoding spec_accept_length: float = 0.0 @@ -279,6 +282,25 @@ class SchedulerMetricsCollector: multiprocess_mode="mostrecent", ) + self.kv_available_tokens = Gauge( + name="sglang:kv_available_tokens", + documentation="Number of free token slots in the KV cache pool.", + labelnames=labels.keys(), + multiprocess_mode="mostrecent", + ) + self.kv_evictable_tokens = Gauge( + name="sglang:kv_evictable_tokens", + documentation="Number of evictable (radix-cached) token slots in the KV cache pool.", + labelnames=labels.keys(), + multiprocess_mode="mostrecent", + ) + self.kv_used_tokens = Gauge( + name="sglang:kv_used_tokens", + documentation="Number of actively used token slots in the KV cache pool.", + labelnames=labels.keys(), + multiprocess_mode="mostrecent", + ) + # Speculative decoding self.spec_accept_length = Gauge( name="sglang:spec_accept_length", @@ -1014,6 +1036,9 @@ class SchedulerMetricsCollector: self._log_gauge(self.cache_hit_rate, stats.cache_hit_rate) self._log_gauge(self.max_total_num_tokens, stats.max_total_num_tokens) + self._log_gauge(self.kv_available_tokens, stats.kv_available_tokens) + self._log_gauge(self.kv_evictable_tokens, stats.kv_evictable_tokens) + self._log_gauge(self.kv_used_tokens, stats.kv_used_tokens) # Speculative decoding self._log_gauge(self.spec_accept_length, stats.spec_accept_length)