kvcache-ai/sglang
1
0
Fork 0
mirror of https://github.com/kvcache-ai/sglang.git synced 2026-07-13 10:56:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(metrics): expose raw KV cache pool token counts as prometheus gauges (#22726)

Browse Source
This commit is contained in:
ishandhanani
2026-04-13 20:30:36 -05:00
committed by GitHub
parent 945d73824f
commit cc449ac4e5
2 changed files with 28 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
python/sglang/srt/managers/scheduler_runtime_checker_mixin.py
View File

@@ -128,6 +128,9 @@ class PoolStats:
stats.swa_token_usage = self.swa_token_usage
if self.is_hybrid_ssm:
stats.mamba_usage = self.mamba_usage
stats.kv_available_tokens = self.full_available_size
stats.kv_evictable_tokens = self.full_evictable_size
stats.kv_used_tokens = self.full_num_used
class SchedulerRuntimeCheckerMixin:

25
python/sglang/srt/observability/metrics_collector.py
View File

@@ -94,6 +94,9 @@ class SchedulerStats:
cache_hit_rate: float = 0.0
max_total_num_tokens: int = 0
kv_available_tokens: int = 0
kv_evictable_tokens: int = 0
kv_used_tokens: int = 0
# Speculative decoding
spec_accept_length: float = 0.0
@@ -279,6 +282,25 @@ class SchedulerMetricsCollector:
multiprocess_mode="mostrecent",
)
self.kv_available_tokens = Gauge(
name="sglang:kv_available_tokens",
documentation="Number of free token slots in the KV cache pool.",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.kv_evictable_tokens = Gauge(
name="sglang:kv_evictable_tokens",
documentation="Number of evictable (radix-cached) token slots in the KV cache pool.",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.kv_used_tokens = Gauge(
name="sglang:kv_used_tokens",
documentation="Number of actively used token slots in the KV cache pool.",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
# Speculative decoding
self.spec_accept_length = Gauge(
name="sglang:spec_accept_length",
@@ -1014,6 +1036,9 @@ class SchedulerMetricsCollector:
self._log_gauge(self.cache_hit_rate, stats.cache_hit_rate)
self._log_gauge(self.max_total_num_tokens, stats.max_total_num_tokens)
self._log_gauge(self.kv_available_tokens, stats.kv_available_tokens)
self._log_gauge(self.kv_evictable_tokens, stats.kv_evictable_tokens)
self._log_gauge(self.kv_used_tokens, stats.kv_used_tokens)
# Speculative decoding
self._log_gauge(self.spec_accept_length, stats.spec_accept_length)