{"resourceId":"nvidia-nim-operator-hpa-inference-metrics-2012","versions":[{"version":"external-7c313db30ef4d7c733c30cd6f948ff8afb8c4b82bb4e4172a242b3fd9a0777fb","resource":{"id":"nvidia-nim-operator-hpa-inference-metrics-2012","title":"NIM Operator guidance requires inference metrics for HPA and long model-load startup probes","organization":"NVIDIA","sector":"AI platform vendor documentation","geography":"Global","publishedAt":"NIM LLM 2.0.12 operator deployment docs (retrieved 2026-09-15)","publicationDate":null,"eventDate":null,"sourceName":"NVIDIA NIM Operator Deployment documentation","sourceLabel":"Vendor standards / deployment guidance","sourceUrl":"https://docs.nvidia.com/nim/large-language-models/2.0.12/deployment/kubernetes-deployment/nim-operator-deployment.html","evidenceClass":"standards-guidance","outcomeClass":"cautionary","topics":["infrastructure","developers-agents","operating-model"],"finding":"Official NIM Operator deployment guidance for 2.0.12 centers on NIMCache + NIMService, requires GPU Operator, PVC caching, and NGC secrets, and states CPU/memory metrics are not useful for HPA—use backend-native inference metrics (e.g., vllm:num_requests_waiting) via Prometheus Adapter. Startup probes allow up to ~20 minutes by default for model load; multi-node spec.args are not propagated on the LeaderWorkerSet path.","sledRelevance":"Campus and agency Kubernetes platforms cannot reuse generic HPA patterns for NIM; observability and probe budgets become acceptance criteria for production AI services.","evidence":"Opened docs detail prerequisites (K8s 1.26+, GPU Operator, storage, NGC key, Operator 3.0.2+), autoscaling example, health endpoint semantics (live vs ready), and troubleshooting for cache jobs and probe failures.","architectureImplications":"Install Prometheus Adapter and scrape /v1/metrics; size PVCs for model caches; tune startupProbe failureThreshold for large models; avoid multi-node args assumptions.","governanceImplications":"Platform standards should mandate inference-metric HPA and explicit model-cache readiness before exposing student/citizen-facing endpoints.","securityPrivacyImplications":"NGC API keys and pull secrets are cluster secrets; namespace isolation and secret rotation are mandatory.","caveats":"Documentation guidance, not a measured multi-tenant SLED deployment. Metric names differ between vLLM and SGLang backends.","streamIds":["nvidia"],"roles":{"sales":"Qualify Kubernetes maturity (GPU Operator, Prometheus custom metrics, storage classes) before promising elastic NIM services.","engineering":"Implement NIMCache/NIMService with inference-metric HPA and appropriate startup probes; verify backend metric names.","delivery":"Include cache-ready gates, HPA soak tests, and secret-handling checks in go-live criteria."},"roleDetails":{"sales":{"customerProblem":"Agencies expect autoscaling chat/agents like web apps using CPU HPA.","stakeholders":["platform SRE","CISO","app owner"],"discoveryQuestions":["Is Prometheus Adapter available?","What storage class supports model caches?","How long may models take to become ready?"],"valueHypothesis":"Inference-aware platform packaging prevents failed rollouts and noisy-neighbor outages.","potentialEngagement":"Platform readiness assessment against Operator prerequisites.","unsupportedClaims":["CPU HPA is sufficient for NIM","Models are ready in seconds regardless of size"]},"engineering":{"fit":"Fit for Kubernetes NIM when GPU Operator, PVCs, and custom metrics exist.","architectureIntegration":"Deploy Operator 3.0.2+ with NIMCache then NIMService; wire ServiceMonitor and HPA to waiting-request metrics.","prerequisites":["K8s 1.26+","GPU Operator","storage provisioner","NGC secrets","Prometheus Adapter"],"constraints":["CPU/mem HPA ineffective","Multi-node args not passed via LeaderWorkerSet","Long model loads need probe budget"],"security":"Protect NGC credentials; isolate nim-service namespaces.","proofOfValue":["Cache Ready before service","HPA scales on inference metric","Startup probe survives model load"]},"delivery":{"work":"Cluster prep, Operator install, sample-to-prod promotion, and observability validation.","dependencies":["platform team","secrets management","app team"],"ownership":"Platform owns Operator; apps own NIMService specs.","skillsAdoption":"SREs need NIM metrics literacy.","governanceCheckpoints":["Prereq checklist","Cache job success","HPA load test"],"acceptanceCriteria":["Service ready only after cache Ready","Autoscaling observed under load","Secrets rotated"],"risks":["Missing custom metrics","Undersized PVC","Probe kills during load"]}},"retrievedAt":"2026-09-15T14:15:00Z","enrichedAt":"2026-09-15T14:17:00Z","enrichmentBasis":"retrieved source","accessibilityWorkforceImplications":"Stable scaling protects shared campus assistants used by large student/staff populations.","procurementImplications":"Include platform observability and storage prerequisites in NIM/AI Enterprise SOWs.","operatingModelImplications":"Define who owns cache PVCs, HPA policies, and NGC secret rotation.","sourceVerification":{"openedUrl":"https://docs.nvidia.com/nim/large-language-models/2.0.12/deployment/kubernetes-deployment/nim-operator-deployment.html","referenceExcerpt":"Standard CPU and memory metrics are not useful for scaling NIM.","promptVersion":"sled-research-v4.0","model":null,"basis":"agent-reported inspection"}}}]}