GenerationStats

prompt_tpsPrompt Tps (number)required

generation_tpsGeneration Tps (number)required

prompt_tokensPrompt Tokens (integer)required

generation_tokensGeneration Tokens (integer)required

peak_memory_usage objectrequired

inBytesInbytes (integer)

Default value: 0

serving_node object

anyOf

• string
• null

string

serving_backend object

anyOf

• string
• null

string

in_flight_at_admission object

anyOf

• integer
• null

integer

serving_batches object

anyOf

• boolean
• null

boolean

GenerationStats

{
  "prompt_tps": 0,
  "generation_tps": 0,
  "prompt_tokens": 0,
  "generation_tokens": 0,
  "peak_memory_usage": {
    "inBytes": 0
  },
  "serving_node": "string",
  "serving_backend": "string",
  "in_flight_at_admission": 0,
  "serving_batches": true
}