ClusterDiagnostics
Read-only diagnostic bundle collected from reachable cluster nodes.
UTC timestamp when collection finished.
Node ID of the API serving this response.
masterNodeId object
Current master node ID, when known.
- string
- null
nodes object[]
Local and reachable peer diagnostic results.
Node ID for this cluster diagnostics result.
url object
Peer API URL used to collect diagnostics, if remote.
- string
- null
Whether diagnostics were collected successfully.
diagnostics object
Collected node diagnostics when ok is true.
- NodeDiagnostics
- null
UTC timestamp when this bundle was built.
runtime objectrequired
Runtime identity and config.
Local node ID.
Local hostname.
friendlyName object
Friendly node name from gathered identity data.
- string
- null
Whether this node is the current master.
masterNodeId object
Current master node ID, when known.
- string
- null
Current working directory of the API process.
Config path resolved by this API process.
Whether the resolved config path exists from this process cwd.
Installed Skulk package version.
Git commit reported by node identity.
libp2PNamespace object
Configured libp2p namespace environment value, if set.
- string
- null
Whether PYTHONUNBUFFERED is enabled for this process.
Current cluster runtime tracing state as seen by this API node.
Whether config enables centralized structured logging.
loggingIngestUrl object
Configured centralized logging ingest URL, when present.
- string
- null
identity object
Last gathered node identity data.
- NodeIdentity
- null
UnknownUnknownUnknownUnknownUnknownUnknownUnknownresources objectrequired
Resource readings.
gatheredMemory object
Last event-sourced memory reading for this node.
- MemoryUsage
- null
ramTotal objectrequired
0ramAvailable objectrequired
0swapTotal objectrequired
0swapAvailable objectrequired
0currentMemory object
Live memory reading from the API process.
- MemoryUsage
- null
ramTotal objectrequired
0ramAvailable objectrequired
0swapTotal objectrequired
0swapAvailable objectrequired
0currentWired object
Live OS-level wired (unpageable) memory in use (macOS only). Read locally on this endpoint — deliberately NOT on the gossiped MemoryUsage, whose schema rides extra=forbid events — to detect leaked wired memory after an abnormal Metal termination (#239).
- Memory
- null
0disk object
Last event-sourced disk reading for this node.
- DiskUsage
- null
total objectrequired
0available objectrequired
0system object
Last event-sourced system performance reading.
- SystemPerformanceProfile
- null
00000accelerator object
- AcceleratorMetrics
- null
Possible values: [apple, amd, nvidia, intel, cpu, unknown]
unknownUnknownutilizationRatio object
- number
- null
vramTotalBytes object
- integer
- null
vramUsedBytes object
- integer
- null
gttTotalBytes object
GPU-mappable host (GTT) memory, for unified-memory APUs (e.g. AMD Strix
Halo). On such a node the GPU addresses system RAM beyond the BIOS VRAM
carve-out through GTT, so the usable GPU pool is far larger than
vram_total_bytes (placement uses this to admit big models on a UMA node).
None on discrete GPUs / collectors that do not report it.
- integer
- null
powerWatts object
- number
- null
temperatureCelsius object
- number
- null
clockMhz object
- integer
- null
network object
Last event-sourced network interface reading.
- NodeNetworkInfo
- null
interfaces object[]
Possible values: [wifi, ethernet, maybe_ethernet, thunderbolt, unknown]
unknownprocesses object[]
Relevant local OS processes.
Operating-system process ID.
parentPid object
Operating-system parent process ID, when visible.
- integer
- null
Best-effort Skulk role inferred from process lineage and command line.
Possible values: [skulk, runner, vector, python, other]
Joined process command line.
status object
Operating-system process status such as running or sleeping.
- string
- null
cpuPercent object
Recent CPU percentage reported by psutil.
- number
- null
memoryPercent object
Percent of physical memory used by this process.
- number
- null
rss object
Resident set size for this process, when available.
- Memory
- null
0elapsedSeconds object
Seconds since process creation, when available.
- number
- null
Whether this process is in the current Skulk API process tree.
falsesupervisorRunners object[]
Live local runner-supervisor diagnostics.
Runner ID.
Instance ID.
Node ID that owns this runner.
Model assigned to this runner.
Distributed device rank.
Distributed world size.
Inclusive first model layer on this shard.
Exclusive final model layer on this shard.
Total number of model layers.
pid object
Runner subprocess PID, when started.
- integer
- null
Whether the runner subprocess is alive.
exitCode object
Runner subprocess exit code, when exited.
- integer
- null
Current runner status variant.
UTC timestamp for the current status.
Wall-clock seconds spent in the current runner status.
Last runner phase reported.
Possible values: [created, idle, connect_group, load_model, warmup, task_submission, task_agreement, prompt_build, vision_preprocess, kv_cache_lookup, prefill_barrier, prefill_pipeline, prefill_stream, decode_barrier, decode_wait_first_token, decode_stream, parser, cancel_requested, cancel_observed, completion, error, shutdown_cleanup]
UTC timestamp when the current phase started.
Wall-clock seconds spent in the current phase.
lastProgressAt object
UTC timestamp for the last flight-recorder update.
- string
- null
activeTaskId object
Task ID associated with the current phase, when known.
- string
- null
activeCommandId object
Command ID associated with the current phase, when known.
- string
- null
phaseDetail object
Compact human-readable detail for the current phase.
- string
- null
lastMlxMemory object
Most recent MLX memory snapshot reported by the runner.
- MlxMemorySnapshot
- null
UTC timestamp when the snapshot was taken.
active object
Currently active MLX memory, when the runtime exposes it.
- Memory
- null
0cache object
MLX cache memory, when the runtime exposes it.
- Memory
- null
0peak object
Peak MLX memory since the last reset, when available.
- Memory
- null
0wiredLimit object
Configured MLX wired memory limit when known. Current MLX releases do not expose a getter on all platforms, so this may be null.
- Memory
- null
0Runtime module that supplied the measurement, such as mlx.core.
flightRecorder object[]
Last 128 local-only runner diagnostic events.
UTC timestamp when the runner emitted the update.
Runner phase at this entry.
Possible values: [created, idle, connect_group, load_model, warmup, task_submission, task_agreement, prompt_build, vision_preprocess, kv_cache_lookup, prefill_barrier, prefill_pipeline, prefill_stream, decode_barrier, decode_wait_first_token, decode_stream, parser, cancel_requested, cancel_observed, completion, error, shutdown_cleanup]
Short event name within the phase.
detail object
Compact human-readable detail for diagnostics.
- string
- null
attrs object
Structured low-cardinality diagnostic attributes.
property name* object
- string
- integer
- number
- boolean
- string[]
context objectrequired
Stable runner identity fields for this entry.
Node ID that owns this runner.
Runner ID.
pid object
Runner subprocess PID.
- integer
- null
Instance ID.
Model assigned to this runner.
Distributed rank for this runner.
Distributed world size.
Inclusive first layer on this shard.
Exclusive final layer on this shard.
Total model layers.
taskId object
Task ID associated with the entry, when known.
- string
- null
commandId object
Command ID associated with the entry, when known.
- string
- null
mlxMemory object
MLX memory snapshot captured with this entry, when present.
- MlxMemorySnapshot
- null
UTC timestamp when the snapshot was taken.
active object
Currently active MLX memory, when the runtime exposes it.
- Memory
- null
0cache object
MLX cache memory, when the runtime exposes it.
- Memory
- null
0peak object
Peak MLX memory since the last reset, when available.
- Memory
- null
0wiredLimit object
Configured MLX wired memory limit when known. Current MLX releases do not expose a getter on all platforms, so this may be null.
- Memory
- null
0Runtime module that supplied the measurement, such as mlx.core.
Tasks sent to the supervisor but not acknowledged by the runner.
inProgressTasks object[]
Tasks currently known as in progress by the supervisor.
Skulk task ID.
Concrete task model name.
Current event-sourced task status.
Instance associated with the task.
commandId object
External command ID for user-facing inference tasks.
- string
- null
runnerId object
Runner assigned to the task, if known.
- string
- null
modelId object
Model associated with the task, if known.
- string
- null
Number of tasks completed by this supervisor.
Task IDs cancelled through this supervisor.
lastTaskSentAt object
UTC timestamp for the last task submitted to the runner.
- string
- null
lastEventReceivedAt object
UTC timestamp for the last event received from the runner.
- string
- null
lastEventType object
Class name of the last event received from the runner.
- string
- null
milestones object[]
Recent lifecycle milestones retained by the supervisor.
UTC timestamp when the milestone was recorded.
Short milestone name.
detail object
Optional compact detail for the milestone.
- string
- null
placements object[]
Event-sourced placement analysis for current instances.
Instance ID.
Placed model ID.
masterNodeId object
Current master node ID, when known.
- string
- null
Whether the current master is part of this model placement.
Whether the API node is part of this model placement.
Node IDs participating in the placement.
runners object[]
Per-runner placement details.
Runner ID.
Node ID assigned to this runner.
friendlyName object
Friendly node name, when known.
- string
- null
statusKind object
Current event-sourced runner status variant.
- string
- null
Distributed device rank.
Distributed world size.
Inclusive first model layer on this shard.
Exclusive final model layer on this shard.
Total number of model layers.
Whether this assignment is on the API node.
Whether this assignment is on the master node.
tasks object[]
Event-sourced tasks associated with this runner assignment.
Skulk task ID.
Concrete task model name.
Current event-sourced task status.
Instance associated with the task.
commandId object
External command ID for user-facing inference tasks.
- string
- null
runnerId object
Runner assigned to the task, if known.
- string
- null
modelId object
Model associated with the task, if known.
- string
- null
Heuristic warnings that may help explain a stuck placement.
Top-level diagnostic warnings for this node.
error object
Collection error when ok is false.
- string
- null
{
"generatedAt": "string",
"localNodeId": "string",
"masterNodeId": "string",
"nodes": [
{
"nodeId": "string",
"url": "string",
"ok": true,
"diagnostics": {
"generatedAt": "string",
"runtime": {
"nodeId": "string",
"hostname": "string",
"friendlyName": "string",
"isMaster": true,
"masterNodeId": "string",
"cwd": "string",
"configPath": "string",
"configFileExists": true,
"skulkVersion": "string",
"skulkCommit": "string",
"libp2PNamespace": "string",
"pythonUnbuffered": true,
"tracingEnabled": true,
"structuredLoggingConfigured": true,
"loggingIngestUrl": "string"
},
"identity": {
"modelId": "Unknown",
"chipId": "Unknown",
"friendlyName": "Unknown",
"osVersion": "Unknown",
"osBuildVersion": "Unknown",
"skulkVersion": "Unknown",
"skulkCommit": "Unknown"
},
"resources": {
"gatheredMemory": {
"ramTotal": {
"inBytes": 0
},
"ramAvailable": {
"inBytes": 0
},
"swapTotal": {
"inBytes": 0
},
"swapAvailable": {
"inBytes": 0
}
},
"currentMemory": {
"ramTotal": {
"inBytes": 0
},
"ramAvailable": {
"inBytes": 0
},
"swapTotal": {
"inBytes": 0
},
"swapAvailable": {
"inBytes": 0
}
},
"currentWired": {
"inBytes": 0
},
"disk": {
"total": {
"inBytes": 0
},
"available": {
"inBytes": 0
}
},
"system": {
"gpuUsage": 0,
"temp": 0,
"sysPower": 0,
"pcpuUsage": 0,
"ecpuUsage": 0,
"accelerator": {
"vendor": "unknown",
"name": "Unknown",
"utilizationRatio": 0,
"vramTotalBytes": 0,
"vramUsedBytes": 0,
"gttTotalBytes": 0,
"powerWatts": 0,
"temperatureCelsius": 0,
"clockMhz": 0
}
},
"network": {
"interfaces": [
{
"name": "string",
"ipAddress": "string",
"interfaceType": "unknown"
}
]
}
},
"processes": [
{
"pid": 0,
"parentPid": 0,
"role": "skulk",
"command": "string",
"status": "string",
"cpuPercent": 0,
"memoryPercent": 0,
"rss": {
"inBytes": 0
},
"elapsedSeconds": 0,
"isChildOfSkulk": false
}
],
"supervisorRunners": [
{
"runnerId": "string",
"instanceId": "string",
"nodeId": "string",
"modelId": "string",
"deviceRank": 0,
"worldSize": 0,
"startLayer": 0,
"endLayer": 0,
"nLayers": 0,
"pid": 0,
"processAlive": true,
"exitCode": 0,
"statusKind": "string",
"statusSince": "string",
"secondsInStatus": 0,
"phase": "created",
"phaseStartedAt": "string",
"secondsInPhase": 0,
"lastProgressAt": "string",
"activeTaskId": "string",
"activeCommandId": "string",
"phaseDetail": "string",
"lastMlxMemory": {
"generatedAt": "string",
"active": {
"inBytes": 0
},
"cache": {
"inBytes": 0
},
"peak": {
"inBytes": 0
},
"wiredLimit": {
"inBytes": 0
},
"source": "string"
},
"flightRecorder": [
{
"at": "string",
"phase": "created",
"event": "string",
"detail": "string",
"attrs": {},
"context": {
"nodeId": "string",
"runnerId": "string",
"pid": 0,
"instanceId": "string",
"modelId": "string",
"rank": 0,
"worldSize": 0,
"startLayer": 0,
"endLayer": 0,
"nLayers": 0
},
"taskId": "string",
"commandId": "string",
"mlxMemory": {
"generatedAt": "string",
"active": {
"inBytes": 0
},
"cache": {
"inBytes": 0
},
"peak": {
"inBytes": 0
},
"wiredLimit": {
"inBytes": 0
},
"source": "string"
}
}
],
"pendingTaskIds": [
"string"
],
"inProgressTasks": [
{
"taskId": "string",
"taskKind": "string",
"taskStatus": "string",
"instanceId": "string",
"commandId": "string",
"runnerId": "string",
"modelId": "string"
}
],
"completedTaskCount": 0,
"cancelledTaskIds": [
"string"
],
"lastTaskSentAt": "string",
"lastEventReceivedAt": "string",
"lastEventType": "string",
"milestones": [
{
"at": "string",
"name": "string",
"detail": "string"
}
]
}
],
"placements": [
{
"instanceId": "string",
"modelId": "string",
"masterNodeId": "string",
"masterIsPlacementNode": true,
"localNodeIsPlacementNode": true,
"placementNodeIds": [
"string"
],
"runners": [
{
"runnerId": "string",
"nodeId": "string",
"friendlyName": "string",
"statusKind": "string",
"deviceRank": 0,
"worldSize": 0,
"startLayer": 0,
"endLayer": 0,
"nLayers": 0,
"isLocal": true,
"isMaster": true,
"tasks": [
{
"taskId": "string",
"taskKind": "string",
"taskStatus": "string",
"instanceId": "string",
"commandId": "string",
"runnerId": "string",
"modelId": "string"
}
]
}
],
"warnings": [
"string"
]
}
],
"warnings": [
"string"
]
},
"error": "string"
}
]
}