{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 311, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true, "inspect": false }, "links": [ { "title": "查看队列${__data.fields.resource_queue}自定义任务GPU利用率", "url": "d/fb06c699-5c2f-49e5-8d1a-3b87483e1382/5a-55a6i6LWE5rqQ55yL5p2_?orgId=1&var-resource_queue=${__data.fields.resource_queue}&${__all_variables}&${__url_time_range}" } ], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "VCPU" }, "properties": [ { "id": "custom.width", "value": 90 } ] }, { "matcher": { "id": "byName", "options": "MEMORY" }, "properties": [ { "id": "custom.width", "value": 90 } ] }, { "matcher": { "id": "byName", "options": "GPU" }, "properties": [ { "id": "custom.width", "value": 90 } ] } ] }, "gridPos": { "h": 7, "w": 15, "x": 0, "y": 0 }, "id": 2, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": [], "reducer": [ "sum" ], "show": true }, "frameIndex": 0, "showHeader": true }, "pluginVersion": "9.5.18", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "sum by (mlp_devinstance,resource) (sum by (pod,mlp_devinstance) (mlp_devinstance_info) * on (pod) group_right(mlp_devinstance) mlp_instance_resource_requests * on(pod) group_left() (mlp_instance_pod_status_phase{phase=\"Running\"}==1)) ", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "avg by (gpu_type,mlp_devinstance,mlp_resource_group,mlp_resource_queue)(mlp_devinstance_info + on (pod) group_left(gpu_type) mlp_instance_resource_requests{gpu_type!=\"\"}*0) OR on(mlp_devinstance) avg by (mlp_devinstance,mlp_resource_group,mlp_resource_queue) (mlp_devinstance_info + on (pod) group_right(mlp_devinstance,mlp_resource_group,mlp_resource_queue) mlp_instance_resource_requests{gpu_type=\"\"}*0)", "format": "table", "hide": false, "instant": true, "legendFormat": "__auto", "range": false, "refId": "B" } ], "title": "开发机", "transformations": [ { "filter": { "id": "byRefId", "options": "A" }, "id": "groupingToMatrix", "options": { "columnField": "resource", "emptyValue": "null", "rowField": "mlp_devinstance", "valueField": "Value #A" } }, { "filter": { "id": "byRefId", "options": "B" }, "id": "organize", "options": { "excludeByName": { "Time": true, "Value #B": true, "__name__": true, "availability_zone": true, "flavor": true, "gpu_type": false, "instance": true, "job": true, "namespace": true, "pod": true }, "indexByName": {}, "renameByName": { "mlp_devinstance": "mlp_devinstance\\resource" } } }, { "id": "joinByField", "options": { "byField": "mlp_devinstance\\resource", "mode": "inner" } }, { "id": "organize", "options": { "excludeByName": { "resource": true, "unit": true }, "indexByName": { "GPU": 4, "MEMORY": 6, "VCPU": 5, "gpu_type": 3, "mlp_devinstance\\resource": 0, "mlp_resource_group": 1, "mlp_resource_queue": 2 }, "renameByName": { "gpu_type": "", "mlp_devinstance\\resource": "devinstance", "mlp_resource_group": "resource_group", "mlp_resource_queue": "resource_queue" } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "GPU" } ] } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 9, "x": 15, "y": 0 }, "id": 3, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "avg by(mlp_resource_queue) (avg by (pod,mlp_resource_queue) (mlp_devinstance_info{mlp_resource_queue=~\"$resource_queue\"}+ on (pod) group_left(gpu_type) mlp_instance_resource_requests{gpu_type=~\"$gpu_type\",gpu_type!=\"\"}*0) * on (pod) group_right(mlp_resource_queue) (avg by (gpu,pod) (DCGM_FI_DEV_GPU_UTIL{})))", "hide": false, "legendFormat": "{{mlp_resource_queue}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "avg by (mlp_resource_queue,mlp_devinstance) (avg by (pod,mlp_resource_queue,mlp_devinstance) (mlp_devinstance_info{mlp_resource_queue=~\"$resource_queue\"}+ on (pod) group_left(gpu_type) mlp_instance_resource_requests{gpu_type=~\"$gpu_type\",gpu_type!=\"\"}*0) * on (pod) group_right(mlp_resource_queue,mlp_devinstance) (avg by (gpu,pod) (DCGM_FI_DEV_GPU_UTIL{})))", "hide": false, "instant": false, "legendFormat": "{{mlp_resource_queue}}/{{mlp_devinstance}}", "range": true, "refId": "A" } ], "title": "队列${resource_queue}开发机GPU利用率", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true, "inspect": false }, "links": [ { "title": "查看队列${__data.fields.resource_queue}自定义任务GPU利用率", "url": "d/fb06c699-5c2f-49e5-8d1a-3b87483e1382/5a-55a6i6LWE5rqQ55yL5p2_?orgId=1&var-resource_queue=${__data.fields.resource_queue}&${__all_variables}&${__url_time_range}" } ], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "VCPU" }, "properties": [ { "id": "custom.width", "value": 90 } ] }, { "matcher": { "id": "byName", "options": "MEMORY" }, "properties": [ { "id": "custom.width", "value": 90 } ] }, { "matcher": { "id": "byName", "options": "GPU" }, "properties": [ { "id": "custom.width", "value": 90 } ] } ] }, "gridPos": { "h": 7, "w": 15, "x": 0, "y": 7 }, "id": 5, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": [], "reducer": [ "sum" ], "show": true }, "showHeader": true }, "pluginVersion": "9.5.18", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "sum by (mlp_customtask,resource) (sum by (pod,mlp_customtask) (mlp_customtask_instance_info) * on (pod) group_right(mlp_customtask) mlp_instance_resource_requests * on(pod) group_left() (mlp_instance_pod_status_phase{phase=\"Running\"}==1))", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "avg by (gpu_type,mlp_customtask,mlp_resource_group,mlp_resource_queue)(mlp_customtask_instance_info + on (pod) group_left(gpu_type) mlp_instance_resource_requests{gpu_type!=\"\"}*0) OR on(mlp_customtask) avg by (mlp_customtask,mlp_resource_group,mlp_resource_queue) (mlp_customtask_instance_info + on (pod) group_right(mlp_customtask,mlp_resource_group,mlp_resource_queue) mlp_instance_resource_requests{gpu_type=\"\"}*0)", "format": "table", "hide": false, "instant": true, "legendFormat": "__auto", "range": false, "refId": "B" } ], "title": "自定义任务", "transformations": [ { "filter": { "id": "byRefId", "options": "A" }, "id": "groupingToMatrix", "options": { "columnField": "resource", "emptyValue": "null", "rowField": "mlp_customtask", "valueField": "Value #A" } }, { "filter": { "id": "byRefId", "options": "B" }, "id": "organize", "options": { "excludeByName": { "Time": true, "Value #A": false, "Value #B": true, "__name__": true, "availability_zone": true, "flavor": true, "gpu_type": false, "instance": true, "job": true, "mlp_customtask": false, "mlp_resource_group": false, "namespace": true, "node": true, "pod": true, "resource": true, "role": true, "role_index": true }, "indexByName": {}, "renameByName": { "mlp_customtask": "mlp_customtask\\resource", "node": "", "pod": "" } } }, { "id": "joinByField", "options": { "byField": "mlp_customtask\\resource", "mode": "inner" } }, { "id": "organize", "options": { "excludeByName": { "unit": true }, "indexByName": { "GPU": 4, "MEMORY": 6, "VCPU": 5, "gpu_type": 3, "mlp_customtask\\resource": 0, "mlp_resource_group": 1, "mlp_resource_queue": 2, "resource": 7, "unit": 8 }, "renameByName": { "mlp_customtask\\resource": "customtask", "mlp_resource_group": "reousrce_group", "mlp_resource_queue": "resource_queue" } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "GPU" } ] } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 9, "x": 15, "y": 7 }, "id": 6, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": " (avg by (mlp_resource_queue) (avg by (pod,mlp_resource_queue,mlp_customtask) (mlp_customtask_instance_info{mlp_resource_queue=~\"$resource_queue\"}+ on (pod) group_left(gpu_type) mlp_instance_resource_requests{gpu_type=~\"$gpu_type\",gpu_type!=\"\"}*0) * on (pod) group_right(mlp_resource_queue,mlp_customtask) ( avg by (gpu,pod) (DCGM_FI_DEV_GPU_UTIL{}))))", "hide": false, "legendFormat": "{{mlp_resource_queue}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "avg by (mlp_resource_queue,mlp_customtask) (avg by (pod,mlp_resource_queue,mlp_customtask) (mlp_customtask_instance_info{mlp_resource_queue=~\"$resource_queue\"}+ on (pod) group_left(gpu_type) mlp_instance_resource_requests{gpu_type=~\"$gpu_type\",gpu_type!=\"\"}*0) * on (pod) group_right(mlp_resource_queue,mlp_customtask) ( avg by (gpu,pod) (DCGM_FI_DEV_GPU_UTIL{})))", "hide": false, "instant": false, "legendFormat": "{{mlp_resource_queue}}/{{mlp_customtask}}", "range": true, "refId": "A" } ], "title": "队列${resource_queue}自定义任务GPU利用率", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true, "inspect": false }, "links": [ { "title": "查看队列${__data.fields.resource_queue}在线任务GPU利用率", "url": "d/fb06c699-5c2f-49e5-8d1a-3b87483e1382/5a-55a6i6LWE5rqQ55yL5p2_?orgId=1&var-resource_queue=${__data.fields.resource_queue}&${__all_variables}&${__url_time_range}" } ], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "VCPU" }, "properties": [ { "id": "custom.width", "value": 95 } ] }, { "matcher": { "id": "byName", "options": "MEMORY" }, "properties": [ { "id": "custom.width", "value": 90 } ] }, { "matcher": { "id": "byName", "options": "GPU" }, "properties": [ { "id": "custom.width", "value": 90 } ] } ] }, "gridPos": { "h": 9, "w": 15, "x": 0, "y": 14 }, "id": 1, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": true }, "showHeader": true, "sortBy": [] }, "pluginVersion": "9.5.18", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "sum by (mlp_service,resource) (avg by(pod,mlp_service) (mlp_deployment_instance_info) * on (pod) group_right(mlp_service) mlp_instance_resource_requests * on(pod) group_left() (mlp_instance_pod_status_phase{phase=\"Running\"}==1))", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "avg by (gpu_type,mlp_service,mlp_resource_group,mlp_resource_queue)(mlp_deployment_instance_info + on (pod) group_left(gpu_type) mlp_instance_resource_requests{gpu_type!=\"\"}*0) OR on(mlp_service) avg by (mlp_service,mlp_resource_group,mlp_resource_queue) (mlp_deployment_instance_info + on (pod) group_right(mlp_service,mlp_resource_group,mlp_resource_queue) mlp_instance_resource_requests{gpu_type=\"\"}*0)", "format": "table", "hide": false, "instant": true, "legendFormat": "__auto", "range": false, "refId": "B" } ], "title": "在线服务", "transformations": [ { "filter": { "id": "byRefId", "options": "A" }, "id": "groupingToMatrix", "options": { "columnField": "resource", "emptyValue": "null", "rowField": "mlp_service", "valueField": "Value #A" } }, { "filter": { "id": "byRefId", "options": "B" }, "id": "organize", "options": { "excludeByName": { "Time": true, "Value #A": false, "Value #B": true, "__name__": true, "availability_zone": true, "flavor": true, "gpu_type": false, "instance": true, "job": true, "mlp_deployment": true, "mlp_resource_group": false, "mlp_service": false, "mlp_service_instance": true, "namespace": true, "node": true, "pod": true, "resource": true, "unit": true }, "indexByName": {}, "renameByName": { "mlp_service": "mlp_service\\resource", "node": "", "pod": "pod\\resource" } } }, { "id": "joinByField", "options": { "byField": "mlp_service\\resource", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "GPU": 4, "MEMORY": 6, "VCPU": 5, "gpu_type": 3, "mlp_resource_group": 1, "mlp_resource_queue": 2, "mlp_service\\resource": 0 }, "renameByName": { "mlp_deployment": "deployment", "mlp_resource_group": "reousrce_group", "mlp_resource_queue": "resource_queue", "mlp_service": "service", "mlp_service\\resource": "service", "pod\\resource": "instance" } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "GPU" } ] } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 9, "x": 15, "y": 14 }, "id": 4, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": " (avg by (mlp_resource_queue) (avg by(gpu,pod,mlp_resource_queue,mlp_service)(mlp_deployment_instance_info{mlp_resource_queue=~\"$resource_queue\"}+ on (pod) group_left(gpu_type) mlp_instance_resource_requests{gpu_type=~\"$gpu_type\",gpu_type!=\"\"}*0) * on (pod) group_right(mlp_resource_queue,mlp_service) ( DCGM_FI_DEV_GPU_UTIL{})))", "hide": false, "legendFormat": "{{mlp_resource_queue}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "avg by (mlp_resource_queue,mlp_service) (avg by(gpu,pod,mlp_resource_queue,mlp_service)(mlp_deployment_instance_info{mlp_resource_queue=~\"$resource_queue\"}+ on (pod) group_left(gpu_type) mlp_instance_resource_requests{gpu_type=~\"$gpu_type\",gpu_type!=\"\"}*0) * on (pod) group_right(mlp_resource_queue,mlp_service) ( DCGM_FI_DEV_GPU_UTIL{}))", "hide": false, "instant": false, "legendFormat": "{{mlp_resource_queue}}/{{mlp_service}}", "range": true, "refId": "A" } ], "title": "队列${resource_queue}在线服务GPU利用率", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 23 }, "id": 7, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": " sum by (mlp_resource_queue) (mlp_instance_info{mlp_resource_queue=~\"$resource_queue\"} * on (pod) group_right(mlp_resource_queue) mlp_instance_resource_requests{resource=\"GPU\"} * on (pod) group_left() (mlp_instance_pod_status_phase{phase=\"Running\"}==1)) / on (mlp_resource_queue) mlp_resourcequeue_quota_capacity{mlp_resource_queue=~\"$resource_queue\",resource=\"GPU\"} *100", "legendFormat": " GPU申请率 {{mlp_resource_queue}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "avg by (mlp_resource_queue) (mlp_instance_info{mlp_resource_queue=~\"$resource_queue\"} * on (pod) group_right(mlp_resource_queue) avg by (gpu,pod) (DCGM_FI_DEV_GPU_UTIL))", "hide": false, "legendFormat": "GPU利用率 {{mlp_resource_queue}} ", "range": true, "refId": "B" }, { "datasource": { "name": "Expression", "type": "__expr__", "uid": "__expr__" }, "expression": "$A*$B/100", "hide": false, "refId": "GPU申请率*利用率", "type": "math" } ], "title": "队列比率监控", "type": "timeseries" } ], "refresh": "", "schemaVersion": 38, "style": "dark", "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "ArkBenchmark-cn-datong", "value": "ArkBenchmark-cn-datong" }, "hide": 0, "includeAll": false, "multi": false, "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "current": { "selected": false, "text": "q-20240419155134-lnsgm", "value": "q-20240419155134-lnsgm" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(mlp_instance_info,mlp_resource_queue)", "hide": 0, "includeAll": false, "multi": true, "name": "resource_queue", "options": [], "query": { "query": "label_values(mlp_instance_info,mlp_resource_queue)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": { "isNone": true, "selected": false, "text": "None", "value": "" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "query_result(avg by (gpu_type) (avg by(mlp_resource_group) (mlp_resourcequeue_info{mlp_resource_queue=~\"${resource_queue}\"}) + on(mlp_resource_group) group_right() mlp_node_status_allocatable{gpu_type!=\"\"}))", "hide": 0, "includeAll": false, "label": "", "multi": false, "name": "gpu_type", "options": [], "query": { "query": "query_result(avg by (gpu_type) (avg by(mlp_resource_group) (mlp_resourcequeue_info{mlp_resource_queue=~\"${resource_queue}\"}) + on(mlp_resource_group) group_right() mlp_node_status_allocatable{gpu_type!=\"\"}))", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "/.*. *gpu_type=\"(.*?)\".*/", "skipUrlSync": false, "sort": 0, "type": "query" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "对客资源看板", "uid": "fb06c699-5c2f-49e5-8d1a-3b87483e1382", "version": 151, "weekStart": "" }