{ "name": "Milvus", "tags": "", "ident": "", "configs": { "version": "3.0.0", "links": [], "var": [ { "name": "datasource", "type": "datasource", "definition": "prometheus" } ], "panels": [ { "version": "2.0.0", "id": "d5a598e7-b40a-40ec-9c1d-ded1e1733dd2", "type": "row", "name": "Proxy", "collapsed": true, "layout": { "h": 1, "w": 24, "x": 0, "y": 0, "i": "d5a598e7-b40a-40ec-9c1d-ded1e1733dd2", "isResizable": false }, "panels": [] }, { "version": "2.0.0", "id": "e4c14e02-a2ee-4611-b606-789ddb964b1f", "type": "timeseries", "name": "Search Vector Count Rate", "description": "per-second increasing rate of searching vectors.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 1, "i": "e4c14e02-a2ee-4611-b606-789ddb964b1f", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_proxy_search_vectors_count{}[2m])/120) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "6327a12c-b06f-4fae-92d7-6c4bfc48eaf9", "type": "timeseries", "name": "Insert Vector Count Rate", "description": "per-second increasing rate of inserting vectors.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 1, "i": "6327a12c-b06f-4fae-92d7-6c4bfc48eaf9", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_proxy_insert_vectors_count{}[2m])/120) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "0c7b67e6-277c-477b-bcd3-2bba97ffd5c7", "type": "timeseries", "name": "Search Latency", "description": "The 99th percentile and average latency of search request over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 1, "i": "0c7b67e6-277c-477b-bcd3-2bba97ffd5c7", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_sq_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_proxy_sq_latency_sum{}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_sq_latency_count{}[2m])) by (pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "05af76dc-b286-4b63-8e92-f9593e2e8c3e", "type": "timeseries", "name": "Collection Search Latency", "description": "The 99th percentile and average latency of search request for a specified collection over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 7, "i": "05af76dc-b286-4b63-8e92-f9593e2e8c3e", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_collection_sq_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_proxy_collection_sq_latency_sum{}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_collection_sq_latency_count{}[2m])) by (pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "a2606b9a-ae50-4327-8761-a801be8e66b1", "type": "timeseries", "name": "Mutation Latency", "description": "The 99th percentile and average latency of mutation request over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 7, "i": "a2606b9a-ae50-4327-8761-a801be8e66b1", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{}[2m])))", "legend": "p99-{{msg_type}}-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_proxy_mutation_latency_sum{}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{}[2m])) by (pod, node_id, msg_type)", "legend": "avg-{{msg_type}}-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "93eb1c49-1af2-42db-a20e-6b96dbd4664a", "type": "timeseries", "name": "Collection Mutation Latency", "description": "The 99th percentile and average latency of mutation request for a specified collection over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 7, "i": "93eb1c49-1af2-42db-a20e-6b96dbd4664a", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_collection_mutation_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}-{{msg_type}}" }, { "refId": "B", "expr": "sum(increase(milvus_proxy_collection_mutation_latency_sum{}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_collection_mutation_latency_count{}[2m])) by (pod, node_id, msg_type)", "legend": "avg-{{pod}}-{{node_id}}-{{msg_type}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "97fffef6-f0d0-4a44-b25e-145bfac18406", "type": "timeseries", "name": "Wait Search Result Latency", "description": "The 99th percentile and average latency of wait search result over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 13, "i": "97fffef6-f0d0-4a44-b25e-145bfac18406", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_sq_wait_result_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_proxy_sq_wait_result_latency_sum{}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_sq_wait_result_latency_count{}[2m])) by (pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "515d7727-3ff3-459d-b87c-604a547ed0b2", "type": "timeseries", "name": "Reduce Search Result Latency", "description": "The 99th percentile and average latency of reduce search result over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 13, "i": "515d7727-3ff3-459d-b87c-604a547ed0b2", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_sq_reduce_result_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_proxy_sq_reduce_result_latency_sum{}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_sq_reduce_result_latency_count{}[2m])) by (pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "3c8b4b78-25df-427b-bb89-8c7edef30626", "type": "timeseries", "name": "Decode Search Result Latency", "description": "The 99th percentile and average latency of decode search result over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 13, "i": "3c8b4b78-25df-427b-bb89-8c7edef30626", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_sq_decode_result_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_proxy_sq_decode_result_latency_sum{}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_sq_decode_resultlatency_count{}[2m])) by (pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "8949845c-4212-4fda-91a0-ddf4d2ebe558", "type": "timeseries", "name": "Msg Stream Object Num", "description": "Average, maximum and minimum values of the number of msgstream objects created on all physical topics.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 19, "i": "8949845c-4212-4fda-91a0-ddf4d2ebe558", "isResizable": true }, "targets": [ { "refId": "A", "expr": "avg(milvus_proxy_msgstream_obj_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-avg" }, { "refId": "B", "expr": "max(milvus_proxy_msgstream_obj_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-max" }, { "refId": "C", "expr": "min(milvus_proxy_msgstream_obj_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-min" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "079be73f-3924-42cb-873b-7444690a1da2", "type": "timeseries", "name": "Mutation Send Latency", "description": "The 99th percentile and average latency of send mutation over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 19, "i": "079be73f-3924-42cb-873b-7444690a1da2", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_send_latency_bucket{}[2m])))", "legend": "p99-{{msg_type}}-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_proxy_mutation_latency_sum{}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{}[2m])) by (pod, node_id, msg_type)", "legend": "avg-{{msg_type}}-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "3921ef96-2991-49c8-8d9b-ccda0aa97360", "type": "timeseries", "name": "Cache Hit rate", "description": "Average cache hits per minute of cache operations.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 19, "i": "3921ef96-2991-49c8-8d9b-ccda0aa97360", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_proxy_cache_hit_count{cache_state=\"hit\"}[2m])/120) by(cache_name, pod, node_id) / sum(increase(milvus_proxy_cache_hit_count{}[2m])/120) by(cache_name, pod, node_id)", "legend": "{{pod}}-{{node_id}}-{{cache_name}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "c5b8e33d-2cf5-458b-bb0f-6f864a8116d8", "type": "timeseries", "name": "Cache Update Latency", "description": "The 99th percentile and average latency of update cache over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 25, "i": "c5b8e33d-2cf5-458b-bb0f-6f864a8116d8", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_cache_update_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_proxy_cache_update_latency_sum{}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_cache_update_latency_count{}[2m])) by (pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "12ed716e-b48a-45d8-afa3-10c636f9b883", "type": "timeseries", "name": "Success Request Rate", "description": "per-second increasing rate of success requests.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 25, "i": "12ed716e-b48a-45d8-afa3-10c636f9b883", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_proxy_req_count{status=\"success\"}[2m])/120) by(function_name, pod, node_id)", "legend": "{{function_name}}-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "037c07cb-4ca0-4e06-a6d4-3b85f897fb32", "type": "timeseries", "name": "Apply PK Latency", "description": "The 99th percentile of latency for applying PK over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 25, "i": "037c07cb-4ca0-4e06-a6d4-3b85f897fb32", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_pk_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_proxy_apply_pk_latency_sum{}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_pk_latency_count{}[2m])) by (pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "159e884f-a5d0-4626-8121-ab8b9f84fc0e", "type": "timeseries", "name": "Apply Timestamp Latency", "description": "The 99th percentile and average latency of apply timestamp over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 31, "i": "159e884f-a5d0-4626-8121-ab8b9f84fc0e", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_timestamp_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_proxy_apply_timestamp_latency_sum{}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_timestamp_latency_count{}[2m])) by (pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "13fa109a-02ce-4f6f-8ef7-a7c4d2a9109c", "type": "timeseries", "name": "Received Byte Rate", "description": "per-second increasing rate of bytes received in proxy", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 31, "i": "13fa109a-02ce-4f6f-8ef7-a7c4d2a9109c", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_proxy_receive_bytes_count{}[2m])/120) by(pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "722b90a7-c067-4fc0-a22b-b154abcb5517", "type": "timeseries", "name": "Faild Request Rate", "description": "per-second increasing rate of faild requests.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 31, "i": "722b90a7-c067-4fc0-a22b-b154abcb5517", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_proxy_req_count{status=\"fail\"}[2m])/120) by(function_name, pod, node_id)", "legend": "{{function_name}}-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "c018977e-1908-4feb-9836-211a8cd3f441", "type": "timeseries", "name": "Request Latency", "description": "The 99th percentile and average latency of request over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 37, "i": "c018977e-1908-4feb-9836-211a8cd3f441", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, function_name) (rate(milvus_proxy_req_latency_bucket{}[2m])))", "legend": "p99-{{function_name}}-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_proxy_req_latency_sum{}[2m])) by (pod, node_id, function_name) / sum(increase(milvus_proxy_req_latency_count{}[2m])) by (pod, node_id, function_name)", "legend": "avg-{{function_name}}-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "66e02fe4-2c49-4e3b-a090-c734095cb414", "type": "timeseries", "name": "Send Byte Rate", "description": "per-second increasing rate of bytes sent back to the client by the Proxy in response to a Search or Query request.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 37, "i": "66e02fe4-2c49-4e3b-a090-c734095cb414", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_proxy_send_bytes_count{}[2m])/120) by(pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "08758c7b-f421-4246-af3c-f449af8a6adf", "type": "row", "name": "Root Coordinator", "collapsed": true, "layout": { "h": 1, "w": 24, "x": 0, "y": 211, "i": "08758c7b-f421-4246-af3c-f449af8a6adf", "isResizable": false }, "panels": [] }, { "version": "2.0.0", "id": "87ca7f16-808e-43e2-9d01-8c93813f539c", "type": "timeseries", "name": "Proxy Node Num", "description": "Number of proxy nodes which has register with etcd", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 212, "i": "87ca7f16-808e-43e2-9d01-8c93813f539c", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_rootcoord_proxy_num{}) by (app_kubernetes_io_instance)", "legend": "num" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "611c7fc6-4a8b-49b7-8028-f04dbfa56851", "type": "timeseries", "name": "Produced Timetick Lag Behind Now", "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 212, "i": "611c7fc6-4a8b-49b7-8028-f04dbfa56851", "isResizable": true }, "targets": [ { "refId": "A", "expr": "avg(milvus_rootcoord_produce_tt_lag_ms{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-avg" }, { "refId": "B", "expr": "max(milvus_rootcoord_produce_tt_lag_ms{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-max" }, { "refId": "C", "expr": "min(milvus_rootcoord_produce_tt_lag_ms{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-min" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "5b5c568e-82be-4fde-a394-88d1d9a38e31", "type": "timeseries", "name": "Produced Timetick Time Taken", "description": "The 99th percentile and average latency for rootcoord to finish synchronizing timestamp messages to all pchanels.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 212, "i": "5b5c568e-82be-4fde-a394-88d1d9a38e31", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{}[2m])))", "legend": "p99-latency" }, { "refId": "B", "expr": "sum(increase(milvus_rootcoord_sync_timetick_latency_sum{}[2m])) / sum(increase(milvus_rootcoord_sync_timetick_latency_count{}[2m]))", "legend": "avg-latency" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "3e225d35-9139-4dbe-a7f5-e98b11f66a3a", "type": "timeseries", "name": "DDL Request Latency", "description": "The 99th percentile and average latency of DDL request over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 218, "i": "3e225d35-9139-4dbe-a7f5-e98b11f66a3a", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{}[2m])))", "legend": "p99-{{function_name}}" }, { "refId": "B", "expr": "sum(increase(milvus_rootcoord_ddl_req_latency_sum{}[2m])) by (function_name) / sum(increase(milvus_rootcoord_ddl_req_latency_count{}[2m])) by (function_name)", "legend": "avg-{{function_name}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "8e06bc20-71b3-495a-ba92-4887d125f9f1", "type": "timeseries", "name": "Timestamp Saved", "description": "RootCoord stores pre-assigned timestamps in the metastore", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 218, "i": "8e06bc20-71b3-495a-ba92-4887d125f9f1", "isResizable": true }, "targets": [ { "refId": "A", "expr": "milvus_rootcoord_timestamp_saved{}", "legend": "tiestamp" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "55c61fd0-7321-44c9-9a22-5c87294f8312", "type": "timeseries", "name": "DDL Request Rate", "description": "per-second increasing rate of DDL request", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 218, "i": "55c61fd0-7321-44c9-9a22-5c87294f8312", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_rootcoord_ddl_req_count{}[2m])/120) by (status, function_name)", "legend": "{{function_name}}-{{status}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "739e3ce4-435c-4ff4-9ca8-e5ff72b030a4", "type": "timeseries", "name": "Timestamp", "description": "RoootCoord current latest timestamp", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 224, "i": "739e3ce4-435c-4ff4-9ca8-e5ff72b030a4", "isResizable": true }, "targets": [ { "refId": "A", "expr": "milvus_rootcoord_timestamp{}", "legend": "tiestamp" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "f47ca641-7a8a-416e-a125-9f3c6c8313cd", "type": "timeseries", "name": "DML Channel Num", "description": "The number of DML channels", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 224, "i": "f47ca641-7a8a-416e-a125-9f3c6c8313cd", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_rootcoord_dml_channel_num{}) by (app_kubernetes_io_instance)", "legend": "num" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "c0eae21c-83f8-4efb-b320-40280cddcab6", "type": "timeseries", "name": "ID Alloc Rate", "description": "per-second increasing rate of IDs assigned by RootCoord", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 224, "i": "c0eae21c-83f8-4efb-b320-40280cddcab6", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_rootcoord_id_alloc_count{}[2m])/120)", "legend": "total" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "1f856a4f-9702-4dd7-b386-619d3e54c502", "type": "timeseries", "name": "Partition Num", "description": "The number of partitions.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 230, "i": "1f856a4f-9702-4dd7-b386-619d3e54c502", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_rootcoord_partition_num{}) by (app_kubernetes_io_instance)", "legend": "num" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "a70990f0-caeb-49e2-9673-ab3f3f03f49b", "type": "timeseries", "name": "Time Tick Delay", "description": "The max time tick delay of flow graphs. unit ms", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 230, "i": "a70990f0-caeb-49e2-9673-ab3f3f03f49b", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_rootcoord_time_tick_delay{}) by (role_name, node_id)", "legend": "{{role_name}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "6bf9959d-f70f-4237-9c11-439d3b28ce54", "type": "timeseries", "name": "Collection Num", "description": "The number of collections.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 230, "i": "6bf9959d-f70f-4237-9c11-439d3b28ce54", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_rootcoord_collection_num{}) by (app_kubernetes_io_instance)", "legend": "num" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "72cce244-b3bc-4b3d-a71b-0e9b703dd095", "type": "timeseries", "name": "Credential Num", "description": "The number of credential", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 236, "i": "72cce244-b3bc-4b3d-a71b-0e9b703dd095", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_rootcoord_credential_num{}) by (app_kubernetes_io_instance)", "legend": "num" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "57800b69-a076-485c-a145-80b2f448a634", "type": "timeseries", "name": "Msgstream Num", "description": "The number of Msgstream objects.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 236, "i": "57800b69-a076-485c-a145-80b2f448a634", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_rootcoord_msgstream_obj_num{}) by (app_kubernetes_io_instance)", "legend": "num" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "376e4044-0c27-43fb-ab56-0729c7875d02", "type": "row", "name": "Query Coordinator", "collapsed": true, "layout": { "h": 1, "w": 24, "x": 0, "y": 332, "i": "376e4044-0c27-43fb-ab56-0729c7875d02", "isResizable": false }, "panels": [] }, { "version": "2.0.0", "id": "e16709d3-3356-402b-afe7-bf49b44d9311", "type": "timeseries", "name": "Collection Loaded Num", "description": "Total number of loaded collections.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 333, "i": "e16709d3-3356-402b-afe7-bf49b44d9311", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querycoord_collection_num{}) by (app_kubernetes_io_instance)", "legend": "num" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "5c45f7dc-d6dd-4520-b484-e342a36e48e7", "type": "timeseries", "name": "Entity Loaded Num", "description": "Total number of loaded entities.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 333, "i": "5c45f7dc-d6dd-4520-b484-e342a36e48e7", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querycoord_entity_num{}) by (app_kubernetes_io_instance)", "legend": "num" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "3e0ff25b-2c1d-435e-8ad6-087cfbe292ee", "type": "timeseries", "name": "Load Request Rate", "description": "per-second increasing rate of load requests.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 333, "i": "3e0ff25b-2c1d-435e-8ad6-087cfbe292ee", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_querycoord_load_req_count{}[2m])/120) by (status)", "legend": "{{status}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "63a76df2-c53b-4daa-8025-5df6fb99a0ba", "type": "timeseries", "name": "Release Request Rate", "description": "per-second increasing rate of relaase requests.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 339, "i": "63a76df2-c53b-4daa-8025-5df6fb99a0ba", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_querycoord_release_req_count{}[2m])/120) by (status)", "legend": "{{status}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "8bf65612-ecde-4ae5-bb67-db32fafbd8a7", "type": "timeseries", "name": "Load Request Latency", "description": "The 99th percentile and average latency of load request over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 339, "i": "8bf65612-ecde-4ae5-bb67-db32fafbd8a7", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_load_latency_bucket{}[2m])))", "legend": "p99-latency" }, { "refId": "B", "expr": "sum(increase(milvus_querycoord_load_latency_sum{}[2m])) / sum(increase(milvus_querycoord_load_latency_count{}[2m]))", "legend": "avg-latency" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "725e7798-48a2-4ca2-a0ac-80b9397e1918", "type": "timeseries", "name": "Release Request Latency", "description": "The 99th percentile and average latency of release request over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 339, "i": "725e7798-48a2-4ca2-a0ac-80b9397e1918", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_release_latency_bucket{}[2m])))", "legend": "p99-latency" }, { "refId": "B", "expr": "sum(increase(milvus_querycoord_release_latency_sum{}[2m])) / sum(increase(milvus_querycoord_release_latency_count{}[2m]))", "legend": "avg-latency" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "c77a4eb5-a16c-43ac-ad05-4f73e0f7b8d4", "type": "timeseries", "name": "Sub-Load Task", "description": "Total number of sub-load task.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 345, "i": "c77a4eb5-a16c-43ac-ad05-4f73e0f7b8d4", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querycoord_child_task_num{}) by (app_kubernetes_io_instance)", "legend": "num" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "01952969-2076-4ff5-a730-1992d7852d0a", "type": "timeseries", "name": "Parent Load Task", "description": "Total number pf parent loading task.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 345, "i": "01952969-2076-4ff5-a730-1992d7852d0a", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querycoord_parent_task_num{}) by (app_kubernetes_io_instance)", "legend": "num" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "4b7ca122-227a-413b-ba61-f82926b7e3ce", "type": "timeseries", "name": "Sub-Load Task Latency", "description": "The 99th percentile and average latency of sub-load task request over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 345, "i": "4b7ca122-227a-413b-ba61-f82926b7e3ce", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_child_task_latency_bucket{}[2m])))", "legend": "p99-latency" }, { "refId": "B", "expr": "sum(increase(milvus_querycoord_child_task_latency_sum{}[2m])) / sum(increase(milvus_querycoord_child_task_latency_count{}[2m]))", "legend": "avg-latency" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "c2980c07-3af7-46ef-8430-91af233d6725", "type": "timeseries", "name": "Query Node Num", "description": "Number of Query nodes which has register with etcd.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 351, "i": "c2980c07-3af7-46ef-8430-91af233d6725", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querycoord_querynode_num{}) by (app_kubernetes_io_instance)", "legend": "num" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "ae59c928-e3bc-4267-90c9-71adc3c21561", "type": "row", "name": "Query Node", "collapsed": false, "layout": { "h": 1, "w": 24, "x": 0, "y": 381, "i": "ae59c928-e3bc-4267-90c9-71adc3c21561", "isResizable": false }, "panels": [ { "version": "2.0.0", "id": "d9f5eed3-0284-4f5a-b2a1-696c7f930c19", "type": "timeseries", "name": "Collection Loaded Num", "description": "Total number of the loaded collections in QueryNode.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 256, "i": "d9f5eed3-0284-4f5a-b2a1-696c7f930c19", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querynode_collection_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "262f89da-c749-4443-80a7-525a4a95b8fd", "type": "timeseries", "name": "Partition Loaded Num", "description": "Total number of the loaded partitions in QueryNode.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 256, "i": "262f89da-c749-4443-80a7-525a4a95b8fd", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querynode_partition_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "a9ca06b6-881b-45b9-925a-d9c97c4385a1", "type": "timeseries", "name": "Segment Loaded Num", "description": "Total number of the loaded segment in QueryNode.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 256, "i": "a9ca06b6-881b-45b9-925a-d9c97c4385a1", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querynode_segment_num{}) by (pod, node_id, segment_state)", "legend": "{{pod}}-{{node_id}}-{{segment_state}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "282986b2-d566-4f07-8af5-fe13999d8a99", "type": "timeseries", "name": "TimeTick Lag Behind Now (Consumed Insert)", "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 262, "i": "282986b2-d566-4f07-8af5-fe13999d8a99", "isResizable": true }, "targets": [ { "refId": "A", "expr": "avg(milvus_querynode_consume_tt_lag_ms{msg_type=\"insert\"}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-avg" }, { "refId": "B", "expr": "max(milvus_querynode_consume_tt_lag_ms{msg_type=\"insert\"}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-max" }, { "refId": "C", "expr": "min(milvus_querynode_consume_tt_lag_ms{msg_type=\"insert\"}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-min" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "e9fda4d8-7da6-4475-9991-3c75d4127444", "type": "timeseries", "name": "TimeTick Lag Behind Now (Consumed Delete)", "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 262, "i": "e9fda4d8-7da6-4475-9991-3c75d4127444", "isResizable": true }, "targets": [ { "refId": "A", "expr": "avg(milvus_querynode_consume_tt_lag_ms{msg_type=\"delete\"}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-avg" }, { "refId": "B", "expr": "max(milvus_querynode_consume_tt_lag_ms{msg_type=\"delete\"}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-max" }, { "refId": "C", "expr": "min(milvus_querynode_consume_tt_lag_ms{msg_type=\"delete\"}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-min" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "04b5c40a-2640-4952-943c-265f4681a589", "type": "timeseries", "name": "Consumed Message Rate", "description": "per-second increasing rate of consuming message", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 262, "i": "04b5c40a-2640-4952-943c-265f4681a589", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_querynode_consume_msg_count{}[2m])/120) by (pod, node_id, msg_type)", "legend": "{{pod}}-{{node_id}}-{{msg_type}}" }, { "refId": "B", "expr": "sum(increase(milvus_querynode_consume_msg_count{}[2m])/120) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-all" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "d901d2ed-817a-42d6-9fe0-7a8b214a6d15", "type": "timeseries", "name": "Queryable Entity Num", "description": "Total number of the queryable entities in QueryNode.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 268, "i": "d901d2ed-817a-42d6-9fe0-7a8b214a6d15", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querynode_entity_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "6e786b33-5990-4c05-85c9-c7ed355f0552", "type": "timeseries", "name": "DML Virtual Channel", "description": "Total number of dml virtual channels for QueryNode watch", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 268, "i": "6e786b33-5990-4c05-85c9-c7ed355f0552", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querynode_dml_vchannel_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "5ed9a82e-7e00-46c3-aebb-70cc7feac6cd", "type": "timeseries", "name": "Delta Virtual Channel", "description": "Total number of delta virtual channels for QueryNode watch", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 268, "i": "5ed9a82e-7e00-46c3-aebb-70cc7feac6cd", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querynode_delta_vchannel_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "dcd7a4f7-3a45-4b5e-9407-8793d5293538", "type": "timeseries", "name": "Consumer Num", "description": "Total number of consumers in the QueryNode.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 274, "i": "dcd7a4f7-3a45-4b5e-9407-8793d5293538", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querynode_consumer_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "4843c7ac-68c9-4a5b-8148-2a1ccfe8e987", "type": "timeseries", "name": "Search Request Rate", "description": "per-second increasing rate of searching requests.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 274, "i": "4843c7ac-68c9-4a5b-8148-2a1ccfe8e987", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_querynode_sq_req_count{}[2m])/120) by (status, pod, node_id)", "legend": "{{pod}}-{{node_id}}-{{status}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "ad41f284-555d-47ae-8dcd-4bb674ce60ab", "type": "timeseries", "name": "Search Request Latency", "description": "The 99th percentile and average latency of search and query request over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 274, "i": "ad41f284-555d-47ae-8dcd-4bb674ce60ab", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_sq_req_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_querynode_sq_req_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_sq_req_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "7be18b1c-13f8-4ac9-b7a7-ab8bd46ea6f2", "type": "timeseries", "name": "Search in Queue Latency", "description": "The 99th percentile and average latency of search or query in queue over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 280, "i": "7be18b1c-13f8-4ac9-b7a7-ab8bd46ea6f2", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_sq_queue_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_sq_queue_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "098905fb-f459-43dc-9641-9e850e9dd5dc", "type": "timeseries", "name": "Search Segment Latency", "description": "The 99th percentile and average latency of search segment over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 280, "i": "098905fb-f459-43dc-9641-9e850e9dd5dc", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}-{{segment_state}}" }, { "refId": "B", "expr": "sum(increase(milvus_querynode_sq_segment_latency_sum{}[2m])) by(pod, node_id, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{}[2m])) by(pod, node_id, segment_state)", "legend": "avg-{{pod}}-{{node_id}}_{{segment_state}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "0bc05070-f570-4eac-824c-6430e29936dd", "type": "timeseries", "name": "Segcore Request Latency", "description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 280, "i": "0bc05070-f570-4eac-824c-6430e29936dd", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_sq_core_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_querynode_sq_core_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_sq_core_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "971cdaef-9a6c-4ef7-a330-550c64dac933", "type": "timeseries", "name": "Search Reduce Latency", "description": "The 99th percentile and average latency of search or query reduce over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 286, "i": "971cdaef-9a6c-4ef7-a330-550c64dac933", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_sq_reduce_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_querynode_sq_reduce_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_sq_reduce_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "12b3e2ee-fcaf-4155-b70a-1809e530ccec", "type": "timeseries", "name": "Load Segment Latency", "description": "The 99th percentile and average latency of load segment over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 286, "i": "12b3e2ee-fcaf-4155-b70a-1809e530ccec", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_load_segment_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_querynode_load_segment_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_load_segment_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "6ca37496-bde8-4a8e-a2cb-5b9454e9081c", "type": "timeseries", "name": "Flowgraph Num", "description": "Total number of flowgraph", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 286, "i": "6ca37496-bde8-4a8e-a2cb-5b9454e9081c", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querynode_flowgraph_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "0eaff922-91a8-4aac-a1ba-1817694fa6ce", "type": "timeseries", "name": "Unsolved Read Task Length", "description": "The length of the task queue for unsolved read requests", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 292, "i": "0eaff922-91a8-4aac-a1ba-1817694fa6ce", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querynode_read_task_unsolved_len{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "c08e3236-678e-46ed-8b50-12721caaf45c", "type": "timeseries", "name": "Ready Read Task Length", "description": "The length of the task queue of read requests to be executed", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 292, "i": "c08e3236-678e-46ed-8b50-12721caaf45c", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querynode_read_task_ready_len{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "469d9c04-99f0-428a-b8ca-c6e8bf9552a1", "type": "timeseries", "name": "Parallel Read Task Num", "description": "Number of read requests currently being executed in parallel", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 292, "i": "469d9c04-99f0-428a-b8ca-c6e8bf9552a1", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querynode_read_task_concurrency{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "c3182c2a-3c36-4e82-926c-0612ca8db636", "type": "timeseries", "name": "Estimate CPU Usage", "description": "cpu utilization under scheduler evaluation", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 298, "i": "c3182c2a-3c36-4e82-926c-0612ca8db636", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_querynode_estimate_cpu_usage{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "13c806f3-5725-40d4-b4ee-96751dc2ffed", "type": "timeseries", "name": "Search Group Size", "description": "The number of original tasks contained in the merged search task", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 298, "i": "13c806f3-5725-40d4-b4ee-96751dc2ffed", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_size_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_querynode_search_group_size_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_size_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "2073c0c0-1f9a-4e07-9840-d53ba1eb7d28", "type": "timeseries", "name": "Search NQ", "description": "Number of queries for search requests", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 298, "i": "2073c0c0-1f9a-4e07-9840-d53ba1eb7d28", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_nq_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_querynode_search_nq_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_nq_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "d57724f9-b5c9-446d-887f-e1d0b57a080e", "type": "timeseries", "name": "Search Group NQ", "description": "Number of queries for the merged search requests", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 304, "i": "d57724f9-b5c9-446d-887f-e1d0b57a080e", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_nq_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_querynode_search_group_nq_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_nq_count{}[2m])) by(pod, node_id)", "legend": "avg-milvus_querynode_search_group_nq" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "8db64d3e-a158-44fe-9744-a25bbdccb907", "type": "timeseries", "name": "Search Top_K", "description": "Top_K for search requests", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 304, "i": "8db64d3e-a158-44fe-9744-a25bbdccb907", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_topk_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_querynode_search_topk_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_topk_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "b0717546-f925-41ef-9e44-92b78888d6fc", "type": "timeseries", "name": "Search Group Top_K", "description": "Top_K for the merged search requests.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 304, "i": "b0717546-f925-41ef-9e44-92b78888d6fc", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_topk_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_querynode_search_group_topk_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_topk_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "f0fe52c6-82a9-4dc3-a72f-613ef26a080c", "type": "timeseries", "name": "Evicted Read Requests Rate", "description": "per-second increasing rate of evicted read requests.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 310, "i": "f0fe52c6-82a9-4dc3-a72f-613ef26a080c", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_querynode_read_evicted_count{}[2m])/120) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" } ] }, { "version": "2.0.0", "id": "9ec90324-6fc5-4c2b-9172-0123ad263f93", "type": "row", "name": "Data Coordinator", "collapsed": true, "layout": { "h": 1, "w": 24, "x": 0, "y": 502, "i": "9ec90324-6fc5-4c2b-9172-0123ad263f93", "isResizable": false }, "panels": [] }, { "version": "2.0.0", "id": "cd4dd60a-243f-49b1-872f-c9e8aee17e83", "type": "timeseries", "name": "Data Node Num", "description": "Number of data nodes which has register with etcd.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 503, "i": "cd4dd60a-243f-49b1-872f-c9e8aee17e83", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_datacoord_datanode_num{}) by (app_kubernetes_io_instance)", "legend": "total" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "9d2b8a55-5478-4d22-af8a-787fdac81883", "type": "timeseries", "name": "TimeTick Lag Behind Now From DataNode", "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now from datanode.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 503, "i": "9d2b8a55-5478-4d22-af8a-787fdac81883", "isResizable": true }, "targets": [ { "refId": "A", "expr": "avg(milvus_datacoord_consume_datanode_tt_lag_ms{msg_type=\"all\"}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-avg" }, { "refId": "B", "expr": "max(milvus_datacoord_consume_datanode_tt_lag_ms{msg_type=\"all\"}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-max" }, { "refId": "C", "expr": "min(milvus_datacoord_consume_datanode_tt_lag_ms{msg_type=\"all\"}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-min" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "594a28ba-30b4-41c8-bacf-4f49ca48f1ec", "type": "timeseries", "name": "Stored Rows", "description": "The number of rows of valid data accumulated in DataCoord that flushed.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 503, "i": "594a28ba-30b4-41c8-bacf-4f49ca48f1ec", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_datacoord_stored_rows_num{}) by (app_kubernetes_io_instance)", "legend": "row" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "1f17a059-41da-42c5-bf29-b3502844bad7", "type": "timeseries", "name": "Stored Rows Rate", "description": "per-second increasing rate of data flushed.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 509, "i": "1f17a059-41da-42c5-bf29-b3502844bad7", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_datacoord_stored_rows_count{}[2m])/120) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "2f9066f1-b4b5-45c2-991c-15fdbe5ca068", "type": "timeseries", "name": "Segment Num", "description": "Number of segments with different states in the Meta of DataCoord", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 509, "i": "2f9066f1-b4b5-45c2-991c-15fdbe5ca068", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_datacoord_segment_num{}) by (segment_state)", "legend": "{{segment_state}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "f4f5d310-e824-4ca7-87a6-b79321ec21a5", "type": "timeseries", "name": "Stored Binlog Size", "description": "binlog size of all collections/segments, unit byte", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 509, "i": "f4f5d310-e824-4ca7-87a6-b79321ec21a5", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_datacoord_stored_binlog_size{}) by (app_kubernetes_io_instance)", "legend": "size" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "d5b5e020-c6ea-405a-80fe-db8a694e3429", "type": "row", "name": "Data Node", "collapsed": true, "layout": { "h": 1, "w": 24, "x": 0, "y": 515, "i": "d5b5e020-c6ea-405a-80fe-db8a694e3429", "isResizable": false }, "panels": [] }, { "version": "2.0.0", "id": "1675baf6-901b-4868-8fbf-747c4f3a25f1", "type": "timeseries", "name": "Flowgraph Num", "description": "Total number of flowgraph", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 516, "i": "1675baf6-901b-4868-8fbf-747c4f3a25f1", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_datanode_flowgraph_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "fea55e0c-bd88-4c10-b6ac-8ee5ede0f99f", "type": "timeseries", "name": "Producer Num", "description": "Total number of prodecer created on the DataNode", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 516, "i": "fea55e0c-bd88-4c10-b6ac-8ee5ede0f99f", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_datanode_producer_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "4843b611-a33d-4520-b833-df8302a18bbc", "type": "timeseries", "name": "TimeTick Lag Behind Now (Consumed All)", "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 516, "i": "4843b611-a33d-4520-b833-df8302a18bbc", "isResizable": true }, "targets": [ { "refId": "A", "expr": "avg(milvus_datanode_consume_tt_lag_ms{msg_type=\"all\"}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-avg" }, { "refId": "B", "expr": "max(milvus_datanode_consume_tt_lag_ms{msg_type=\"all\"}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-max" }, { "refId": "C", "expr": "min(milvus_datanode_consume_tt_lag_ms{msg_type=\"all\"}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-min" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "d1bacb48-9a9a-4598-a897-059ae8fe710a", "type": "timeseries", "name": "Produced TimeTick Lag Behind Now For DC", "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now, which send to DataCoord", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 522, "i": "d1bacb48-9a9a-4598-a897-059ae8fe710a", "isResizable": true }, "targets": [ { "refId": "A", "expr": "avg(milvus_datanode_produce_tt_lag_ms{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-avg" }, { "refId": "B", "expr": "max(milvus_datanode_produce_tt_lag_ms{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-max" }, { "refId": "C", "expr": "min(milvus_datanode_produce_tt_lag_ms{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}-min" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "2c55f664-549d-424f-8e36-9a6f4c2a2f4b", "type": "timeseries", "name": "Consumed Message Rate", "description": "per-second increasing rate of consuming message", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 522, "i": "2c55f664-549d-424f-8e36-9a6f4c2a2f4b", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_datanode_consume_msg_count{}[2m])/120) by (pod, node_id, msg_type)", "legend": "{{pod}}-{{node_id}}-{{msg_type}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "7cd0716f-352c-4ad4-b8af-7ddd254f33c7", "type": "timeseries", "name": "Consumed Message Rate", "description": "per-second increasing rate of consuming message", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 522, "i": "7cd0716f-352c-4ad4-b8af-7ddd254f33c7", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_datanode_consume_msg_count{}[2m])/120) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "a78c0bfc-b2c6-40ce-82b6-9d33b3a972e9", "type": "timeseries", "name": "Consumer Num", "description": "Total number of consumers created on the DataNode", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 528, "i": "a78c0bfc-b2c6-40ce-82b6-9d33b3a972e9", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_datanode_consumer_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "04339dcc-1005-4469-bc3c-4c932aad1bea", "type": "timeseries", "name": "Encode Buffer Latency", "description": "The 99th percentile and average latency of encode the data in the buffer over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 528, "i": "04339dcc-1005-4469-bc3c-4c932aad1bea", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_datanode_encode_buffer_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_encode_buffer_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "922f9828-9a25-4e4b-a965-e54e488e5646", "type": "timeseries", "name": "Msg Rows Consumed Rate", "description": "per-second increasing rate of messages consumed for insert and delete operation.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 528, "i": "922f9828-9a25-4e4b-a965-e54e488e5646", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_datanode_msg_rows_count{}[2m])/120) by (msg_type, pod, node_id)", "legend": "{{pod}}-{{node_id}}-{{msg_type}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "0c818412-35c4-425f-a244-da4889627e02", "type": "timeseries", "name": "Unflushed Segment Num", "description": "Total number of segment that has been not flushed.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 534, "i": "0c818412-35c4-425f-a244-da4889627e02", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_datanode_unflushed_segment_num{}) by (pod, node_id)", "legend": "{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "57100ec3-1ce6-43f8-a2f2-0fa1bc3986ef", "type": "timeseries", "name": "Autoflush Operate Rate", "description": "per-second increasing rate of auto flush operate.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 534, "i": "57100ec3-1ce6-43f8-a2f2-0fa1bc3986ef", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_datanode_autoflush_buffer_op_count{}[2m])/120) by (status, pod, node_id)", "legend": "{{pod}}-{{node_id}}-{{status}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "b0c0d920-6596-48b2-95e2-7dddc937407c", "type": "timeseries", "name": "Flush Data Size Rate", "description": "per-second increasing rate of each message that has been flushed.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 534, "i": "b0c0d920-6596-48b2-95e2-7dddc937407c", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_datanode_flushed_data_size{}[2m])/120) by (msg_type, pod, node_id)", "legend": "{{pod}}-{{node_id}}-{{msg_type}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "32b5254c-99ac-44ce-8158-d4a83a225358", "type": "timeseries", "name": "Flush Operate Rate", "description": "per-second increasing rate of flush operete.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 540, "i": "32b5254c-99ac-44ce-8158-d4a83a225358", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_datanode_flush_buffer_op_count{}[2m])/120) by (status, pod, node_id)", "legend": "{{pod}}-{{node_id}}-{{status}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "8bc53187-d9c2-4080-aeda-054ce8d45130", "type": "timeseries", "name": "Save Data Latency", "description": "The 99th percentile and average latency of writte the data in buffer to storage over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 540, "i": "8bc53187-d9c2-4080-aeda-054ce8d45130", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_save_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_datanode_save_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_save_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "80464313-6921-404e-aae5-a6713ce4552a", "type": "timeseries", "name": "Compaction Latency", "description": "The 99th percentile and average latency of compaction over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 546, "i": "80464313-6921-404e-aae5-a6713ce4552a", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_compaction_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_datanode_compaction_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_compaction_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "12e4a10d-893d-477d-b1ac-ed3276cd8f39", "type": "timeseries", "name": "Flush Request Rate", "description": "per-second increasing rate of flush requests.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 546, "i": "12e4a10d-893d-477d-b1ac-ed3276cd8f39", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_datanode_flush_req_count{}[2m])/120) by (status, pod, node_id)", "legend": "{{pod}}-{{node_id}}-{{status}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "23d6fede-2002-41a9-bb01-aa2a63aa0dbb", "type": "row", "name": "Index Coordinator", "collapsed": true, "layout": { "h": 1, "w": 24, "x": 0, "y": 552, "i": "23d6fede-2002-41a9-bb01-aa2a63aa0dbb", "isResizable": false }, "panels": [] }, { "version": "2.0.0", "id": "9469603d-59af-46ff-8543-ddee9b75b89e", "type": "timeseries", "name": "Index Request Rate", "description": "per-second increasing rate of building index requests recived by IndexCoord.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 553, "i": "9469603d-59af-46ff-8543-ddee9b75b89e", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_indexcoord_index_req_count{}[2m])/120) by (status)", "legend": "{{status}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "4adb542d-b6ba-49a1-bb9a-407574261095", "type": "timeseries", "name": "Index Task Count", "description": "Number of indexing tasks with different states in IndexCoord's meta", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 553, "i": "4adb542d-b6ba-49a1-bb9a-407574261095", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_indexcoord_index_task_count{}) by (index_task_status)", "legend": "{{index_task_status}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "0d9c3d84-1e2e-4135-8bfe-a5338c0628a5", "type": "timeseries", "name": "Index Node Num", "description": "Number of index nodes which has register with etcd", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 553, "i": "0d9c3d84-1e2e-4135-8bfe-a5338c0628a5", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(milvus_indexcoord_index_node_num{}) by (app_kubernetes_io_instance)", "legend": "total" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "e51a0b5e-7bf8-4da1-8e7a-a55f90784c24", "type": "row", "name": "Index Node", "collapsed": true, "layout": { "h": 1, "w": 24, "x": 0, "y": 559, "i": "e51a0b5e-7bf8-4da1-8e7a-a55f90784c24", "isResizable": false }, "panels": [] }, { "version": "2.0.0", "id": "2babfd44-ab71-46d4-bb99-4dd5640d4271", "type": "timeseries", "name": "Index Task Rate", "description": "per-second increasing rate of index tasks recevied by IndexNode", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 560, "i": "2babfd44-ab71-46d4-bb99-4dd5640d4271", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(increase(milvus_indexnode_index_task_count{}[2m])/120) by (status, pod, node_id)", "legend": "{{pod}}-{{node_id}}-{{status}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "c4625c00-a210-4267-b14e-ddd03823a280", "type": "timeseries", "name": "Load Field Latency", "description": "The 99th percentile and average latency ofr load the FieldData over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 560, "i": "c4625c00-a210-4267-b14e-ddd03823a280", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_load_field_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_indexnode_load_field_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_load_field_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "78019ff9-aabf-4862-b4b7-7d2c9fe2c73e", "type": "timeseries", "name": "Decode Field Latency", "description": "The 99th percentile and average latency of decod the FieldData over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 560, "i": "78019ff9-aabf-4862-b4b7-7d2c9fe2c73e", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_decode_field_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_indexnode_decode_field_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_decode_field_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "1743a73e-18ea-4c39-86b8-12cfac20fcea", "type": "timeseries", "name": "Build Index Latency", "description": "The 99th percentile and average latency of build index over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 566, "i": "1743a73e-18ea-4c39-86b8-12cfac20fcea", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_build_index_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_indexnode_build_index_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_build_index_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "f4779450-c9bf-4a28-a510-84a2f04cb621", "type": "timeseries", "name": "Encode Index Latency", "description": "The 99th percentile and average latency of encode index over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 566, "i": "f4779450-c9bf-4a28-a510-84a2f04cb621", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_encode_index_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_indexnode_encode_index_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_encode_index_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "9c818192-793d-42d1-92f3-96d074faf5b4", "type": "timeseries", "name": "Save Index Latency", "description": "The 99th percentile of latency for save index over the last 2 minutes.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 566, "i": "9c818192-793d-42d1-92f3-96d074faf5b4", "isResizable": true }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_save_index_latency_bucket{}[2m])))", "legend": "p99-{{pod}}-{{node_id}}" }, { "refId": "B", "expr": "sum(increase(milvus_indexnode_save_index_latency_sum{}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_save_index_latency_count{}[2m])) by(pod, node_id)", "legend": "avg-{{pod}}-{{node_id}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "35422665-e911-4396-8355-7df7bfa3295f", "type": "row", "name": "Runtime", "collapsed": true, "layout": { "h": 1, "w": 24, "x": 0, "y": 572, "i": "35422665-e911-4396-8355-7df7bfa3295f", "isResizable": false }, "panels": [] }, { "version": "2.0.0", "id": "0fad2248-a662-4313-b3c8-3e41ac958ac3", "type": "timeseries", "name": "CPU Usage Total", "description": "Total cpu usage of all milvus components.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 573, "i": "0fad2248-a662-4313-b3c8-3e41ac958ac3", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(rate(process_cpu_seconds_total{}[5m]))", "legend": "cpu usage" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "5b6315e0-fbe7-4c55-8eb3-f336475185ac", "type": "timeseries", "name": "Memory Total", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 573, "i": "5b6315e0-fbe7-4c55-8eb3-f336475185ac", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(process_resident_memory_bytes{})", "legend": "memory" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "e017c8f8-93f6-45a5-a5af-ee8d765724f8", "type": "timeseries", "name": "Goroutines Total", "description": "Number of goroutines that currently exist.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 573, "i": "e017c8f8-93f6-45a5-a5af-ee8d765724f8", "isResizable": true }, "targets": [ { "refId": "A", "expr": "sum(go_goroutines{})", "legend": "goroutines" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "62f1b7ff-7828-409c-8a04-f66de9f1486b", "type": "timeseries", "name": "CPU Usage", "description": "process cpu usage", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 579, "i": "62f1b7ff-7828-409c-8a04-f66de9f1486b", "isResizable": true }, "targets": [ { "refId": "A", "expr": "rate(process_cpu_seconds_total{}[5m])", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "a6dbf933-ba76-4263-b613-54a2961df2f2", "type": "timeseries", "name": "Memory", "description": "Process memory of milvus pod.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 579, "i": "a6dbf933-ba76-4263-b613-54a2961df2f2", "isResizable": true }, "targets": [ { "refId": "A", "expr": "process_resident_memory_bytes{}", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "23770045-f909-4901-a73d-42fc24d20ca1", "type": "timeseries", "name": "Goroutines", "description": "Number of goroutines that currently exist.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 579, "i": "23770045-f909-4901-a73d-42fc24d20ca1", "isResizable": true }, "targets": [ { "refId": "A", "expr": "go_goroutines{}", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "7fcb2225-ecf1-463d-88c3-8e0efb5d0df9", "type": "timeseries", "name": "Go Allocated Memory", "description": "Number of bytes allocated and still in use.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 585, "i": "7fcb2225-ecf1-463d-88c3-8e0efb5d0df9", "isResizable": true }, "targets": [ { "refId": "A", "expr": "go_memstats_alloc_bytes{}", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "5924a8d6-3c3e-4a16-af7c-20a5d7c2f5fa", "type": "timeseries", "name": "GC Max duration seconds", "description": "GC Max duration seconds.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 585, "i": "5924a8d6-3c3e-4a16-af7c-20a5d7c2f5fa", "isResizable": true }, "targets": [ { "refId": "A", "expr": "go_gc_duration_seconds{quantile=\"1\", }", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "99c9813f-a1f9-43ea-a15f-91e238b1b469", "type": "timeseries", "name": "OS Threads", "description": "Number of OS threads created.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 585, "i": "99c9813f-a1f9-43ea-a15f-91e238b1b469", "isResizable": true }, "targets": [ { "refId": "A", "expr": "go_threads{}", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "d938ca8b-1dec-4b33-adb1-1384b79ade95", "type": "timeseries", "name": "Next GC Bytes", "description": "Next GC Bytes.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 591, "i": "d938ca8b-1dec-4b33-adb1-1384b79ade95", "isResizable": true }, "targets": [ { "refId": "A", "expr": "go_memstats_next_gc_bytes{}", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "6b5311e1-1063-4c0b-857b-701cb9168aa3", "type": "timeseries", "name": "Process Opened Fds", "description": "Number of process opened fds.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 591, "i": "6b5311e1-1063-4c0b-857b-701cb9168aa3", "isResizable": true }, "targets": [ { "refId": "A", "expr": "process_open_fds{}", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "6201e47c-126d-4b78-8d08-79a19396e709", "type": "timeseries", "name": "GC CPU time", "description": "The fraction of this program's available CPU time used by the GC since the program started.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 591, "i": "6201e47c-126d-4b78-8d08-79a19396e709", "isResizable": true }, "targets": [ { "refId": "A", "expr": "go_memstats_gc_cpu_fraction{}", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "0804cdb0-d6ba-4c0c-ae63-b7beceb18382", "type": "timeseries", "name": "Memory Free Rate", "description": "Rate of memory frees.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 597, "i": "0804cdb0-d6ba-4c0c-ae63-b7beceb18382", "isResizable": true }, "targets": [ { "refId": "A", "expr": "rate(go_memstats_frees_total{}[5m])", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "4517708a-2d40-49e6-b38a-d958393b3117", "type": "timeseries", "name": "Go Memory mallocs rate.", "description": "Rate of go memory mallocs.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 597, "i": "4517708a-2d40-49e6-b38a-d958393b3117", "isResizable": true }, "targets": [ { "refId": "A", "expr": "rate(go_memstats_mallocs_total{}[2m])", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "30103057-7fc4-476f-afd1-5b507dfb9884", "type": "timeseries", "name": "Allocated Heap Objects", "description": "Number of allocated heap objects.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 597, "i": "30103057-7fc4-476f-afd1-5b507dfb9884", "isResizable": true }, "targets": [ { "refId": "A", "expr": "go_memstats_heap_objects{}", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "f7a8a997-5db3-4de6-8c2c-9faf0ad1a4fe", "type": "timeseries", "name": "Memory Allocated Rate", "description": "Rate of bytes allocated, even if freed.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 603, "i": "f7a8a997-5db3-4de6-8c2c-9faf0ad1a4fe", "isResizable": true }, "targets": [ { "refId": "A", "expr": "rate(go_memstats_alloc_bytes_total{}[5m])", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "34d2adcd-53a8-4832-b887-fb04122158b9", "type": "timeseries", "name": "Go Heap released", "description": "Number of heap bytes released to OS.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 603, "i": "34d2adcd-53a8-4832-b887-fb04122158b9", "isResizable": true }, "targets": [ { "refId": "A", "expr": "go_memstats_heap_released_bytes{}", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "876aa0db-14f7-4867-8a21-af97d3ab0261", "type": "timeseries", "name": "Go Heap idle", "description": "Number of heap bytes waiting to be used.", "links": [], "layout": { "h": 6, "w": 8, "x": 16, "y": 603, "i": "876aa0db-14f7-4867-8a21-af97d3ab0261", "isResizable": true }, "targets": [ { "refId": "A", "expr": "go_memstats_heap_idle_bytes{}", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "5202a77d-4fa5-4698-81fa-1535246db474", "type": "timeseries", "name": "Go Heap in Use", "description": "Number of heap bytes that be used.", "links": [], "layout": { "h": 6, "w": 8, "x": 0, "y": 609, "i": "5202a77d-4fa5-4698-81fa-1535246db474", "isResizable": true }, "targets": [ { "refId": "A", "expr": "go_memstats_heap_inuse_bytes{}", "legend": "{{pod}}" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" }, { "version": "2.0.0", "id": "2855acd5-4a27-4cba-abfa-5b595c8bb3e6", "type": "timeseries", "name": "Container Threads", "description": "Number of container threads created.", "links": [], "layout": { "h": 6, "w": 8, "x": 8, "y": 609, "i": "2855acd5-4a27-4cba-abfa-5b595c8bb3e6", "isResizable": true }, "targets": [ { "refId": "A", "expr": "container_threads{}", "legend": "" } ], "options": { "tooltip": { "mode": "all", "sort": "none" }, "legend": { "displayMode": "hidden" }, "thresholds": { "steps": [ { "color": "#634CD9", "value": null, "type": "base" } ] } }, "custom": { "version": "2.0.0", "drawStyle": "lines", "lineInterpolation": "linear", "fillOpacity": 0.5, "stack": "off" }, "datasourceCate": "prometheus", "datasourceValue": "${datasource}" } ], "datasourceValue": "hwcloud-bj4" } }