| 1 |
/* |
| 2 |
Copyright 2019 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package flowcontrol |
| 18 |
|
| 19 |
import ( |
| 20 |
"fmt" |
| 21 |
"io" |
| 22 |
"net/http" |
| 23 |
"sort" |
| 24 |
"strconv" |
| 25 |
"strings" |
| 26 |
"text/tabwriter" |
| 27 |
"time" |
| 28 |
|
| 29 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 30 |
"k8s.io/apimachinery/pkg/util/runtime" |
| 31 |
"k8s.io/apiserver/pkg/server/mux" |
| 32 |
"k8s.io/apiserver/pkg/util/flowcontrol/debug" |
| 33 |
) |
| 34 |
|
| 35 |
const ( |
| 36 |
queryIncludeRequestDetails = "includeRequestDetails" |
| 37 |
) |
| 38 |
|
| 39 |
func (cfgCtlr *configController) Install(c *mux.PathRecorderMux) { |
| 40 |
// TODO(yue9944882): handle "Accept" header properly |
| 41 |
// debugging dumps a CSV content for three levels of granularity |
| 42 |
// 1. row per priority-level |
| 43 |
c.UnlistedHandleFunc("/debug/api_priority_and_fairness/dump_priority_levels", cfgCtlr.dumpPriorityLevels) |
| 44 |
// 2. row per queue |
| 45 |
c.UnlistedHandleFunc("/debug/api_priority_and_fairness/dump_queues", cfgCtlr.dumpQueues) |
| 46 |
// 3. row per request |
| 47 |
c.UnlistedHandleFunc("/debug/api_priority_and_fairness/dump_requests", cfgCtlr.dumpRequests) |
| 48 |
} |
| 49 |
|
| 50 |
func (cfgCtlr *configController) dumpPriorityLevels(w http.ResponseWriter, r *http.Request) { |
| 51 |
cfgCtlr.lock.Lock() |
| 52 |
defer cfgCtlr.lock.Unlock() |
| 53 |
tabWriter := tabwriter.NewWriter(w, 8, 0, 1, ' ', 0) |
| 54 |
columnHeaders := []string{ |
| 55 |
"PriorityLevelName", // 1 |
| 56 |
"ActiveQueues", // 2 |
| 57 |
"IsIdle", // 3 |
| 58 |
"IsQuiescing", // 4 |
| 59 |
"WaitingRequests", // 5 |
| 60 |
"ExecutingRequests", // 6 |
| 61 |
"DispatchedRequests", // 7 |
| 62 |
"RejectedRequests", // 8 |
| 63 |
"TimedoutRequests", // 9 |
| 64 |
"CancelledRequests", // 10 |
| 65 |
} |
| 66 |
tabPrint(tabWriter, rowForHeaders(columnHeaders)) |
| 67 |
endLine(tabWriter) |
| 68 |
plNames := make([]string, 0, len(cfgCtlr.priorityLevelStates)) |
| 69 |
for plName := range cfgCtlr.priorityLevelStates { |
| 70 |
plNames = append(plNames, plName) |
| 71 |
} |
| 72 |
sort.Strings(plNames) |
| 73 |
for i := range plNames { |
| 74 |
plState, ok := cfgCtlr.priorityLevelStates[plNames[i]] |
| 75 |
if !ok { |
| 76 |
continue |
| 77 |
} |
| 78 |
|
| 79 |
queueSetDigest := plState.queues.Dump(false) |
| 80 |
activeQueueNum := 0 |
| 81 |
for _, q := range queueSetDigest.Queues { |
| 82 |
if len(q.Requests) > 0 { |
| 83 |
activeQueueNum++ |
| 84 |
} |
| 85 |
} |
| 86 |
|
| 87 |
tabPrint(tabWriter, rowForPriorityLevel( |
| 88 |
plState.pl.Name, // 1 |
| 89 |
activeQueueNum, // 2 |
| 90 |
plState.queues.IsIdle(), // 3 |
| 91 |
plState.quiescing, // 4 |
| 92 |
queueSetDigest.Waiting, // 5 |
| 93 |
queueSetDigest.Executing, // 6 |
| 94 |
queueSetDigest.Dispatched, // 7 |
| 95 |
queueSetDigest.Rejected, // 8 |
| 96 |
queueSetDigest.Timedout, // 9 |
| 97 |
queueSetDigest.Cancelled, // 10 |
| 98 |
)) |
| 99 |
endLine(tabWriter) |
| 100 |
} |
| 101 |
runtime.HandleError(tabWriter.Flush()) |
| 102 |
} |
| 103 |
|
| 104 |
func (cfgCtlr *configController) dumpQueues(w http.ResponseWriter, r *http.Request) { |
| 105 |
cfgCtlr.lock.Lock() |
| 106 |
defer cfgCtlr.lock.Unlock() |
| 107 |
tabWriter := tabwriter.NewWriter(w, 8, 0, 1, ' ', 0) |
| 108 |
columnHeaders := []string{ |
| 109 |
"PriorityLevelName", // 1 |
| 110 |
"Index", // 2 |
| 111 |
"PendingRequests", // 3 |
| 112 |
"ExecutingRequests", // 4 |
| 113 |
"SeatsInUse", // 5 |
| 114 |
"NextDispatchR", // 6 |
| 115 |
"InitialSeatsSum", // 7 |
| 116 |
"MaxSeatsSum", // 8 |
| 117 |
"TotalWorkSum", // 9 |
| 118 |
} |
| 119 |
tabPrint(tabWriter, rowForHeaders(columnHeaders)) |
| 120 |
endLine(tabWriter) |
| 121 |
for _, plState := range cfgCtlr.priorityLevelStates { |
| 122 |
queueSetDigest := plState.queues.Dump(false) |
| 123 |
for i, q := range queueSetDigest.Queues { |
| 124 |
tabPrint(tabWriter, row( |
| 125 |
plState.pl.Name, // 1 - "PriorityLevelName" |
| 126 |
strconv.Itoa(i), // 2 - "Index" |
| 127 |
strconv.Itoa(len(q.Requests)), // 3 - "PendingRequests" |
| 128 |
strconv.Itoa(q.ExecutingRequests), // 4 - "ExecutingRequests" |
| 129 |
strconv.Itoa(q.SeatsInUse), // 5 - "SeatsInUse" |
| 130 |
q.NextDispatchR, // 6 - "NextDispatchR" |
| 131 |
strconv.Itoa(q.QueueSum.InitialSeatsSum), // 7 - "InitialSeatsSum" |
| 132 |
strconv.Itoa(q.QueueSum.MaxSeatsSum), // 8 - "MaxSeatsSum" |
| 133 |
q.QueueSum.TotalWorkSum, // 9 - "TotalWorkSum" |
| 134 |
)) |
| 135 |
endLine(tabWriter) |
| 136 |
} |
| 137 |
} |
| 138 |
runtime.HandleError(tabWriter.Flush()) |
| 139 |
} |
| 140 |
|
| 141 |
func (cfgCtlr *configController) dumpRequests(w http.ResponseWriter, r *http.Request) { |
| 142 |
cfgCtlr.lock.Lock() |
| 143 |
defer cfgCtlr.lock.Unlock() |
| 144 |
|
| 145 |
includeRequestDetails := len(r.URL.Query().Get(queryIncludeRequestDetails)) > 0 |
| 146 |
|
| 147 |
tabWriter := tabwriter.NewWriter(w, 8, 0, 1, ' ', 0) |
| 148 |
tabPrint(tabWriter, rowForHeaders([]string{ |
| 149 |
"PriorityLevelName", // 1 |
| 150 |
"FlowSchemaName", // 2 |
| 151 |
"QueueIndex", // 3 |
| 152 |
"RequestIndexInQueue", // 4 |
| 153 |
"FlowDistingsher", // 5 |
| 154 |
"ArriveTime", // 6 |
| 155 |
"InitialSeats", // 7 |
| 156 |
"FinalSeats", // 8 |
| 157 |
"AdditionalLatency", // 9 |
| 158 |
"StartTime", // 10 |
| 159 |
})) |
| 160 |
if includeRequestDetails { |
| 161 |
continueLine(tabWriter) |
| 162 |
tabPrint(tabWriter, rowForHeaders([]string{ |
| 163 |
"UserName", // 11 |
| 164 |
"Verb", // 12 |
| 165 |
"APIPath", // 13 |
| 166 |
"Namespace", // 14 |
| 167 |
"Name", // 15 |
| 168 |
"APIVersion", // 16 |
| 169 |
"Resource", // 17 |
| 170 |
"SubResource", // 18 |
| 171 |
})) |
| 172 |
} |
| 173 |
endLine(tabWriter) |
| 174 |
for _, plState := range cfgCtlr.priorityLevelStates { |
| 175 |
queueSetDigest := plState.queues.Dump(includeRequestDetails) |
| 176 |
dumpRequest := func(iq, ir int, r debug.RequestDump) { |
| 177 |
tabPrint(tabWriter, row( |
| 178 |
plState.pl.Name, // 1 |
| 179 |
r.MatchedFlowSchema, // 2 |
| 180 |
strconv.Itoa(iq), // 3 |
| 181 |
strconv.Itoa(ir), // 4 |
| 182 |
r.FlowDistinguisher, // 5 |
| 183 |
r.ArriveTime.UTC().Format(time.RFC3339Nano), // 6 |
| 184 |
strconv.Itoa(int(r.WorkEstimate.InitialSeats)), // 7 |
| 185 |
strconv.Itoa(int(r.WorkEstimate.FinalSeats)), // 8 |
| 186 |
r.WorkEstimate.AdditionalLatency.String(), // 9 |
| 187 |
r.StartTime.UTC().Format(time.RFC3339Nano), // 10 |
| 188 |
)) |
| 189 |
if includeRequestDetails { |
| 190 |
continueLine(tabWriter) |
| 191 |
tabPrint(tabWriter, rowForRequestDetails( |
| 192 |
r.UserName, // 11 |
| 193 |
r.RequestInfo.Verb, // 12 |
| 194 |
r.RequestInfo.Path, // 13 |
| 195 |
r.RequestInfo.Namespace, // 14 |
| 196 |
r.RequestInfo.Name, // 15 |
| 197 |
schema.GroupVersion{ |
| 198 |
Group: r.RequestInfo.APIGroup, |
| 199 |
Version: r.RequestInfo.APIVersion, |
| 200 |
}.String(), // 16 |
| 201 |
r.RequestInfo.Resource, // 17 |
| 202 |
r.RequestInfo.Subresource, // 18 |
| 203 |
)) |
| 204 |
} |
| 205 |
endLine(tabWriter) |
| 206 |
} |
| 207 |
for iq, q := range queueSetDigest.Queues { |
| 208 |
for ir, r := range q.Requests { |
| 209 |
dumpRequest(iq, ir, r) |
| 210 |
} |
| 211 |
for _, r := range q.RequestsExecuting { |
| 212 |
dumpRequest(iq, -1, r) |
| 213 |
} |
| 214 |
} |
| 215 |
for _, r := range queueSetDigest.QueuelessExecutingRequests { |
| 216 |
dumpRequest(-1, -1, r) |
| 217 |
} |
| 218 |
} |
| 219 |
runtime.HandleError(tabWriter.Flush()) |
| 220 |
} |
| 221 |
|
| 222 |
func tabPrint(w io.Writer, row string) { |
| 223 |
_, err := fmt.Fprint(w, row) |
| 224 |
runtime.HandleError(err) |
| 225 |
} |
| 226 |
|
| 227 |
func continueLine(w io.Writer) { |
| 228 |
_, err := fmt.Fprint(w, ",\t") |
| 229 |
runtime.HandleError(err) |
| 230 |
} |
| 231 |
func endLine(w io.Writer) { |
| 232 |
_, err := fmt.Fprint(w, "\n") |
| 233 |
runtime.HandleError(err) |
| 234 |
} |
| 235 |
|
| 236 |
func rowForHeaders(headers []string) string { |
| 237 |
return row(headers...) |
| 238 |
} |
| 239 |
|
| 240 |
func rowForPriorityLevel(plName string, activeQueues int, isIdle, isQuiescing bool, waitingRequests, executingRequests int, |
| 241 |
dispatchedReqeusts, rejectedRequests, timedoutRequests, cancelledRequests int) string { |
| 242 |
return row( |
| 243 |
plName, |
| 244 |
strconv.Itoa(activeQueues), |
| 245 |
strconv.FormatBool(isIdle), |
| 246 |
strconv.FormatBool(isQuiescing), |
| 247 |
strconv.Itoa(waitingRequests), |
| 248 |
strconv.Itoa(executingRequests), |
| 249 |
strconv.Itoa(dispatchedReqeusts), |
| 250 |
strconv.Itoa(rejectedRequests), |
| 251 |
strconv.Itoa(timedoutRequests), |
| 252 |
strconv.Itoa(cancelledRequests), |
| 253 |
) |
| 254 |
} |
| 255 |
|
| 256 |
func rowForRequestDetails(username, verb, path, namespace, name, apiVersion, resource, subResource string) string { |
| 257 |
return row( |
| 258 |
username, |
| 259 |
verb, |
| 260 |
path, |
| 261 |
namespace, |
| 262 |
name, |
| 263 |
apiVersion, |
| 264 |
resource, |
| 265 |
subResource, |
| 266 |
) |
| 267 |
} |
| 268 |
|
| 269 |
func row(columns ...string) string { |
| 270 |
return strings.Join(columns, ",\t") |
| 271 |
} |