Debian Codegraph @

pkg/server/filters/timeout_test.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package filters
18
19 import (
20 "bytes"
21 "context"
22 "crypto/tls"
23 "crypto/x509"
24 "encoding/json"
25 "fmt"
26 "io/ioutil"
27 "net"
28 "net/http"
29 "net/http/httptest"
30 "net/http/httptrace"
31 "reflect"
32 "strings"
33 "sync"
34 "testing"
35 "time"
36
37 "github.com/google/go-cmp/cmp"
38 "golang.org/x/net/http2"
39
40 apierrors "k8s.io/apimachinery/pkg/api/errors"
41 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
42 "k8s.io/apimachinery/pkg/runtime/schema"
43 "k8s.io/apimachinery/pkg/util/runtime"
44 "k8s.io/apimachinery/pkg/util/sets"
45 "k8s.io/apiserver/pkg/endpoints/request"
46 "k8s.io/apiserver/pkg/endpoints/responsewriter"
47 "k8s.io/klog/v2"
48 )
49
50 type recorder struct {
51 lock sync.Mutex
52 count int
53 }
54
55 func (r *recorder) Record() {
56 r.lock.Lock()
57 defer r.lock.Unlock()
58 r.count++
59 }
60
61 func (r *recorder) Count() int {
62 r.lock.Lock()
63 defer r.lock.Unlock()
64 return r.count
65 }
66
67 func newHandler(responseCh <-chan string, panicCh <-chan interface{}, writeErrCh chan<- error) http.HandlerFunc {
68 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
69 select {
70 case resp := <-responseCh:
71 _, err := w.Write([]byte(resp))
72 writeErrCh <- err
73 case panicReason := <-panicCh:
74 panic(panicReason)
75 }
76 })
77 }
78
79 func TestTimeout(t *testing.T) {
80 origReallyCrash := runtime.ReallyCrash
81 runtime.ReallyCrash = false
82 defer func() {
83 runtime.ReallyCrash = origReallyCrash
84 }()
85
86 sendResponse := make(chan string, 1)
87 doPanic := make(chan interface{}, 1)
88 writeErrors := make(chan error, 1)
89 gotPanic := make(chan interface{}, 1)
90 timeout := make(chan time.Time, 1)
91 resp := "test response"
92 timeoutErr := apierrors.NewServerTimeout(schema.GroupResource{Group: "foo", Resource: "bar"}, "get", 0)
93 record := &recorder{}
94
95 var ctx context.Context
96 withDeadline := func(handler http.Handler) http.Handler {
97 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
98 req = req.WithContext(ctx)
99 handler.ServeHTTP(w, req)
100 })
101 }
102
103 handler := newHandler(sendResponse, doPanic, writeErrors)
104 ts := httptest.NewServer(withDeadline(withPanicRecovery(
105 WithTimeout(handler, func(req *http.Request) (*http.Request, bool, func(), *apierrors.StatusError) {
106 return req, false, record.Record, timeoutErr
107 }), func(w http.ResponseWriter, req *http.Request, err interface{}) {
108 gotPanic <- err
109 http.Error(w, "This request caused apiserver to panic. Look in the logs for details.", http.StatusInternalServerError)
110 }),
111 ))
112 defer ts.Close()
113
114 // No timeouts
115 ctx = context.Background()
116 sendResponse <- resp
117 res, err := http.Get(ts.URL)
118 if err != nil {
119 t.Fatal(err)
120 }
121 if res.StatusCode != http.StatusOK {
122 t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusOK)
123 }
124 body, _ := ioutil.ReadAll(res.Body)
125 if string(body) != resp {
126 t.Errorf("got body %q; expected %q", string(body), resp)
127 }
128 if err := <-writeErrors; err != nil {
129 t.Errorf("got unexpected Write error on first request: %v", err)
130 }
131 if record.Count() != 0 {
132 t.Errorf("invoked record method: %#v", record)
133 }
134
135 // Times out
136 ctx, cancel := context.WithCancel(context.Background())
137 cancel()
138 timeout <- time.Time{}
139 res, err = http.Get(ts.URL)
140 if err != nil {
141 t.Fatal(err)
142 }
143 if res.StatusCode != http.StatusGatewayTimeout {
144 t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusGatewayTimeout)
145 }
146 body, _ = ioutil.ReadAll(res.Body)
147 status := &metav1.Status{}
148 if err := json.Unmarshal(body, status); err != nil {
149 t.Fatal(err)
150 }
151 if !reflect.DeepEqual(status, &timeoutErr.ErrStatus) {
152 t.Errorf("unexpected object: %s", cmp.Diff(&timeoutErr.ErrStatus, status))
153 }
154 if record.Count() != 1 {
155 t.Errorf("did not invoke record method: %#v", record)
156 }
157
158 // Now try to send a response
159 ctx = context.Background()
160 sendResponse <- resp
161 if err := <-writeErrors; err != http.ErrHandlerTimeout {
162 t.Errorf("got Write error of %v; expected %v", err, http.ErrHandlerTimeout)
163 }
164
165 // Panics
166 doPanic <- "inner handler panics"
167 res, err = http.Get(ts.URL)
168 if err != nil {
169 t.Fatal(err)
170 }
171 if res.StatusCode != http.StatusInternalServerError {
172 t.Errorf("got res.StatusCode %d; expected %d due to panic", res.StatusCode, http.StatusInternalServerError)
173 }
174 select {
175 case err := <-gotPanic:
176 msg := fmt.Sprintf("%v", err)
177 if !strings.Contains(msg, "newHandler") {
178 t.Errorf("expected line with root cause panic in the stack trace, but didn't: %v", err)
179 }
180 case <-time.After(30 * time.Second):
181 t.Fatalf("expected to see a handler panic, but didn't")
182 }
183
184 // Panics with http.ErrAbortHandler
185 ctx = context.Background()
186 doPanic <- http.ErrAbortHandler
187 res, err = http.Get(ts.URL)
188 if err != nil {
189 t.Fatal(err)
190 }
191 if res.StatusCode != http.StatusInternalServerError {
192 t.Errorf("got res.StatusCode %d; expected %d due to panic", res.StatusCode, http.StatusInternalServerError)
193 }
194 select {
195 case err := <-gotPanic:
196 if err != http.ErrAbortHandler {
197 t.Errorf("expected unwrapped http.ErrAbortHandler, got %#v", err)
198 }
199 case <-time.After(30 * time.Second):
200 t.Fatalf("expected to see a handler panic, but didn't")
201 }
202 }
203
204 func TestTimeoutHeaders(t *testing.T) {
205 origReallyCrash := runtime.ReallyCrash
206 runtime.ReallyCrash = false
207 defer func() {
208 runtime.ReallyCrash = origReallyCrash
209 }()
210
211 ctx, cancel := context.WithCancel(context.Background())
212 defer cancel()
213
214 withDeadline := func(handler http.Handler) http.Handler {
215 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
216 handler.ServeHTTP(w, req.WithContext(ctx))
217 })
218 }
219
220 postTimeoutCh := make(chan struct{})
221 ts := httptest.NewServer(
222 withDeadline(
223 WithTimeout(
224 http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
225 h := w.Header()
226 // trigger the timeout
227 cancel()
228 // keep mutating response Headers until the request times out
229 for {
230 select {
231 case <-postTimeoutCh:
232 return
233 default:
234 h.Set("Test", "post")
235 }
236 }
237 }),
238 func(req *http.Request) (*http.Request, bool, func(), *apierrors.StatusError) {
239 return req, false, func() { close(postTimeoutCh) }, apierrors.NewServerTimeout(schema.GroupResource{Group: "foo", Resource: "bar"}, "get", 0)
240 },
241 ),
242 ),
243 )
244 defer ts.Close()
245
246 res, err := http.Get(ts.URL)
247 if err != nil {
248 t.Fatal(err)
249 }
250 if res.StatusCode != http.StatusGatewayTimeout {
251 t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusGatewayTimeout)
252 }
253 res.Body.Close()
254 }
255
256 func TestTimeoutRequestHeaders(t *testing.T) {
257 origReallyCrash := runtime.ReallyCrash
258 runtime.ReallyCrash = false
259 defer func() {
260 runtime.ReallyCrash = origReallyCrash
261 }()
262
263 ctx, cancel := context.WithCancel(context.Background())
264 defer cancel()
265
266 // Add dummy request info, otherwise we skip postTimeoutFn
267 ctx = request.WithRequestInfo(ctx, &request.RequestInfo{})
268
269 withDeadline := func(handler http.Handler) http.Handler {
270 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
271 handler.ServeHTTP(w, req.WithContext(ctx))
272 })
273 }
274
275 testDone := make(chan struct{})
276 defer close(testDone)
277 ts := httptest.NewServer(
278 withDeadline(
279 WithTimeoutForNonLongRunningRequests(
280 http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
281 // trigger the timeout
282 cancel()
283 // mutate request Headers
284 // Authorization filter does it for example
285 for {
286 select {
287 case <-testDone:
288 return
289 default:
290 req.Header.Set("Test", "post")
291 }
292 }
293 }),
294 func(r *http.Request, requestInfo *request.RequestInfo) bool {
295 return false
296 },
297 ),
298 ),
299 )
300 defer ts.Close()
301
302 client := &http.Client{}
303 req, err := http.NewRequest(http.MethodPatch, ts.URL, nil)
304 if err != nil {
305 t.Fatal(err)
306 }
307 res, err := client.Do(req)
308 if err != nil {
309 t.Fatal(err)
310 }
311 if actual, expected := res.StatusCode, http.StatusGatewayTimeout; actual != expected {
312 t.Errorf("got status code %d; expected %d", actual, expected)
313 }
314 res.Body.Close()
315 }
316
317 func TestTimeoutWithLogging(t *testing.T) {
318 origReallyCrash := runtime.ReallyCrash
319 runtime.ReallyCrash = false
320 defer func() {
321 runtime.ReallyCrash = origReallyCrash
322 }()
323
324 ctx, cancel := context.WithCancel(context.Background())
325 defer cancel()
326
327 withDeadline := func(handler http.Handler) http.Handler {
328 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
329 handler.ServeHTTP(w, req.WithContext(ctx))
330 })
331 }
332
333 testDone := make(chan struct{})
334 defer close(testDone)
335 ts := httptest.NewServer(
336 WithHTTPLogging(
337 withDeadline(
338 WithTimeoutForNonLongRunningRequests(
339 http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
340 // trigger the timeout
341 cancel()
342 // mutate request Headers
343 // Authorization filter does it for example
344 for {
345 select {
346 case <-testDone:
347 return
348 default:
349 req.Header.Set("Test", "post")
350 }
351 }
352 }),
353 func(r *http.Request, requestInfo *request.RequestInfo) bool {
354 return false
355 },
356 ),
357 ),
358 ),
359 )
360 defer ts.Close()
361
362 client := &http.Client{}
363 req, err := http.NewRequest(http.MethodPatch, ts.URL, nil)
364 if err != nil {
365 t.Fatal(err)
366 }
367 res, err := client.Do(req)
368 if err != nil {
369 t.Fatal(err)
370 }
371 if actual, expected := res.StatusCode, http.StatusGatewayTimeout; actual != expected {
372 t.Errorf("got status code %d; expected %d", actual, expected)
373 }
374 res.Body.Close()
375 }
376
377 func TestErrConnKilled(t *testing.T) {
378 var buf bytes.Buffer
379 klog.SetOutput(&buf)
380 klog.LogToStderr(false)
381 defer klog.LogToStderr(true)
382
383 handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
384 // this error must be ignored by the WithPanicRecovery handler
385 // it is thrown by WithTimeoutForNonLongRunningRequests handler when a response has been already sent to the client and the handler timed out
386 // panicking with http.ErrAbortHandler also suppresses logging of a stack trace to the server's error log and closes the underlying connection
387 w.Write([]byte("hello from the handler"))
388 panic(http.ErrAbortHandler)
389 })
390 resolver := &request.RequestInfoFactory{
391 APIPrefixes: sets.NewString("api", "apis"),
392 GrouplessAPIPrefixes: sets.NewString("api"),
393 }
394
395 ts := httptest.NewServer(WithPanicRecovery(handler, resolver))
396 defer ts.Close()
397
398 _, err := http.Get(ts.URL)
399 if err == nil {
400 t.Fatal("expected to receive an error")
401 }
402
403 klog.Flush()
404 klog.SetOutput(&bytes.Buffer{}) // prevent further writes into buf
405 capturedOutput := buf.String()
406
407 // We don't expect stack trace from the panic to be included in the log.
408 if isStackTraceLoggedByRuntime(capturedOutput) {
409 t.Errorf("unexpected stack trace in log, actual = %v", capturedOutput)
410 }
411 // For the sake of simplicity and clarity this matches the full log line.
412 // This is not part of the Kubernetes API and could change.
413 if !strings.Contains(capturedOutput, `"Timeout or abort while handling" logger="UnhandledError" method="GET" URI="/" auditID=""`) {
414 t.Errorf("unexpected output captured actual = %v", capturedOutput)
415 }
416 }
417
418 type panicOnNonReuseTransport struct {
419 Transport http.RoundTripper
420 gotConnSeen bool
421 }
422
423 func (t *panicOnNonReuseTransport) RoundTrip(req *http.Request) (*http.Response, error) {
424 return t.Transport.RoundTrip(req)
425 }
426
427 func (t *panicOnNonReuseTransport) GotConn(info httptrace.GotConnInfo) {
428 if !t.gotConnSeen {
429 t.gotConnSeen = true
430 return
431 }
432 if !info.Reused {
433 panic(fmt.Sprintf("expected the connection to be reused, info %#v", info))
434 }
435 }
436
437 // TestErrConnKilledHTTP2 check if HTTP/2 connection is not closed when an HTTP handler panics
438 // The net/http library recovers the panic and sends an HTTP/2 RST_STREAM.
439 func TestErrConnKilledHTTP2(t *testing.T) {
440 var buf bytes.Buffer
441 klog.SetOutput(&buf)
442 klog.LogToStderr(false)
443 defer klog.LogToStderr(true)
444
445 handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
446 // this error must be ignored by the WithPanicRecovery handler
447 // it is thrown by WithTimeoutForNonLongRunningRequests handler when a response has been already sent to the client and the handler timed out
448 // panicking with http.ErrAbortHandler also suppresses logging of a stack trace to the server's error log and closes the underlying connection
449 w.Write([]byte("hello from the handler"))
450 panic(http.ErrAbortHandler)
451 })
452 resolver := &request.RequestInfoFactory{
453 APIPrefixes: sets.NewString("api", "apis"),
454 GrouplessAPIPrefixes: sets.NewString("api"),
455 }
456
457 // test server
458 ts := httptest.NewUnstartedServer(WithPanicRecovery(handler, resolver))
459 tsCert, err := tls.X509KeyPair(tsCrt, tsKey)
460 if err != nil {
461 t.Fatalf("backend: invalid x509/key pair: %v", err)
462 }
463 ts.TLS = &tls.Config{
464 Certificates: []tls.Certificate{tsCert},
465 NextProtos: []string{http2.NextProtoTLS},
466 }
467 ts.StartTLS()
468 defer ts.Close()
469
470 newServerRequest := func(tr *panicOnNonReuseTransport) *http.Request {
471 req, _ := http.NewRequest("GET", fmt.Sprintf("https://127.0.0.1:%d", ts.Listener.Addr().(*net.TCPAddr).Port), nil)
472 trace := &httptrace.ClientTrace{
473 GotConn: tr.GotConn,
474 }
475 return req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
476 }
477
478 // client
479 clientCACertPool := x509.NewCertPool()
480 clientCACertPool.AppendCertsFromPEM(tsCrt)
481 clientTLSConfig := &tls.Config{
482 RootCAs: clientCACertPool,
483 NextProtos: []string{http2.NextProtoTLS},
484 }
485 tr := &panicOnNonReuseTransport{}
486 client := &http.Client{}
487 tr.Transport = &http2.Transport{
488 TLSClientConfig: clientTLSConfig,
489 }
490 client.Transport = tr
491
492 // act
493 _, err = client.Do(newServerRequest(tr))
494 if err == nil {
495 t.Fatal("expected to receive an error")
496 }
497
498 klog.Flush()
499 klog.SetOutput(&bytes.Buffer{}) // prevent further writes into buf
500 capturedOutput := buf.String()
501
502 // We don't expect stack trace from the panic to be included in the log.
503 if isStackTraceLoggedByRuntime(capturedOutput) {
504 t.Errorf("unexpected stack trace in log, actual = %v", capturedOutput)
505 }
506 // For the sake of simplicity and clarity this matches the full log line.
507 // This is not part of the Kubernetes API and could change.
508 if !strings.Contains(capturedOutput, `"Timeout or abort while handling" logger="UnhandledError" method="GET" URI="/" auditID=""`) {
509 t.Errorf("unexpected output captured actual = %v", capturedOutput)
510 }
511
512 // make another req to the server
513 // the connection should be reused
514 // the client uses a custom transport that checks and panics when the con wasn't reused.
515 _, err = client.Do(newServerRequest(tr))
516 if err == nil {
517 t.Fatal("expected to receive an error")
518 }
519 }
520
521 func TestResponseWriterDecorator(t *testing.T) {
522 decorator := &baseTimeoutWriter{
523 w: &responsewriter.FakeResponseWriter{},
524 }
525 var w http.ResponseWriter = decorator
526
527 if inner := w.(responsewriter.UserProvidedDecorator).Unwrap(); inner != decorator.w {
528 t.Errorf("Expected the decorator to return the inner http.ResponseWriter object")
529 }
530 }
531
532 func isStackTraceLoggedByRuntime(message string) bool {
533 // Check the captured output for the following patterns to find out if the
534 // stack trace is included in the log:
535 // - 'Observed a panic' (apimachinery runtime.go logs panic with this message)
536 // - 'goroutine 44 [running]:' (stack trace always starts with this)
537 if strings.Contains(message, "Observed a panic") &&
538 strings.Contains(message, "goroutine") &&
539 strings.Contains(message, "[running]:") {
540 return true
541 }
542
543 return false
544 }
545
546 var tsCrt = []byte(`-----BEGIN CERTIFICATE-----
547 MIIDTjCCAjagAwIBAgIJAJdcQEBN2CjoMA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNV
548 BAYTAlBMMQ8wDQYDVQQIDAZQb2xhbmQxDzANBgNVBAcMBkdkYW5zazELMAkGA1UE
549 CgwCU0sxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yMDA5MjgxMTU1MjhaFw0zMDA5
550 MjYxMTU1MjhaMFAxCzAJBgNVBAYTAlBMMQ8wDQYDVQQIDAZQb2xhbmQxDzANBgNV
551 BAcMBkdkYW5zazELMAkGA1UECgwCU0sxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIw
552 DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMr6b/uTHkIDEd88x3t3jnroOVwh
553 jWMwZ6qXN2NV/If1L9FNvtoZzZi6yCDE1uLdD1kWZ0R2XOPEwUPn+Z8A/lg9kF8J
554 GloLCF8q+XeYp8aWRKzwtdi+MPaKFf0wsuxEEHU4pypFrszNY0yLRbWAbMtgBFy0
555 KhyNGahFO9V69cRHUj6EJ9kSBg0nG5bsypon2rinzKpUrzAEl2MbM3F34Zit5yOv
556 rYQcbDME+9XmOJPD97XBvMZCbmPnmpst3tX7ZhdKgSKtIjoYt+d//wtPMXOhrRzM
557 xcc6HuIHAovtB4kvZl5wvVU8ra8DKZviYyjfW36kQHo+yFwP3XXZFWezZi0CAwEA
558 AaMrMCkwCQYDVR0TBAIwADALBgNVHQ8EBAMCBaAwDwYDVR0RBAgwBocEfwAAATAN
559 BgkqhkiG9w0BAQsFAAOCAQEAMoAlhZ6yRwPmFL2ql9ZYWqaPu2NC4dXYV6kUIXUA
560 pG3IqFWb3L4ePkgYBMDlJGpAJcXlSSgEGiLj+3qQojawIMzzxmqr2KX888S5Mr+9
561 I1qseRwcftwYqV/ewJSWE90HJ21pb1ixA6bSRJLV7DyxO6zKsdVJ4xIvehZtGbux
562 0RTf+8zUx8z2Goy1GUztOIqfMRt1P1hlQG0uvYsGQM84HO4+YhFwejrGaj8ajpgF
563 uo3B8BVHeh57FNGE6C45NkFGHq3tkNLMdAa32Az8DDvPmsJuycf6vgIfBEQxLZSF
564 OUKrKmtfdFv4XrInqFUYBYp5GkL8SGM2wmv6aSw9Aju4lA==
565 -----END CERTIFICATE-----`)
566
567 var tsKey = []byte(`-----BEGIN PRIVATE KEY-----
568 MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDK+m/7kx5CAxHf
569 PMd7d4566DlcIY1jMGeqlzdjVfyH9S/RTb7aGc2YusggxNbi3Q9ZFmdEdlzjxMFD
570 5/mfAP5YPZBfCRpaCwhfKvl3mKfGlkSs8LXYvjD2ihX9MLLsRBB1OKcqRa7MzWNM
571 i0W1gGzLYARctCocjRmoRTvVevXER1I+hCfZEgYNJxuW7MqaJ9q4p8yqVK8wBJdj
572 GzNxd+GYrecjr62EHGwzBPvV5jiTw/e1wbzGQm5j55qbLd7V+2YXSoEirSI6GLfn
573 f/8LTzFzoa0czMXHOh7iBwKL7QeJL2ZecL1VPK2vAymb4mMo31t+pEB6PshcD911
574 2RVns2YtAgMBAAECggEAA2Qx0MtBeyrf9pHmZ1q1B7qvkqmA2kJpyQDjzQYXxRHE
575 rcOVx8EcnUupolqHmJzG798e9JbhsHCOJhtPIWf71++XZO8bAJwklKp8JpJnYzsJ
576 hLY0450x5jyiZ2uT4by1Za//owYtCID6AsJk9MZjivZcvEvKVFXLMvONL2DxkEj1
577 KaGQJh6/GT4jtNX07bW9+5w069KAAf+BNuqv8+Y/FseV3ovlpLTKjMV9xCCp9i62
578 PJs/hs5CW2X+JCE7OCLsAiu0JTpXYyHcLwYwnCONdvj6veiMWjRyNDr3ew5NeZNf
579 nGU4WX7mXjPd/1OvzJy6iyrBlAA63ZfFZYjWQnfsIQKBgQDmo3AMIr+9rE79NnaD
580 woQyO539YSO45KSM39/Xrp/NJVpOxtzgZrYo7O6f6kQ3S5zQOddy9Oj7gN3RXhZ7
581 Vi+Oja78ig7KUrqxcBiBGRsKZGm5CGdZ0EFd3rIEh4Qb+f+2c4f+6NWANb4kwvfq
582 K24c1o71+77lEVlzE2/L33K+mQKBgQDhTFr/f2e9gnRNX9bjF4p7DQI0RsFADjx0
583 jgJYHfm/lCIdH9vf6SmmvJv2E76Bqx9XVilhav/egqKO/wzJWHyNo2RFBXNqfwoF
584 UxRZKgqhcU52y2LKAYoTYfodktatZk74rinMDLmA6arnlAWQELk3Mx48DlND43Zc
585 DUHTKcJEtQKBgQDYdL1c9mPjnEqJxMqXwEAXcPJG8hr3lMaGXDoVjxL1EsBdvK9h
586 f6QoZq1RsiiRiMpEdnSotAfQutHzhA0vdeSuMnTvGJbm9Zu3mc+1oZ1KNJEwkh2F
587 Ijmm4rFKJPEs3IVMc8NHzrdJW6b3k2/e+yGduRR08e7nx0+e+7fpq+1hyQKBgHY9
588 l4h9+hkYjSdKhEG8yh3Ybu62r5eJoSremNZcLQXhnaHBZaj2+rgaRpP4OsRc5d71
589 RlRtTood72iy7KgDO6MuPGKJANDEiaLPvl8pVFj0WWS5S0iPVELl6dl5hheNGSck
590 aKVBjF3exKYzJlQ8oqgYuOZ18jcv+p9HCePkB6P9AoGBAJSYpkNDc/lnCpfIlxVw
591 n+VroX6QDIMZzC7BGiUSrmVsu6xEbI+8/C7ecN2oCZZLMj96EXe6j+np4zmkQezc
592 c1EwB7fNAiS0fWyE2RU6QAOZJ71bDpzQa4q4DxbOkYSybGPM/nqDRwovdjUnWeuM
593 +vrJUjAZAPHJcvos0iylnc8E
594 -----END PRIVATE KEY-----`)