| 1 |
/* |
| 2 |
Copyright 2015 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package server |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"encoding/json" |
| 22 |
"fmt" |
| 23 |
"io" |
| 24 |
"net" |
| 25 |
"net/http" |
| 26 |
"net/http/httptest" |
| 27 |
goruntime "runtime" |
| 28 |
"strconv" |
| 29 |
"sync" |
| 30 |
"testing" |
| 31 |
"time" |
| 32 |
|
| 33 |
"github.com/stretchr/testify/assert" |
| 34 |
|
| 35 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 36 |
"k8s.io/apimachinery/pkg/runtime" |
| 37 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 38 |
"k8s.io/apimachinery/pkg/runtime/serializer" |
| 39 |
utilruntime "k8s.io/apimachinery/pkg/util/runtime" |
| 40 |
"k8s.io/apimachinery/pkg/util/sets" |
| 41 |
utilversion "k8s.io/apimachinery/pkg/util/version" |
| 42 |
"k8s.io/apimachinery/pkg/util/wait" |
| 43 |
"k8s.io/apimachinery/pkg/version" |
| 44 |
"k8s.io/apiserver/pkg/apis/example" |
| 45 |
examplev1 "k8s.io/apiserver/pkg/apis/example/v1" |
| 46 |
"k8s.io/apiserver/pkg/authorization/authorizer" |
| 47 |
"k8s.io/apiserver/pkg/endpoints/discovery" |
| 48 |
genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" |
| 49 |
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" |
| 50 |
"k8s.io/apiserver/pkg/registry/rest" |
| 51 |
genericfilters "k8s.io/apiserver/pkg/server/filters" |
| 52 |
"k8s.io/apiserver/pkg/warning" |
| 53 |
"k8s.io/client-go/informers" |
| 54 |
"k8s.io/client-go/kubernetes/fake" |
| 55 |
restclient "k8s.io/client-go/rest" |
| 56 |
basecompatibility "k8s.io/component-base/compatibility" |
| 57 |
"k8s.io/klog/v2/ktesting" |
| 58 |
kubeopenapi "k8s.io/kube-openapi/pkg/common" |
| 59 |
"k8s.io/kube-openapi/pkg/validation/spec" |
| 60 |
netutils "k8s.io/utils/net" |
| 61 |
) |
| 62 |
|
| 63 |
const ( |
| 64 |
extensionsGroupName = "extensions" |
| 65 |
) |
| 66 |
|
| 67 |
var ( |
| 68 |
v1GroupVersion = schema.GroupVersion{Group: "", Version: "v1"} |
| 69 |
|
| 70 |
scheme = runtime.NewScheme() |
| 71 |
codecs = serializer.NewCodecFactory(scheme) |
| 72 |
parameterCodec = runtime.NewParameterCodec(scheme) |
| 73 |
) |
| 74 |
|
| 75 |
func init() { |
| 76 |
metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion) |
| 77 |
scheme.AddUnversionedTypes(v1GroupVersion, |
| 78 |
&metav1.Status{}, |
| 79 |
&metav1.APIVersions{}, |
| 80 |
&metav1.APIGroupList{}, |
| 81 |
&metav1.APIGroup{}, |
| 82 |
&metav1.APIResourceList{}, |
| 83 |
) |
| 84 |
utilruntime.Must(example.AddToScheme(scheme)) |
| 85 |
utilruntime.Must(examplev1.AddToScheme(scheme)) |
| 86 |
} |
| 87 |
|
| 88 |
func buildTestOpenAPIDefinition() kubeopenapi.OpenAPIDefinition { |
| 89 |
return kubeopenapi.OpenAPIDefinition{ |
| 90 |
Schema: spec.Schema{ |
| 91 |
SchemaProps: spec.SchemaProps{ |
| 92 |
Description: "Description", |
| 93 |
Properties: map[string]spec.Schema{}, |
| 94 |
}, |
| 95 |
VendorExtensible: spec.VendorExtensible{ |
| 96 |
Extensions: spec.Extensions{ |
| 97 |
"x-kubernetes-group-version-kind": []interface{}{ |
| 98 |
map[string]interface{}{ |
| 99 |
"group": "", |
| 100 |
"version": "v1", |
| 101 |
"kind": "Getter", |
| 102 |
}, |
| 103 |
map[string]interface{}{ |
| 104 |
"group": "batch", |
| 105 |
"version": "v1", |
| 106 |
"kind": "Getter", |
| 107 |
}, |
| 108 |
map[string]interface{}{ |
| 109 |
"group": "extensions", |
| 110 |
"version": "v1", |
| 111 |
"kind": "Getter", |
| 112 |
}, |
| 113 |
}, |
| 114 |
}, |
| 115 |
}, |
| 116 |
}, |
| 117 |
} |
| 118 |
} |
| 119 |
|
| 120 |
func testGetOpenAPIDefinitions(_ kubeopenapi.ReferenceCallback) map[string]kubeopenapi.OpenAPIDefinition { |
| 121 |
return map[string]kubeopenapi.OpenAPIDefinition{ |
| 122 |
"k8s.io/apimachinery/pkg/apis/meta/v1.Status": {}, |
| 123 |
"k8s.io/apimachinery/pkg/apis/meta/v1.APIVersions": {}, |
| 124 |
"k8s.io/apimachinery/pkg/apis/meta/v1.APIGroupList": {}, |
| 125 |
"k8s.io/apimachinery/pkg/apis/meta/v1.APIGroup": buildTestOpenAPIDefinition(), |
| 126 |
"k8s.io/apimachinery/pkg/apis/meta/v1.APIResourceList": {}, |
| 127 |
} |
| 128 |
} |
| 129 |
|
| 130 |
// setUp is a convience function for setting up for (most) tests. |
| 131 |
func setUp(t *testing.T) (Config, *assert.Assertions) { |
| 132 |
config := NewConfig(codecs) |
| 133 |
config.ExternalAddress = "192.168.10.4:443" |
| 134 |
config.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") |
| 135 |
config.LegacyAPIGroupPrefixes = sets.NewString("/api") |
| 136 |
config.LoopbackClientConfig = &restclient.Config{} |
| 137 |
|
| 138 |
clientset := fake.NewSimpleClientset() |
| 139 |
if clientset == nil { |
| 140 |
t.Fatal("unable to create fake client set") |
| 141 |
} |
| 142 |
config.EffectiveVersion = basecompatibility.NewEffectiveVersionFromString("", "", "") |
| 143 |
config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme())) |
| 144 |
config.OpenAPIConfig.Info.Version = "unversioned" |
| 145 |
config.OpenAPIV3Config = DefaultOpenAPIV3Config(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme())) |
| 146 |
config.OpenAPIV3Config.Info.Version = "unversioned" |
| 147 |
sharedInformers := informers.NewSharedInformerFactory(clientset, config.LoopbackClientConfig.Timeout) |
| 148 |
config.Complete(sharedInformers) |
| 149 |
|
| 150 |
return *config, assert.New(t) |
| 151 |
} |
| 152 |
|
| 153 |
func newMaster(t *testing.T) (*GenericAPIServer, Config, *assert.Assertions) { |
| 154 |
config, assert := setUp(t) |
| 155 |
|
| 156 |
s, err := config.Complete(nil).New("test", NewEmptyDelegate()) |
| 157 |
if err != nil { |
| 158 |
t.Fatalf("Error in bringing up the server: %v", err) |
| 159 |
} |
| 160 |
return s, config, assert |
| 161 |
} |
| 162 |
|
| 163 |
// TestNew verifies that the New function returns a GenericAPIServer |
| 164 |
// using the configuration properly. |
| 165 |
func TestNew(t *testing.T) { |
| 166 |
s, config, assert := newMaster(t) |
| 167 |
|
| 168 |
// Verify many of the variables match their config counterparts |
| 169 |
assert.Equal(s.legacyAPIGroupPrefixes, config.LegacyAPIGroupPrefixes) |
| 170 |
assert.Equal(s.admissionControl, config.AdmissionControl) |
| 171 |
} |
| 172 |
|
| 173 |
// Verifies that AddGroupVersions works as expected. |
| 174 |
func TestInstallAPIGroups(t *testing.T) { |
| 175 |
config, assert := setUp(t) |
| 176 |
|
| 177 |
config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix") |
| 178 |
config.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: "ExternalAddress"} |
| 179 |
|
| 180 |
s, err := config.Complete(nil).New("test", NewEmptyDelegate()) |
| 181 |
if err != nil { |
| 182 |
t.Fatalf("Error in bringing up the server: %v", err) |
| 183 |
} |
| 184 |
|
| 185 |
testAPI := func(gv schema.GroupVersion) APIGroupInfo { |
| 186 |
getter, noVerbs := testGetterStorage{}, testNoVerbsStorage{} |
| 187 |
|
| 188 |
scheme := runtime.NewScheme() |
| 189 |
scheme.AddKnownTypeWithName(gv.WithKind("Getter"), getter.New()) |
| 190 |
scheme.AddKnownTypeWithName(gv.WithKind("NoVerb"), noVerbs.New()) |
| 191 |
scheme.AddKnownTypes(v1GroupVersion, &metav1.Status{}) |
| 192 |
metav1.AddToGroupVersion(scheme, v1GroupVersion) |
| 193 |
|
| 194 |
return APIGroupInfo{ |
| 195 |
PrioritizedVersions: []schema.GroupVersion{gv}, |
| 196 |
VersionedResourcesStorageMap: map[string]map[string]rest.Storage{ |
| 197 |
gv.Version: { |
| 198 |
"getter": &testGetterStorage{Version: gv.Version}, |
| 199 |
"noverbs": &testNoVerbsStorage{Version: gv.Version}, |
| 200 |
}, |
| 201 |
}, |
| 202 |
OptionsExternalVersion: &schema.GroupVersion{Version: "v1"}, |
| 203 |
ParameterCodec: parameterCodec, |
| 204 |
NegotiatedSerializer: codecs, |
| 205 |
Scheme: scheme, |
| 206 |
} |
| 207 |
} |
| 208 |
|
| 209 |
apis := []APIGroupInfo{ |
| 210 |
testAPI(schema.GroupVersion{Group: "", Version: "v1"}), |
| 211 |
testAPI(schema.GroupVersion{Group: extensionsGroupName, Version: "v1"}), |
| 212 |
testAPI(schema.GroupVersion{Group: "batch", Version: "v1"}), |
| 213 |
} |
| 214 |
|
| 215 |
err = s.InstallLegacyAPIGroup("/apiPrefix", &apis[0]) |
| 216 |
assert.NoError(err) |
| 217 |
groupPaths := []string{ |
| 218 |
config.LegacyAPIGroupPrefixes.List()[0], // /apiPrefix |
| 219 |
} |
| 220 |
for _, api := range apis[1:] { |
| 221 |
err = s.InstallAPIGroup(&api) |
| 222 |
assert.NoError(err) |
| 223 |
groupPaths = append(groupPaths, APIGroupPrefix+"/"+api.PrioritizedVersions[0].Group) // /apis/<group> |
| 224 |
} |
| 225 |
|
| 226 |
server := httptest.NewServer(s.Handler) |
| 227 |
defer server.Close() |
| 228 |
|
| 229 |
for i := range apis { |
| 230 |
// should serve APIGroup at group path |
| 231 |
info := &apis[i] |
| 232 |
path := groupPaths[i] |
| 233 |
resp, err := http.Get(server.URL + path) |
| 234 |
if err != nil { |
| 235 |
t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err) |
| 236 |
continue |
| 237 |
} |
| 238 |
|
| 239 |
body, err := io.ReadAll(resp.Body) |
| 240 |
if err != nil { |
| 241 |
t.Errorf("[%d] unexpected error reading body at path %q: %v", i, path, err) |
| 242 |
continue |
| 243 |
} |
| 244 |
|
| 245 |
t.Logf("[%d] json at %s: %s", i, path, string(body)) |
| 246 |
|
| 247 |
if i == 0 { |
| 248 |
// legacy API returns APIVersions |
| 249 |
group := metav1.APIVersions{} |
| 250 |
err = json.Unmarshal(body, &group) |
| 251 |
if err != nil { |
| 252 |
t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err) |
| 253 |
continue |
| 254 |
} |
| 255 |
} else { |
| 256 |
// API groups return APIGroup |
| 257 |
group := metav1.APIGroup{} |
| 258 |
err = json.Unmarshal(body, &group) |
| 259 |
if err != nil { |
| 260 |
t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err) |
| 261 |
continue |
| 262 |
} |
| 263 |
|
| 264 |
if got, expected := group.Name, info.PrioritizedVersions[0].Group; got != expected { |
| 265 |
t.Errorf("[%d] unexpected group name at path %q: got=%q expected=%q", i, path, got, expected) |
| 266 |
continue |
| 267 |
} |
| 268 |
|
| 269 |
if got, expected := group.PreferredVersion.Version, info.PrioritizedVersions[0].Version; got != expected { |
| 270 |
t.Errorf("[%d] unexpected group version at path %q: got=%q expected=%q", i, path, got, expected) |
| 271 |
continue |
| 272 |
} |
| 273 |
} |
| 274 |
|
| 275 |
// should serve APIResourceList at group path + /<group-version> |
| 276 |
path = path + "/" + info.PrioritizedVersions[0].Version |
| 277 |
resp, err = http.Get(server.URL + path) |
| 278 |
if err != nil { |
| 279 |
t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err) |
| 280 |
continue |
| 281 |
} |
| 282 |
|
| 283 |
body, err = io.ReadAll(resp.Body) |
| 284 |
if err != nil { |
| 285 |
t.Errorf("[%d] unexpected error reading body at path %q: %v", i, path, err) |
| 286 |
continue |
| 287 |
} |
| 288 |
|
| 289 |
t.Logf("[%d] json at %s: %s", i, path, string(body)) |
| 290 |
|
| 291 |
resources := metav1.APIResourceList{} |
| 292 |
err = json.Unmarshal(body, &resources) |
| 293 |
if err != nil { |
| 294 |
t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err) |
| 295 |
continue |
| 296 |
} |
| 297 |
|
| 298 |
if got, expected := resources.GroupVersion, info.PrioritizedVersions[0].String(); got != expected { |
| 299 |
t.Errorf("[%d] unexpected groupVersion at path %q: got=%q expected=%q", i, path, got, expected) |
| 300 |
continue |
| 301 |
} |
| 302 |
|
| 303 |
// the verbs should match the features of resources |
| 304 |
for _, r := range resources.APIResources { |
| 305 |
switch r.Name { |
| 306 |
case "getter": |
| 307 |
if got, expected := sets.NewString([]string(r.Verbs)...), sets.NewString("get"); !got.Equal(expected) { |
| 308 |
t.Errorf("[%d] unexpected verbs for resource %s/%s: got=%v expected=%v", i, resources.GroupVersion, r.Name, got, expected) |
| 309 |
} |
| 310 |
case "noverbs": |
| 311 |
if r.Verbs == nil { |
| 312 |
t.Errorf("[%d] unexpected nil verbs slice. Expected: []string{}", i) |
| 313 |
} |
| 314 |
if got, expected := sets.NewString([]string(r.Verbs)...), sets.NewString(); !got.Equal(expected) { |
| 315 |
t.Errorf("[%d] unexpected verbs for resource %s/%s: got=%v expected=%v", i, resources.GroupVersion, r.Name, got, expected) |
| 316 |
} |
| 317 |
} |
| 318 |
} |
| 319 |
} |
| 320 |
} |
| 321 |
|
| 322 |
func TestPrepareRun(t *testing.T) { |
| 323 |
_, ctx := ktesting.NewTestContext(t) |
| 324 |
s, config, assert := newMaster(t) |
| 325 |
|
| 326 |
assert.NotNil(config.OpenAPIConfig) |
| 327 |
|
| 328 |
server := httptest.NewServer(s.Handler.Director) |
| 329 |
defer server.Close() |
| 330 |
|
| 331 |
s.PrepareRun() |
| 332 |
s.RunPostStartHooks(ctx) |
| 333 |
|
| 334 |
// openapi is installed in PrepareRun |
| 335 |
resp, err := http.Get(server.URL + "/openapi/v2") |
| 336 |
assert.NoError(err) |
| 337 |
assert.Equal(http.StatusOK, resp.StatusCode) |
| 338 |
|
| 339 |
// wait for health (max-in-flight-filter is initialized asynchronously, can take a few milliseconds to initialize) |
| 340 |
assert.NoError(wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { |
| 341 |
// healthz checks are installed in PrepareRun |
| 342 |
resp, err = http.Get(server.URL + "/healthz") |
| 343 |
assert.NoError(err) |
| 344 |
data, _ := io.ReadAll(resp.Body) |
| 345 |
if http.StatusOK != resp.StatusCode { |
| 346 |
t.Logf("got %d", resp.StatusCode) |
| 347 |
t.Log(string(data)) |
| 348 |
return false, nil |
| 349 |
} |
| 350 |
return true, nil |
| 351 |
})) |
| 352 |
resp, err = http.Get(server.URL + "/healthz/ping") |
| 353 |
assert.NoError(err) |
| 354 |
assert.Equal(http.StatusOK, resp.StatusCode) |
| 355 |
} |
| 356 |
|
| 357 |
func TestUpdateOpenAPISpec(t *testing.T) { |
| 358 |
_, ctx := ktesting.NewTestContext(t) |
| 359 |
s, _, assert := newMaster(t) |
| 360 |
s.PrepareRun() |
| 361 |
s.RunPostStartHooks(ctx) |
| 362 |
|
| 363 |
server := httptest.NewServer(s.Handler.Director) |
| 364 |
defer server.Close() |
| 365 |
|
| 366 |
// verify the static spec in record is what we currently serve |
| 367 |
oldSpec, err := json.Marshal(s.StaticOpenAPISpec) |
| 368 |
assert.NoError(err) |
| 369 |
|
| 370 |
resp, err := http.Get(server.URL + "/openapi/v2") |
| 371 |
assert.NoError(err) |
| 372 |
assert.Equal(http.StatusOK, resp.StatusCode) |
| 373 |
|
| 374 |
body, err := io.ReadAll(resp.Body) |
| 375 |
assert.NoError(err) |
| 376 |
assert.Equal(oldSpec, body) |
| 377 |
resp.Body.Close() |
| 378 |
|
| 379 |
// verify we are able to update the served spec using the exposed service |
| 380 |
newSpec := []byte(`{"swagger":"2.0","info":{"title":"Test Updated Generic API Server Swagger","version":"v0.1.0"},"paths":null}`) |
| 381 |
swagger := new(spec.Swagger) |
| 382 |
err = json.Unmarshal(newSpec, swagger) |
| 383 |
assert.NoError(err) |
| 384 |
|
| 385 |
err = s.OpenAPIVersionedService.UpdateSpec(swagger) |
| 386 |
assert.NoError(err) |
| 387 |
|
| 388 |
resp, err = http.Get(server.URL + "/openapi/v2") |
| 389 |
assert.NoError(err) |
| 390 |
defer resp.Body.Close() |
| 391 |
assert.Equal(http.StatusOK, resp.StatusCode) |
| 392 |
|
| 393 |
body, err = io.ReadAll(resp.Body) |
| 394 |
assert.NoError(err) |
| 395 |
assert.Equal(newSpec, body) |
| 396 |
} |
| 397 |
|
| 398 |
// TestCustomHandlerChain verifies the handler chain with custom handler chain builder functions. |
| 399 |
func TestCustomHandlerChain(t *testing.T) { |
| 400 |
config, _ := setUp(t) |
| 401 |
|
| 402 |
var protected, called bool |
| 403 |
|
| 404 |
config.BuildHandlerChainFunc = func(apiHandler http.Handler, c *Config) http.Handler { |
| 405 |
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { |
| 406 |
protected = true |
| 407 |
apiHandler.ServeHTTP(w, req) |
| 408 |
}) |
| 409 |
} |
| 410 |
handler := http.HandlerFunc(func(r http.ResponseWriter, req *http.Request) { |
| 411 |
called = true |
| 412 |
}) |
| 413 |
|
| 414 |
s, err := config.Complete(nil).New("test", NewEmptyDelegate()) |
| 415 |
if err != nil { |
| 416 |
t.Fatalf("Error in bringing up the server: %v", err) |
| 417 |
} |
| 418 |
|
| 419 |
s.Handler.NonGoRestfulMux.Handle("/nonswagger", handler) |
| 420 |
s.Handler.NonGoRestfulMux.Handle("/secret", handler) |
| 421 |
|
| 422 |
type Test struct { |
| 423 |
handler http.Handler |
| 424 |
path string |
| 425 |
protected bool |
| 426 |
} |
| 427 |
for i, test := range []Test{ |
| 428 |
{s.Handler, "/nonswagger", true}, |
| 429 |
{s.Handler, "/secret", true}, |
| 430 |
} { |
| 431 |
protected, called = false, false |
| 432 |
|
| 433 |
var w io.Reader |
| 434 |
req, err := http.NewRequest("GET", test.path, w) |
| 435 |
if err != nil { |
| 436 |
t.Errorf("%d: Unexpected http error: %v", i, err) |
| 437 |
continue |
| 438 |
} |
| 439 |
|
| 440 |
test.handler.ServeHTTP(httptest.NewRecorder(), req) |
| 441 |
|
| 442 |
if !called { |
| 443 |
t.Errorf("%d: Expected handler to be called.", i) |
| 444 |
} |
| 445 |
if test.protected != protected { |
| 446 |
t.Errorf("%d: Expected protected=%v, got protected=%v.", i, test.protected, protected) |
| 447 |
} |
| 448 |
} |
| 449 |
} |
| 450 |
|
| 451 |
// TestNotRestRoutesHaveAuth checks that special non-routes are behind authz/authn. |
| 452 |
func TestNotRestRoutesHaveAuth(t *testing.T) { |
| 453 |
config, _ := setUp(t) |
| 454 |
|
| 455 |
authz := mockAuthorizer{} |
| 456 |
|
| 457 |
config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix") |
| 458 |
config.Authorization.Authorizer = &authz |
| 459 |
|
| 460 |
config.EnableIndex = true |
| 461 |
config.EnableProfiling = true |
| 462 |
|
| 463 |
kubeVersion := fakeVersion() |
| 464 |
binaryVersion := utilversion.MustParse(kubeVersion.String()) |
| 465 |
effectiveVersion := basecompatibility.NewEffectiveVersion(binaryVersion, false, binaryVersion, binaryVersion.SubtractMinor(1)) |
| 466 |
config.EffectiveVersion = effectiveVersion |
| 467 |
|
| 468 |
s, err := config.Complete(nil).New("test", NewEmptyDelegate()) |
| 469 |
if err != nil { |
| 470 |
t.Fatalf("Error in bringing up the server: %v", err) |
| 471 |
} |
| 472 |
|
| 473 |
for _, test := range []struct { |
| 474 |
route string |
| 475 |
}{ |
| 476 |
{"/"}, |
| 477 |
{"/debug/pprof/"}, |
| 478 |
{"/debug/flags/"}, |
| 479 |
{"/version"}, |
| 480 |
} { |
| 481 |
resp := httptest.NewRecorder() |
| 482 |
req, _ := http.NewRequest("GET", test.route, nil) |
| 483 |
s.Handler.ServeHTTP(resp, req) |
| 484 |
if resp.Code != 200 { |
| 485 |
t.Errorf("route %q expected to work: code %d", test.route, resp.Code) |
| 486 |
continue |
| 487 |
} |
| 488 |
|
| 489 |
if authz.lastURI != test.route { |
| 490 |
t.Errorf("route %q expected to go through authorization, last route did: %q", test.route, authz.lastURI) |
| 491 |
} |
| 492 |
} |
| 493 |
} |
| 494 |
|
| 495 |
func TestMuxAndDiscoveryCompleteSignals(t *testing.T) { |
| 496 |
// setup |
| 497 |
cfg, assert := setUp(t) |
| 498 |
|
| 499 |
// scenario 1: single server with some signals |
| 500 |
root, err := cfg.Complete(nil).New("rootServer", NewEmptyDelegate()) |
| 501 |
assert.NoError(err) |
| 502 |
if len(root.MuxAndDiscoveryCompleteSignals()) != 0 { |
| 503 |
assert.Error(fmt.Errorf("unexpected signals %v registered in the root server", root.MuxAndDiscoveryCompleteSignals())) |
| 504 |
} |
| 505 |
root.RegisterMuxAndDiscoveryCompleteSignal("rootTestSignal", make(chan struct{})) |
| 506 |
if len(root.MuxAndDiscoveryCompleteSignals()) != 1 { |
| 507 |
assert.Error(fmt.Errorf("unexpected signals %v registered in the root server", root.MuxAndDiscoveryCompleteSignals())) |
| 508 |
} |
| 509 |
|
| 510 |
// scenario 2: multiple servers with some signals |
| 511 |
delegate, err := cfg.Complete(nil).New("delegateServer", NewEmptyDelegate()) |
| 512 |
assert.NoError(err) |
| 513 |
delegate.RegisterMuxAndDiscoveryCompleteSignal("delegateTestSignal", make(chan struct{})) |
| 514 |
if len(delegate.MuxAndDiscoveryCompleteSignals()) != 1 { |
| 515 |
assert.Error(fmt.Errorf("unexpected signals %v registered in the delegate server", delegate.MuxAndDiscoveryCompleteSignals())) |
| 516 |
} |
| 517 |
newRoot, err := cfg.Complete(nil).New("newRootServer", delegate) |
| 518 |
assert.NoError(err) |
| 519 |
if len(newRoot.MuxAndDiscoveryCompleteSignals()) != 1 { |
| 520 |
assert.Error(fmt.Errorf("unexpected signals %v registered in the newRoot server", newRoot.MuxAndDiscoveryCompleteSignals())) |
| 521 |
} |
| 522 |
newRoot.RegisterMuxAndDiscoveryCompleteSignal("newRootTestSignal", make(chan struct{})) |
| 523 |
if len(newRoot.MuxAndDiscoveryCompleteSignals()) != 2 { |
| 524 |
assert.Error(fmt.Errorf("unexpected signals %v registered in the newRoot server", newRoot.MuxAndDiscoveryCompleteSignals())) |
| 525 |
} |
| 526 |
} |
| 527 |
|
| 528 |
type mockAuthorizer struct { |
| 529 |
lastURI string |
| 530 |
} |
| 531 |
|
| 532 |
func (authz *mockAuthorizer) Authorize(ctx context.Context, a authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) { |
| 533 |
authz.lastURI = a.GetPath() |
| 534 |
return authorizer.DecisionAllow, "", nil |
| 535 |
} |
| 536 |
|
| 537 |
type testGetterStorage struct { |
| 538 |
Version string |
| 539 |
} |
| 540 |
|
| 541 |
func (p *testGetterStorage) NamespaceScoped() bool { |
| 542 |
return true |
| 543 |
} |
| 544 |
|
| 545 |
func (p *testGetterStorage) New() runtime.Object { |
| 546 |
return &metav1.APIGroup{ |
| 547 |
TypeMeta: metav1.TypeMeta{ |
| 548 |
Kind: "Getter", |
| 549 |
APIVersion: p.Version, |
| 550 |
}, |
| 551 |
} |
| 552 |
} |
| 553 |
|
| 554 |
func (p *testGetterStorage) Destroy() { |
| 555 |
} |
| 556 |
|
| 557 |
func (p *testGetterStorage) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) { |
| 558 |
return nil, nil |
| 559 |
} |
| 560 |
|
| 561 |
func (p *testGetterStorage) GetSingularName() string { |
| 562 |
return "getter" |
| 563 |
} |
| 564 |
|
| 565 |
type testNoVerbsStorage struct { |
| 566 |
Version string |
| 567 |
} |
| 568 |
|
| 569 |
func (p *testNoVerbsStorage) NamespaceScoped() bool { |
| 570 |
return true |
| 571 |
} |
| 572 |
|
| 573 |
func (p *testNoVerbsStorage) New() runtime.Object { |
| 574 |
return &metav1.APIGroup{ |
| 575 |
TypeMeta: metav1.TypeMeta{ |
| 576 |
Kind: "NoVerbs", |
| 577 |
APIVersion: p.Version, |
| 578 |
}, |
| 579 |
} |
| 580 |
} |
| 581 |
|
| 582 |
func (p *testNoVerbsStorage) Destroy() { |
| 583 |
} |
| 584 |
|
| 585 |
func (p *testNoVerbsStorage) GetSingularName() string { |
| 586 |
return "noverb" |
| 587 |
} |
| 588 |
|
| 589 |
func fakeVersion() version.Info { |
| 590 |
return version.Info{ |
| 591 |
Major: "42", |
| 592 |
Minor: "42", |
| 593 |
GitVersion: "42.42", |
| 594 |
GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689", |
| 595 |
GitTreeState: "Dirty", |
| 596 |
BuildDate: time.Now().String(), |
| 597 |
GoVersion: goruntime.Version(), |
| 598 |
Compiler: goruntime.Compiler, |
| 599 |
Platform: fmt.Sprintf("%s/%s", goruntime.GOOS, goruntime.GOARCH), |
| 600 |
} |
| 601 |
} |
| 602 |
|
| 603 |
// TestGracefulShutdown verifies server shutdown after request handler finish. |
| 604 |
func TestGracefulShutdown(t *testing.T) { |
| 605 |
config, _ := setUp(t) |
| 606 |
|
| 607 |
var graceShutdown bool |
| 608 |
wg := sync.WaitGroup{} |
| 609 |
wg.Add(1) |
| 610 |
|
| 611 |
config.BuildHandlerChainFunc = func(apiHandler http.Handler, c *Config) http.Handler { |
| 612 |
handler := genericfilters.WithWaitGroup(apiHandler, c.LongRunningFunc, c.NonLongRunningRequestWaitGroup) |
| 613 |
handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver) |
| 614 |
return handler |
| 615 |
} |
| 616 |
|
| 617 |
s, err := config.Complete(nil).New("test", NewEmptyDelegate()) |
| 618 |
if err != nil { |
| 619 |
t.Fatalf("Error in bringing up the server: %v", err) |
| 620 |
} |
| 621 |
|
| 622 |
twoSecondHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { |
| 623 |
wg.Done() |
| 624 |
time.Sleep(2 * time.Second) |
| 625 |
w.WriteHeader(http.StatusOK) |
| 626 |
graceShutdown = true |
| 627 |
}) |
| 628 |
okHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { |
| 629 |
w.WriteHeader(http.StatusOK) |
| 630 |
}) |
| 631 |
|
| 632 |
s.Handler.NonGoRestfulMux.Handle("/test", twoSecondHandler) |
| 633 |
s.Handler.NonGoRestfulMux.Handle("/200", okHandler) |
| 634 |
|
| 635 |
insecureServer := &http.Server{ |
| 636 |
Addr: "0.0.0.0:0", |
| 637 |
Handler: s.Handler, |
| 638 |
} |
| 639 |
stopCh := make(chan struct{}) |
| 640 |
|
| 641 |
ln, err := net.Listen("tcp", insecureServer.Addr) |
| 642 |
if err != nil { |
| 643 |
t.Errorf("failed to listen on %v: %v", insecureServer.Addr, err) |
| 644 |
} |
| 645 |
|
| 646 |
// get port |
| 647 |
serverPort := ln.Addr().(*net.TCPAddr).Port |
| 648 |
stoppedCh, _, err := RunServer(insecureServer, ln, 10*time.Second, stopCh) |
| 649 |
if err != nil { |
| 650 |
t.Fatalf("RunServer err: %v", err) |
| 651 |
} |
| 652 |
|
| 653 |
graceCh := make(chan struct{}) |
| 654 |
// mock a client request |
| 655 |
go func() { |
| 656 |
resp, err := http.Get("http://127.0.0.1:" + strconv.Itoa(serverPort) + "/test") |
| 657 |
if err != nil { |
| 658 |
t.Errorf("Unexpected http error: %v", err) |
| 659 |
} |
| 660 |
if resp.StatusCode != http.StatusOK { |
| 661 |
t.Errorf("Unexpected http status code: %v", resp.StatusCode) |
| 662 |
} |
| 663 |
close(graceCh) |
| 664 |
}() |
| 665 |
|
| 666 |
// close stopCh after request sent to server to guarantee request handler is running. |
| 667 |
wg.Wait() |
| 668 |
close(stopCh) |
| 669 |
|
| 670 |
time.Sleep(500 * time.Millisecond) |
| 671 |
if _, err := http.Get("http://127.0.0.1:" + strconv.Itoa(serverPort) + "/200"); err == nil { |
| 672 |
t.Errorf("Unexpected http success after stopCh was closed") |
| 673 |
} |
| 674 |
|
| 675 |
// wait for wait group handler finish |
| 676 |
s.NonLongRunningRequestWaitGroup.Wait() |
| 677 |
<-stoppedCh |
| 678 |
|
| 679 |
// check server all handlers finished. |
| 680 |
if !graceShutdown { |
| 681 |
t.Errorf("server shutdown not gracefully.") |
| 682 |
} |
| 683 |
// check client to make sure receive response. |
| 684 |
select { |
| 685 |
case <-graceCh: |
| 686 |
t.Logf("server shutdown gracefully.") |
| 687 |
case <-time.After(30 * time.Second): |
| 688 |
t.Errorf("Timed out waiting for response.") |
| 689 |
} |
| 690 |
} |
| 691 |
|
| 692 |
func TestWarningWithRequestTimeout(t *testing.T) { |
| 693 |
type result struct { |
| 694 |
err interface{} |
| 695 |
stackTrace string |
| 696 |
} |
| 697 |
clientDoneCh, resultCh := make(chan struct{}), make(chan result, 1) |
| 698 |
testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 699 |
// this will catch recoverable panic like 'Header called after Handler finished'. |
| 700 |
// go runtime crashes the program if it detects a program-ending |
| 701 |
// panic like 'concurrent map iteration and map write', so this |
| 702 |
// panic can not be caught. |
| 703 |
defer func() { |
| 704 |
result := result{} |
| 705 |
result.err = recover() |
| 706 |
if result.err != nil { |
| 707 |
// Same as stdlib http server code. Manually allocate stack |
| 708 |
// trace buffer size to prevent excessively large logs |
| 709 |
const size = 64 << 10 |
| 710 |
buf := make([]byte, size) |
| 711 |
buf = buf[:goruntime.Stack(buf, false)] |
| 712 |
result.stackTrace = string(buf) |
| 713 |
} |
| 714 |
resultCh <- result |
| 715 |
}() |
| 716 |
|
| 717 |
// add warnings while we're waiting for the request to timeout to catch read/write races |
| 718 |
loop: |
| 719 |
for { |
| 720 |
select { |
| 721 |
case <-r.Context().Done(): |
| 722 |
break loop |
| 723 |
default: |
| 724 |
warning.AddWarning(r.Context(), "a", "1") |
| 725 |
} |
| 726 |
} |
| 727 |
// the request has just timed out, write to catch read/write races |
| 728 |
warning.AddWarning(r.Context(), "agent", "text") |
| 729 |
|
| 730 |
// give time for the timeout response to be written, then try to |
| 731 |
// write a response header to catch the "Header after Handler finished" panic |
| 732 |
<-clientDoneCh |
| 733 |
|
| 734 |
warning.AddWarning(r.Context(), "agent", "text") |
| 735 |
}) |
| 736 |
handler := newGenericAPIServerHandlerChain(t, "/test", testHandler) |
| 737 |
|
| 738 |
server := httptest.NewUnstartedServer(handler) |
| 739 |
server.EnableHTTP2 = true |
| 740 |
server.StartTLS() |
| 741 |
defer server.Close() |
| 742 |
|
| 743 |
request, err := http.NewRequest("GET", server.URL+"/test?timeout=100ms", nil) |
| 744 |
if err != nil { |
| 745 |
t.Fatalf("unexpected error: %v", err) |
| 746 |
} |
| 747 |
|
| 748 |
client := server.Client() |
| 749 |
response, err := client.Do(request) |
| 750 |
close(clientDoneCh) |
| 751 |
if err != nil { |
| 752 |
t.Errorf("expected server to return an HTTP response: %v", err) |
| 753 |
} |
| 754 |
if want := http.StatusGatewayTimeout; response == nil || response.StatusCode != want { |
| 755 |
t.Errorf("expected server to return %d, but got: %v", want, response) |
| 756 |
} |
| 757 |
|
| 758 |
var resultGot result |
| 759 |
select { |
| 760 |
case resultGot = <-resultCh: |
| 761 |
case <-time.After(wait.ForeverTestTimeout): |
| 762 |
t.Errorf("the handler never returned a result") |
| 763 |
} |
| 764 |
if resultGot.err != nil { |
| 765 |
t.Errorf("Expected no panic, but got: %v", resultGot.err) |
| 766 |
t.Errorf("Stack Trace: %s", resultGot.stackTrace) |
| 767 |
} |
| 768 |
} |
| 769 |
|
| 770 |
// builds a handler chain with the given user handler as used by GenericAPIServer. |
| 771 |
func newGenericAPIServerHandlerChain(t *testing.T, path string, handler http.Handler) http.Handler { |
| 772 |
config, _ := setUp(t) |
| 773 |
s, err := config.Complete(nil).New("test", NewEmptyDelegate()) |
| 774 |
if err != nil { |
| 775 |
t.Fatalf("Error in bringing up the server: %v", err) |
| 776 |
} |
| 777 |
|
| 778 |
s.Handler.NonGoRestfulMux.Handle(path, handler) |
| 779 |
return s.Handler |
| 780 |
} |