Debian Codegraph @

govc/namespace/cluster/enable.go

raw
1 /*
2 Copyright (c) 2020 VMware, Inc. All Rights Reserved.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package cluster
18
19 import (
20 "context"
21 "flag"
22 "fmt"
23 "strconv"
24 "strings"
25
26 "github.com/vmware/govmomi/govc/cli"
27 "github.com/vmware/govmomi/govc/flags"
28 "github.com/vmware/govmomi/govc/storage/policy"
29 "github.com/vmware/govmomi/pbm"
30 "github.com/vmware/govmomi/vapi/namespace"
31 )
32
33 type enableCluster struct {
34 ControlPlaneDNSSearchDomains string
35 ImageStoragePolicy string
36 NcpClusterNetworkSpec workloadNetwork
37 ControlPlaneManagementNetwork namespace.MasterManagementNetwork
38 ControlPlaneDNSNames string
39 ControlPlaneNTPServers string
40 EphemeralStoragePolicy string
41 DefaultImageRepository string
42 ServiceCidr string
43 LoginBanner string
44 SizeHint string
45 WorkerDNS string
46 DefaultImageRegistry string
47 ControlPlaneDNS string
48 NetworkProvider string
49 ControlPlaneStoragePolicy string
50 DefaultKubernetesServiceContentLibrary string
51
52 *flags.ClusterFlag
53 }
54
55 type workloadNetwork struct {
56 NsxEdgeCluster string
57 PodCidrs string
58 EgressCidrs string
59 Switch string
60 IngressCidrs string
61 }
62
63 type objectReferences struct {
64 Cluster string
65 Network string
66 ImageStoragePolicy string
67 ControlPlaneStoragePolicy string
68 EphemeralStoragePolicy string
69 WorkerNetworkSwitch string
70 EdgeCluster string
71 }
72
73 func init() {
74 newEnableCluster := &enableCluster{
75 ControlPlaneManagementNetwork: namespace.MasterManagementNetwork{
76 AddressRange: &namespace.AddressRange{},
77 },
78 }
79 cli.Register("namespace.cluster.enable", newEnableCluster)
80 }
81
82 func (cmd *enableCluster) Register(ctx context.Context, f *flag.FlagSet) {
83 cmd.ClusterFlag, ctx = flags.NewClusterFlag(ctx)
84 // this brings in some additional flags
85 // Datacenter flag, which we do want in order to configure a Finder we use to do some lookups
86 // ClientFlag, which we do need to control auth and connection details
87 // OutputFlag, which doesn't have any effect in this case as there's no response to output
88 cmd.ClusterFlag.Register(ctx, f)
89 // Descriptions are mostly extracted from:
90 // https://vmware.github.io/vsphere-automation-sdk-rest/vsphere/operations/com/vmware/vcenter/namespace_management/clusters.enable-operation.html
91 f.StringVar(&cmd.SizeHint, "size", "",
92 "The size of the Kubernetes API server and the worker nodes. Value is one of: TINY, SMALL, MEDIUM, LARGE.")
93 f.StringVar(&cmd.ServiceCidr, "service-cidr", "",
94 "CIDR block from which Kubernetes allocates service cluster IP addresses. Shouldn't overlap with pod, ingress or egress CIDRs")
95 f.StringVar(&cmd.NetworkProvider, "network-provider", "NSXT_CONTAINER_PLUGIN",
96 "Optional. Provider of cluster networking for this vSphere Namespaces cluster. Currently only value supported is: NSXT_CONTAINER_PLUGIN.")
97 f.StringVar(&cmd.NcpClusterNetworkSpec.PodCidrs, "pod-cidrs", "",
98 "CIDR blocks from which Kubernetes allocates pod IP addresses. Comma-separated list. Shouldn't overlap with service, ingress or egress CIDRs.")
99 f.StringVar(&cmd.NcpClusterNetworkSpec.IngressCidrs, "workload-network.ingress-cidrs", "",
100 "CIDR blocks from which NSX assigns IP addresses for Kubernetes Ingresses and Kubernetes Services of type LoadBalancer. Comma-separated list. Shouldn't overlap with pod, service or egress CIDRs.")
101 f.StringVar(&cmd.NcpClusterNetworkSpec.EgressCidrs, "workload-network.egress-cidrs", "",
102 "CIDR blocks from which NSX assigns IP addresses used for performing SNAT from container IPs to external IPs. Comma-separated list. Shouldn't overlap with pod, service or ingress CIDRs.")
103 f.StringVar(&cmd.NcpClusterNetworkSpec.Switch, "workload-network.switch", "",
104 "vSphere Distributed Switch used to connect this cluster.")
105 f.StringVar(&cmd.NcpClusterNetworkSpec.NsxEdgeCluster, "workload-network.edge-cluster", "",
106 "NSX Edge Cluster to be used for Kubernetes Services of type LoadBalancer, Kubernetes Ingresses, and NSX SNAT.")
107 f.StringVar(&cmd.ControlPlaneManagementNetwork.Network, "mgmt-network.network", "",
108 "Identifier for the management network.")
109 f.StringVar(&cmd.ControlPlaneManagementNetwork.Mode, "mgmt-network.mode", "STATICRANGE",
110 "IPv4 address assignment modes. Value is one of: DHCP, STATICRANGE")
111 f.StringVar(&cmd.ControlPlaneManagementNetwork.FloatingIP, "mgmt-network.floating-IP", "",
112 "Optional. The Floating IP used by the HA master cluster in the when network Mode is DHCP.")
113 f.StringVar(&cmd.ControlPlaneManagementNetwork.AddressRange.StartingAddress, "mgmt-network.starting-address", "",
114 "Denotes the start of the IP range to be used. Optional, but required with network mode STATICRANGE.")
115 f.IntVar(&cmd.ControlPlaneManagementNetwork.AddressRange.AddressCount, "mgmt-network.address-count", 5,
116 "The number of IP addresses in the management range. Optional, but required with network mode STATICRANGE.")
117 f.StringVar(&cmd.ControlPlaneManagementNetwork.AddressRange.SubnetMask, "mgmt-network.subnet-mask", "",
118 "Subnet mask of the management network. Optional, but required with network mode STATICRANGE.")
119 f.StringVar(&cmd.ControlPlaneManagementNetwork.AddressRange.Gateway, "mgmt-network.gateway", "",
120 "Gateway to be used for the management IP range")
121 f.StringVar(&cmd.ControlPlaneDNS, "control-plane-dns", "",
122 "Comma-separated list of DNS server IP addresses to use on Kubernetes API server, specified in order of preference.")
123 f.StringVar(&cmd.ControlPlaneDNSNames, "control-plane-dns-names", "",
124 "Comma-separated list of DNS names to associate with the Kubernetes API server. These DNS names are embedded in the TLS certificate presented by the API server.")
125 f.StringVar(&cmd.ControlPlaneDNSSearchDomains, "control-plane-dns-search-domains", "",
126 "Comma-separated list of domains to be searched when trying to lookup a host name on Kubernetes API server, specified in order of preference.")
127 f.StringVar(&cmd.ControlPlaneNTPServers, "control-plane-ntp-servers", "",
128 "Optional. Comma-separated list of NTP server DNS names or IP addresses to use on Kubernetes API server, specified in order of preference. If unset, VMware Tools based time synchronization is enabled.")
129 f.StringVar(&cmd.WorkerDNS, "worker-dns", "",
130 "Comma-separated list of DNS server IP addresses to use on the worker nodes, specified in order of preference.")
131 f.StringVar(&cmd.ControlPlaneStoragePolicy, "control-plane-storage-policy", "",
132 "Storage policy associated with Kubernetes API server.")
133 f.StringVar(&cmd.EphemeralStoragePolicy, "ephemeral-storage-policy", "",
134 "Storage policy associated with ephemeral disks of all the Kubernetes Pods in the cluster.")
135 f.StringVar(&cmd.ImageStoragePolicy, "image-storage-policy", "",
136 "Storage policy to be used for container images.")
137 f.StringVar(&cmd.LoginBanner, "login-banner", "",
138 "Optional. Disclaimer to be displayed prior to login via the Kubectl plugin.")
139 // documented API is currently ambiguous with these duplicated fields, need to wait for this to be resolved
140 //f.StringVar(&cmd.DefaultImageRegistry, "default-image-registry", "",
141 // "Optional. Default image registry to use when unspecified in the container image name. Defaults to Docker Hub.")
142 //f.StringVar(&cmd.DefaultImageRepository, "default-image-repository", "",
143 // "Optional. Default image registry to use when unspecified in the container image name. Defaults to Docker Hub.")
144 // TODO
145 // f.StringVar(&cmd.DefaultKubernetesServiceContentLibrary, "default-kubernetes-service-content-library", "",
146 // "Optional. Content Library which holds the VM Images for vSphere Kubernetes Service. This Content Library should be subscribed to VMware's hosted vSphere Kubernetes Service Repository.")
147 }
148
149 func (cmd *enableCluster) Description() string {
150 return `Enable vSphere Namespaces on the cluster.
151 This operation sets up Kubernetes instance for the cluster along with worker nodes.
152
153 Examples:
154 govc namespace.cluster.enable \
155 -cluster "Workload-Cluster" \
156 -service-cidr 10.96.0.0/23 \
157 -pod-cidrs 10.244.0.0/20 \
158 -control-plane-dns-names wcp.example.com \
159 -workload-network.egress-cidrs 10.0.0.128/26 \
160 -workload-network.ingress-cidrs "10.0.0.64/26" \
161 -workload-network.switch VDS \
162 -workload-network.edge-cluster Edge-Cluster-1 \
163 -size TINY \
164 -mgmt-network.network "DVPG-Management Network" \
165 -mgmt-network.gateway 10.0.0.1 \
166 -mgmt-network.starting-address 10.0.0.45 \
167 -mgmt-network.subnet-mask 255.255.255.0 \
168 -ephemeral-storage-policy "vSAN Default Storage Policy" \
169 -control-plane-storage-policy "vSAN Default Storage Policy" \
170 -image-storage-policy "vSAN Default Storage Policy"`
171 }
172
173 func (cmd *enableCluster) Run(ctx context.Context, f *flag.FlagSet) error {
174 c, err := cmd.RestClient()
175 if err != nil {
176 return err
177 }
178
179 // Cluster object reference lookup
180 cluster, err := cmd.Cluster()
181 if err != nil {
182 return err
183 }
184 id := cluster.Reference().Value
185
186 finder, err := cmd.ClusterFlag.Finder()
187 if err != nil {
188 return err
189 }
190
191 // Network object reference lookup.
192 networkRef, err := finder.Network(ctx, cmd.ControlPlaneManagementNetwork.Network)
193 if err != nil {
194 return err
195 }
196
197 // Storage policy object references lookup
198 vc, err := cmd.Client()
199 if err != nil {
200 return fmt.Errorf("error creating soap client: %s", err)
201 }
202
203 pbmc, err := pbm.NewClient(ctx, vc)
204 if err != nil {
205 return fmt.Errorf("error creating client for storage policy lookup: %s", err)
206 }
207
208 storagePolicyNames := make(map[string]string)
209 storagePolicyNames["control-plane"] = cmd.ControlPlaneStoragePolicy
210 storagePolicyNames["ephemeral"] = cmd.EphemeralStoragePolicy
211 storagePolicyNames["image"] = cmd.ImageStoragePolicy
212 // keep track of names we looked up, so we don't repeat lookups
213 visited := make(map[string]string)
214 storagePolicyRefs := make(map[string]string)
215 for k, v := range storagePolicyNames {
216 if _, exists := visited[v]; !exists {
217 policies, err := policy.ListProfiles(ctx, pbmc, v)
218 if err != nil {
219 return fmt.Errorf("error looking up storage policy %q: %s", v, err)
220 } else if len(policies) != 1 {
221 return fmt.Errorf("could not find a unique storage policy ID for query %q", v)
222 }
223 visited[v] = policies[0].GetPbmProfile().ProfileId.UniqueId
224 storagePolicyRefs[k] = policies[0].GetPbmProfile().ProfileId.UniqueId
225 } else {
226 storagePolicyRefs[k] = visited[v]
227 }
228
229 }
230
231 // DVS Object reference lookup
232 // We need an id returned from the namespace lookup here, not a regular managed object reference.
233 // Similar approach in powerCLI here:
234 // https://github.com/lamw/PowerCLI-Example-Scripts/blob/7e4b9b9c93c5ffaa0ac2fefa8e02e5f751c044b7/Modules/VMware.WorkloadManagement/VMware.WorkloadManagement.psm1#L123
235 // Note that the data model returned means we get no chance to choose the switch by name.
236 // We assume there's just one switch per cluster and bail out otherwise.
237 m := namespace.NewManager(c)
238 clusterId := cluster.Reference().Value
239 switches, err := m.ListCompatibleDistributedSwitches(ctx, clusterId)
240 if err != nil {
241 return fmt.Errorf("error in compatible switch lookup: %s", err)
242 } else if len(switches) != 1 {
243 return fmt.Errorf("expected to find 1 namespace compatible switch in cluster %q, found %d",
244 clusterId, len(switches))
245 }
246
247 switchId := switches[0].DistributedSwitch
248
249 edgeClusterDisplayName := cmd.NcpClusterNetworkSpec.NsxEdgeCluster
250 edgeClusters, err := m.ListCompatibleEdgeClusters(ctx, clusterId, switchId)
251 if err != nil {
252 return fmt.Errorf("error in compatible switch lookup: %s", err)
253 }
254
255 matchingEdgeClusters := make([]string, 0)
256 for _, v := range edgeClusters {
257 if v.DisplayName == edgeClusterDisplayName {
258 matchingEdgeClusters = append(matchingEdgeClusters, v.EdgeCluster)
259 }
260 }
261 if len(matchingEdgeClusters) != 1 {
262 return fmt.Errorf("Didn't find unique match for edge cluster %q, found %d objects",
263 edgeClusterDisplayName, len(matchingEdgeClusters))
264 }
265
266 resolvedObjectRefs := objectReferences{
267 Cluster: cluster.Reference().Value,
268 Network: networkRef.Reference().Value,
269 ControlPlaneStoragePolicy: storagePolicyRefs["control-plane"],
270 EphemeralStoragePolicy: storagePolicyRefs["ephemeral"],
271 ImageStoragePolicy: storagePolicyRefs["image"],
272 WorkerNetworkSwitch: switchId,
273 EdgeCluster: matchingEdgeClusters[0],
274 }
275
276 enableClusterSpec, err := cmd.toVapiSpec(resolvedObjectRefs)
277 if err != nil {
278 return err
279 }
280
281 err = m.EnableCluster(ctx, id, enableClusterSpec)
282 if err != nil {
283 return err
284 }
285
286 return nil
287 }
288
289 func (cmd *enableCluster) toVapiSpec(refs objectReferences) (*namespace.EnableClusterSpec, error) {
290
291 podCidrs, err := splitCidrList(splitCommaSeparatedList(cmd.NcpClusterNetworkSpec.PodCidrs))
292 if err != nil {
293 return nil, fmt.Errorf("invalid workload-network.pod-cidrs value: %s", err)
294 }
295 egressCidrs, err := splitCidrList(splitCommaSeparatedList(cmd.NcpClusterNetworkSpec.EgressCidrs))
296 if err != nil {
297 return nil, fmt.Errorf("invalid workload-network.egress-cidrs value: %s", err)
298 }
299 ingressCidrs, err := splitCidrList(splitCommaSeparatedList(cmd.NcpClusterNetworkSpec.IngressCidrs))
300 if err != nil {
301 return nil, fmt.Errorf("invalid workload-network.ingress-cidrs value: %s", err)
302 }
303 serviceCidr, err := splitCidr(cmd.ServiceCidr)
304 if err != nil {
305 return nil, fmt.Errorf("invalid service-cidr value: %s", err)
306 }
307 var masterManagementNetwork *namespace.MasterManagementNetwork
308 if (cmd.ControlPlaneManagementNetwork.Mode != "") ||
309 (cmd.ControlPlaneManagementNetwork.FloatingIP != "") ||
310 (cmd.ControlPlaneManagementNetwork.Network != "") {
311 masterManagementNetwork = &cmd.ControlPlaneManagementNetwork
312 }
313 if masterManagementNetwork != nil {
314 if (masterManagementNetwork.AddressRange.SubnetMask == "") &&
315 (masterManagementNetwork.AddressRange.StartingAddress == "") &&
316 (masterManagementNetwork.AddressRange.Gateway == "") &&
317 (masterManagementNetwork.AddressRange.AddressCount == 0) {
318 masterManagementNetwork.AddressRange = nil
319 }
320 masterManagementNetwork.Network = refs.Network
321 }
322
323 spec := namespace.EnableClusterSpec{
324 MasterDNSSearchDomains: splitCommaSeparatedList(cmd.ControlPlaneDNSSearchDomains),
325 ImageStorage: namespace.ImageStorage{StoragePolicy: refs.ImageStoragePolicy},
326 NcpClusterNetworkSpec: &namespace.NcpClusterNetworkSpec{
327 NsxEdgeCluster: refs.EdgeCluster,
328 PodCidrs: podCidrs,
329 EgressCidrs: egressCidrs,
330 ClusterDistributedSwitch: refs.WorkerNetworkSwitch,
331 IngressCidrs: ingressCidrs,
332 },
333 MasterManagementNetwork: masterManagementNetwork,
334 MasterDNSNames: splitCommaSeparatedList(cmd.ControlPlaneDNSNames),
335 MasterNTPServers: splitCommaSeparatedList(cmd.ControlPlaneNTPServers),
336 EphemeralStoragePolicy: refs.EphemeralStoragePolicy,
337 DefaultImageRepository: cmd.DefaultImageRepository,
338 ServiceCidr: serviceCidr,
339 LoginBanner: cmd.LoginBanner,
340 SizeHint: cmd.SizeHint,
341 WorkerDNS: splitCommaSeparatedList(cmd.WorkerDNS),
342 DefaultImageRegistry: nil,
343 MasterDNS: splitCommaSeparatedList(cmd.ControlPlaneDNS),
344 NetworkProvider: cmd.NetworkProvider,
345 MasterStoragePolicy: refs.ControlPlaneStoragePolicy,
346 DefaultKubernetesServiceContentLibrary: cmd.DefaultKubernetesServiceContentLibrary,
347 }
348 return &spec, nil
349 }
350
351 func splitCidr(input string) (*namespace.Cidr, error) {
352 parts := strings.Split(input, "/")
353 if !(len(parts) == 2) || parts[0] == "" {
354 return nil, fmt.Errorf("invalid cidr %q supplied, needs to be in form '192.168.1.0/24'", input)
355 }
356
357 prefix, err := strconv.Atoi(parts[1])
358 if err != nil {
359 return nil, err
360 }
361
362 result := namespace.Cidr{
363 Address: parts[0],
364 Prefix: prefix,
365 }
366 return &result, nil
367 }
368
369 func splitCidrList(input []string) ([]namespace.Cidr, error) {
370 var result []namespace.Cidr
371 for i, cidrIn := range input {
372 cidr, err := splitCidr(cidrIn)
373 if err != nil {
374 return nil, fmt.Errorf("parsing cidr %q in list position %d : %q", cidrIn, i, err)
375 }
376 result = append(result, *cidr)
377 }
378 return result, nil
379 }
380
381 func splitCommaSeparatedList(cslist string) []string {
382 return deleteEmpty(strings.Split(cslist, ","))
383 }
384
385 func deleteEmpty(s []string) []string {
386 var r []string
387 for _, str := range s {
388 if str != "" {
389 r = append(r, str)
390 }
391 }
392 return r
393 }