Debian Codegraph @

.github/workflows/e2e.yaml

raw
1 name: e2e
2
3 on:
4 push:
5 branches:
6 - 'master'
7 tags:
8 - 'v*'
9 pull_request:
10 branches: [ master ]
11 workflow_dispatch:
12
13 env:
14 GO_TOOLCHAIN: "golang"
15 # TODO: match BASEIMAGE with Makefile default (nonroot variant)
16 BASEIMAGE: "gcr.io/distroless/static-debian11"
17 KIND_CLUSTER_NAME: "kind"
18 KIND_VERSION: "v0.27.0"
19
20 jobs:
21 build:
22 name: build
23 runs-on: ubuntu-24.04
24 steps:
25 - name: Check out code
26 uses: actions/checkout@v4
27 - name: Set up Go
28 uses: actions/setup-go@v5
29 with:
30 go-version-file: go.mod
31 id: go
32 - name: Build
33 run: |
34 mkdir _output
35
36 make docker-build/proxy-agent-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }}
37 docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master
38 docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master > _output/konnectivity-agent.tar
39 make docker-build/proxy-server-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }}
40 docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local gcr.io/k8s-staging-kas-network-proxy/proxy-server:master
41 docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server:master > _output/konnectivity-server.tar
42
43 - uses: actions/upload-artifact@v4
44 with:
45 name: konnectivity-server
46 path: _output/konnectivity-server.tar
47
48 - uses: actions/upload-artifact@v4
49 with:
50 name: konnectivity-agent
51 path: _output/konnectivity-agent.tar
52 kind-e2e:
53 name: kind-e2e
54 runs-on: ubuntu-24.04
55 timeout-minutes: 100
56 needs:
57 - build
58 env:
59 REGISTRY: gcr.io/k8s-staging-kas-network-proxy
60 KIND_IMAGE: kindest/node:${{ matrix.k8s }}
61 TAG: master
62 CONNECTION_MODE: ${{ matrix.connection-mode }}
63 strategy:
64 fail-fast: false
65 matrix:
66 k8s: [ v1.30.8, v1.31.4, v1.32.0 ]
67 connection-mode: [ grpc, http-connect ]
68 steps:
69 - name: Install kind
70 run: |
71 curl -Lo ./kind https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64
72 chmod +x ./kind
73 sudo mv ./kind /usr/local/bin/kind
74 - name: Check out code
75 uses: actions/checkout@v4
76 - name: Set up Go
77 uses: actions/setup-go@v5
78 with:
79 go-version-file: go.mod
80 id: go
81 - name: Download prebuilt konnectivity-server image
82 uses: actions/download-artifact@v4
83 with:
84 name: konnectivity-server
85 - name: Download prebuilt konnectivity-agent image
86 uses: actions/download-artifact@v4
87 with:
88 name: konnectivity-agent
89 - name: Load prebuilt konnectivity images
90 run: |
91 docker load --input konnectivity-server.tar
92 docker load --input konnectivity-agent.tar
93 - name: Fix konnectivity docker image tags
94 run: |
95 docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-server:master gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:master
96 docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:master
97 - name: Run e2e tests
98 run: make test-e2e-ci
99 e2e:
100 name: e2e
101 runs-on: ubuntu-24.04
102 timeout-minutes: 100
103 needs:
104 - build
105 strategy:
106 fail-fast: false
107 matrix:
108 ipFamily: ["ipv4", "ipv6", "dual"]
109 k8s: [ v1.30.8, v1.31.4, v1.32.0 ]
110 env:
111 JOB_NAME: "kindnetd-e2e-${{ matrix.ipFamily }}"
112 IP_FAMILY: ${{ matrix.ipFamily }}
113 steps:
114 - name: Check out code
115 uses: actions/checkout@v4
116
117 - name: Enable ipv4 and ipv6 forwarding
118 run: |
119 sudo sysctl -w net.ipv6.conf.all.forwarding=1
120 sudo sysctl -w net.ipv4.ip_forward=1
121
122 - name: Set up environment (download dependencies)
123 run: |
124 TMP_DIR=$(mktemp -d)
125 # Test binaries
126 curl -L https://dl.k8s.io/${{ matrix.k8s }}/kubernetes-test-linux-amd64.tar.gz -o ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz
127 tar xvzf ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz \
128 --directory ${TMP_DIR} \
129 --strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test
130 # kubectl
131 curl -L https://dl.k8s.io/${{ matrix.k8s }}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl
132 # kind
133 curl -Lo ${TMP_DIR}/kind https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64
134 # Install
135 sudo cp ${TMP_DIR}/ginkgo /usr/local/bin/ginkgo
136 sudo cp ${TMP_DIR}/e2e.test /usr/local/bin/e2e.test
137 sudo cp ${TMP_DIR}/kubectl /usr/local/bin/kubectl
138 sudo cp ${TMP_DIR}/kind /usr/local/bin/kind
139 sudo chmod +x /usr/local/bin/ginkgo /usr/local/bin/e2e.test /usr/local/bin/kubectl /usr/local/bin/kind
140
141 - name: Create multi node cluster
142 run: |
143 # output_dir
144 mkdir -p _artifacts
145 # create cluster
146 cat <<EOF | /usr/local/bin/kind create cluster \
147 --name ${{ env.KIND_CLUSTER_NAME}} \
148 --image kindest/node:${{ matrix.k8s }} \
149 -v7 --wait 1m --retain --config=-
150 kind: Cluster
151 apiVersion: kind.x-k8s.io/v1alpha4
152 networking:
153 ipFamily: ${IP_FAMILY}
154 nodes:
155 - role: control-plane
156 image: ${{ matrix.k8s }}
157 kubeadmConfigPatchesJSON6902:
158 - kind: ClusterConfiguration
159 patch: |
160 - op: add
161 path: /apiServer/certSANs/-
162 value: konnectivity-server.kube-system.svc.cluster.local
163 kubeadmConfigPatches:
164 - |
165 kind: ClusterConfiguration
166 apiServer:
167 extraArgs:
168 "egress-selector-config-file": "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
169 extraVolumes:
170 - name: egress-selector-config-file
171 hostPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
172 mountPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
173 readOnly: true
174 - name: konnectivity-server
175 hostPath: "/etc/kubernetes/konnectivity-server"
176 mountPath: "/etc/kubernetes/konnectivity-server"
177 readOnly: true
178 extraMounts:
179 - hostPath: ./examples/kind/egress_selector_configuration.yaml
180 containerPath: /etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml
181 - role: worker
182 image: ${{ matrix.k8s }}
183 - role: worker
184 image: ${{ matrix.k8s }}
185 EOF
186
187 # dump the kubeconfig for later
188 /usr/local/bin/kind get kubeconfig --name ${{ env.KIND_CLUSTER_NAME}} > _artifacts/kubeconfig.conf
189
190 - uses: actions/download-artifact@v4
191 with:
192 name: konnectivity-server
193
194 - uses: actions/download-artifact@v4
195 with:
196 name: konnectivity-agent
197
198 - name: Install konnectivity
199 run: |
200 # preload konnectivity images
201 docker load --input konnectivity-server.tar
202 docker load --input konnectivity-agent.tar
203 /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-server:master --name ${{ env.KIND_CLUSTER_NAME}}
204 /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master --name ${{ env.KIND_CLUSTER_NAME}}
205 kubectl apply -f examples/kind/konnectivity-server.yaml
206 kubectl apply -f examples/kind/konnectivity-agent-ds.yaml
207
208 - name: Get Cluster status
209 run: |
210 # wait network is ready
211 sleep 5
212 /usr/local/bin/kubectl get nodes -o wide
213 /usr/local/bin/kubectl get pods -A
214 /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns
215 # smoke test
216 /usr/local/bin/kubectl run test --image httpd:2
217 /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods test
218 /usr/local/bin/kubectl get pods -A -owide
219 /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=konnectivity-agent
220 /usr/local/bin/kubectl logs test
221
222
223 - name: Workaround CoreDNS for IPv6 airgapped
224 if: ${{ matrix.ipFamily == 'ipv6' }}
225 run: |
226 # Patch CoreDNS to work in Github CI
227 # 1. Github CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
228 # to work in an offline environment:
229 # https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
230 # 2. Github CI adds following domains to resolv.conf search field:
231 # .net.
232 # CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
233 # otherwise pods stops trying to resolve the domain.
234 # Get the current config
235 original_coredns=$(/usr/local/bin/kubectl get -oyaml -n=kube-system configmap/coredns)
236 echo "Original CoreDNS config:"
237 echo "${original_coredns}"
238 # Patch it
239 fixed_coredns=$(
240 printf '%s' "${original_coredns}" | sed \
241 -e 's/^.*kubernetes cluster\.local/& net/' \
242 -e '/^.*upstream$/d' \
243 -e '/^.*fallthrough.*$/d' \
244 -e '/^.*forward . \/etc\/resolv.conf$/d' \
245 -e '/^.*loop$/d' \
246 )
247 echo "Patched CoreDNS config:"
248 echo "${fixed_coredns}"
249 printf '%s' "${fixed_coredns}" | /usr/local/bin/kubectl apply -f -
250
251 - name: Run tests
252 run: |
253 export KUBERNETES_CONFORMANCE_TEST='y'
254 export E2E_REPORT_DIR=${PWD}/_artifacts
255
256 # Run tests
257 /usr/local/bin/ginkgo --nodes=25 \
258 --focus="\[Conformance\]" \
259 --skip="Feature|Federation|machinery|PerformanceDNS|DualStack|Disruptive|Serial|Slow|KubeProxy|LoadBalancer|GCE|Netpol|NetworkPolicy|NodeConformance" \
260 /usr/local/bin/e2e.test \
261 -- \
262 --kubeconfig=${PWD}/_artifacts/kubeconfig.conf \
263 --provider=local \
264 --dump-logs-on-failure=false \
265 --report-dir=${E2E_REPORT_DIR} \
266 --disable-log-dump=true
267
268 - name: Upload Junit Reports
269 if: always()
270 uses: actions/upload-artifact@v4
271 with:
272 name: kind-junit-${{ env.JOB_NAME }}-${{ matrix.k8s }}-${{ matrix.ipFamily }}-${{ github.run_id }}
273 path: './_artifacts/*.xml'
274
275 - name: Export logs
276 if: always()
277 run: |
278 /usr/local/bin/kind export logs --name ${KIND_CLUSTER_NAME} -v 4 ./_artifacts/logs
279
280 - name: Upload logs
281 if: always()
282 uses: actions/upload-artifact@v4
283 with:
284 name: kind-logs-${{ env.JOB_NAME }}-${{ matrix.k8s }}-${{ matrix.ipFamily }}-${{ github.run_id }}
285 path: ./_artifacts/logs