| 1 |
name: e2e |
| 2 |
|
| 3 |
on: |
| 4 |
push: |
| 5 |
branches: |
| 6 |
- 'master' |
| 7 |
tags: |
| 8 |
- 'v*' |
| 9 |
pull_request: |
| 10 |
branches: [ master ] |
| 11 |
workflow_dispatch: |
| 12 |
|
| 13 |
env: |
| 14 |
GO_TOOLCHAIN: "golang" |
| 15 |
# TODO: match BASEIMAGE with Makefile default (nonroot variant) |
| 16 |
BASEIMAGE: "gcr.io/distroless/static-debian11" |
| 17 |
KIND_CLUSTER_NAME: "kind" |
| 18 |
KIND_VERSION: "v0.27.0" |
| 19 |
|
| 20 |
jobs: |
| 21 |
build: |
| 22 |
name: build |
| 23 |
runs-on: ubuntu-24.04 |
| 24 |
steps: |
| 25 |
- name: Check out code |
| 26 |
uses: actions/checkout@v4 |
| 27 |
- name: Set up Go |
| 28 |
uses: actions/setup-go@v5 |
| 29 |
with: |
| 30 |
go-version-file: go.mod |
| 31 |
id: go |
| 32 |
- name: Build |
| 33 |
run: | |
| 34 |
mkdir _output |
| 35 |
|
| 36 |
make docker-build/proxy-agent-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }} |
| 37 |
docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master |
| 38 |
docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master > _output/konnectivity-agent.tar |
| 39 |
make docker-build/proxy-server-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }} |
| 40 |
docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local gcr.io/k8s-staging-kas-network-proxy/proxy-server:master |
| 41 |
docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server:master > _output/konnectivity-server.tar |
| 42 |
|
| 43 |
- uses: actions/upload-artifact@v4 |
| 44 |
with: |
| 45 |
name: konnectivity-server |
| 46 |
path: _output/konnectivity-server.tar |
| 47 |
|
| 48 |
- uses: actions/upload-artifact@v4 |
| 49 |
with: |
| 50 |
name: konnectivity-agent |
| 51 |
path: _output/konnectivity-agent.tar |
| 52 |
kind-e2e: |
| 53 |
name: kind-e2e |
| 54 |
runs-on: ubuntu-24.04 |
| 55 |
timeout-minutes: 100 |
| 56 |
needs: |
| 57 |
- build |
| 58 |
env: |
| 59 |
REGISTRY: gcr.io/k8s-staging-kas-network-proxy |
| 60 |
KIND_IMAGE: kindest/node:${{ matrix.k8s }} |
| 61 |
TAG: master |
| 62 |
CONNECTION_MODE: ${{ matrix.connection-mode }} |
| 63 |
strategy: |
| 64 |
fail-fast: false |
| 65 |
matrix: |
| 66 |
k8s: [ v1.30.8, v1.31.4, v1.32.0 ] |
| 67 |
connection-mode: [ grpc, http-connect ] |
| 68 |
steps: |
| 69 |
- name: Install kind |
| 70 |
run: | |
| 71 |
curl -Lo ./kind https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64 |
| 72 |
chmod +x ./kind |
| 73 |
sudo mv ./kind /usr/local/bin/kind |
| 74 |
- name: Check out code |
| 75 |
uses: actions/checkout@v4 |
| 76 |
- name: Set up Go |
| 77 |
uses: actions/setup-go@v5 |
| 78 |
with: |
| 79 |
go-version-file: go.mod |
| 80 |
id: go |
| 81 |
- name: Download prebuilt konnectivity-server image |
| 82 |
uses: actions/download-artifact@v4 |
| 83 |
with: |
| 84 |
name: konnectivity-server |
| 85 |
- name: Download prebuilt konnectivity-agent image |
| 86 |
uses: actions/download-artifact@v4 |
| 87 |
with: |
| 88 |
name: konnectivity-agent |
| 89 |
- name: Load prebuilt konnectivity images |
| 90 |
run: | |
| 91 |
docker load --input konnectivity-server.tar |
| 92 |
docker load --input konnectivity-agent.tar |
| 93 |
- name: Fix konnectivity docker image tags |
| 94 |
run: | |
| 95 |
docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-server:master gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:master |
| 96 |
docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:master |
| 97 |
- name: Run e2e tests |
| 98 |
run: make test-e2e-ci |
| 99 |
e2e: |
| 100 |
name: e2e |
| 101 |
runs-on: ubuntu-24.04 |
| 102 |
timeout-minutes: 100 |
| 103 |
needs: |
| 104 |
- build |
| 105 |
strategy: |
| 106 |
fail-fast: false |
| 107 |
matrix: |
| 108 |
ipFamily: ["ipv4", "ipv6", "dual"] |
| 109 |
k8s: [ v1.30.8, v1.31.4, v1.32.0 ] |
| 110 |
env: |
| 111 |
JOB_NAME: "kindnetd-e2e-${{ matrix.ipFamily }}" |
| 112 |
IP_FAMILY: ${{ matrix.ipFamily }} |
| 113 |
steps: |
| 114 |
- name: Check out code |
| 115 |
uses: actions/checkout@v4 |
| 116 |
|
| 117 |
- name: Enable ipv4 and ipv6 forwarding |
| 118 |
run: | |
| 119 |
sudo sysctl -w net.ipv6.conf.all.forwarding=1 |
| 120 |
sudo sysctl -w net.ipv4.ip_forward=1 |
| 121 |
|
| 122 |
- name: Set up environment (download dependencies) |
| 123 |
run: | |
| 124 |
TMP_DIR=$(mktemp -d) |
| 125 |
# Test binaries |
| 126 |
curl -L https://dl.k8s.io/${{ matrix.k8s }}/kubernetes-test-linux-amd64.tar.gz -o ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz |
| 127 |
tar xvzf ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz \ |
| 128 |
--directory ${TMP_DIR} \ |
| 129 |
--strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test |
| 130 |
# kubectl |
| 131 |
curl -L https://dl.k8s.io/${{ matrix.k8s }}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl |
| 132 |
# kind |
| 133 |
curl -Lo ${TMP_DIR}/kind https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64 |
| 134 |
# Install |
| 135 |
sudo cp ${TMP_DIR}/ginkgo /usr/local/bin/ginkgo |
| 136 |
sudo cp ${TMP_DIR}/e2e.test /usr/local/bin/e2e.test |
| 137 |
sudo cp ${TMP_DIR}/kubectl /usr/local/bin/kubectl |
| 138 |
sudo cp ${TMP_DIR}/kind /usr/local/bin/kind |
| 139 |
sudo chmod +x /usr/local/bin/ginkgo /usr/local/bin/e2e.test /usr/local/bin/kubectl /usr/local/bin/kind |
| 140 |
|
| 141 |
- name: Create multi node cluster |
| 142 |
run: | |
| 143 |
# output_dir |
| 144 |
mkdir -p _artifacts |
| 145 |
# create cluster |
| 146 |
cat <<EOF | /usr/local/bin/kind create cluster \ |
| 147 |
--name ${{ env.KIND_CLUSTER_NAME}} \ |
| 148 |
--image kindest/node:${{ matrix.k8s }} \ |
| 149 |
-v7 --wait 1m --retain --config=- |
| 150 |
kind: Cluster |
| 151 |
apiVersion: kind.x-k8s.io/v1alpha4 |
| 152 |
networking: |
| 153 |
ipFamily: ${IP_FAMILY} |
| 154 |
nodes: |
| 155 |
- role: control-plane |
| 156 |
image: ${{ matrix.k8s }} |
| 157 |
kubeadmConfigPatchesJSON6902: |
| 158 |
- kind: ClusterConfiguration |
| 159 |
patch: | |
| 160 |
- op: add |
| 161 |
path: /apiServer/certSANs/- |
| 162 |
value: konnectivity-server.kube-system.svc.cluster.local |
| 163 |
kubeadmConfigPatches: |
| 164 |
- | |
| 165 |
kind: ClusterConfiguration |
| 166 |
apiServer: |
| 167 |
extraArgs: |
| 168 |
"egress-selector-config-file": "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml" |
| 169 |
extraVolumes: |
| 170 |
- name: egress-selector-config-file |
| 171 |
hostPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml" |
| 172 |
mountPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml" |
| 173 |
readOnly: true |
| 174 |
- name: konnectivity-server |
| 175 |
hostPath: "/etc/kubernetes/konnectivity-server" |
| 176 |
mountPath: "/etc/kubernetes/konnectivity-server" |
| 177 |
readOnly: true |
| 178 |
extraMounts: |
| 179 |
- hostPath: ./examples/kind/egress_selector_configuration.yaml |
| 180 |
containerPath: /etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml |
| 181 |
- role: worker |
| 182 |
image: ${{ matrix.k8s }} |
| 183 |
- role: worker |
| 184 |
image: ${{ matrix.k8s }} |
| 185 |
EOF |
| 186 |
|
| 187 |
# dump the kubeconfig for later |
| 188 |
/usr/local/bin/kind get kubeconfig --name ${{ env.KIND_CLUSTER_NAME}} > _artifacts/kubeconfig.conf |
| 189 |
|
| 190 |
- uses: actions/download-artifact@v4 |
| 191 |
with: |
| 192 |
name: konnectivity-server |
| 193 |
|
| 194 |
- uses: actions/download-artifact@v4 |
| 195 |
with: |
| 196 |
name: konnectivity-agent |
| 197 |
|
| 198 |
- name: Install konnectivity |
| 199 |
run: | |
| 200 |
# preload konnectivity images |
| 201 |
docker load --input konnectivity-server.tar |
| 202 |
docker load --input konnectivity-agent.tar |
| 203 |
/usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-server:master --name ${{ env.KIND_CLUSTER_NAME}} |
| 204 |
/usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master --name ${{ env.KIND_CLUSTER_NAME}} |
| 205 |
kubectl apply -f examples/kind/konnectivity-server.yaml |
| 206 |
kubectl apply -f examples/kind/konnectivity-agent-ds.yaml |
| 207 |
|
| 208 |
- name: Get Cluster status |
| 209 |
run: | |
| 210 |
# wait network is ready |
| 211 |
sleep 5 |
| 212 |
/usr/local/bin/kubectl get nodes -o wide |
| 213 |
/usr/local/bin/kubectl get pods -A |
| 214 |
/usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns |
| 215 |
# smoke test |
| 216 |
/usr/local/bin/kubectl run test --image httpd:2 |
| 217 |
/usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods test |
| 218 |
/usr/local/bin/kubectl get pods -A -owide |
| 219 |
/usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=konnectivity-agent |
| 220 |
/usr/local/bin/kubectl logs test |
| 221 |
|
| 222 |
|
| 223 |
- name: Workaround CoreDNS for IPv6 airgapped |
| 224 |
if: ${{ matrix.ipFamily == 'ipv6' }} |
| 225 |
run: | |
| 226 |
# Patch CoreDNS to work in Github CI |
| 227 |
# 1. Github CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured |
| 228 |
# to work in an offline environment: |
| 229 |
# https://github.com/coredns/coredns/issues/2494#issuecomment-457215452 |
| 230 |
# 2. Github CI adds following domains to resolv.conf search field: |
| 231 |
# .net. |
| 232 |
# CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL |
| 233 |
# otherwise pods stops trying to resolve the domain. |
| 234 |
# Get the current config |
| 235 |
original_coredns=$(/usr/local/bin/kubectl get -oyaml -n=kube-system configmap/coredns) |
| 236 |
echo "Original CoreDNS config:" |
| 237 |
echo "${original_coredns}" |
| 238 |
# Patch it |
| 239 |
fixed_coredns=$( |
| 240 |
printf '%s' "${original_coredns}" | sed \ |
| 241 |
-e 's/^.*kubernetes cluster\.local/& net/' \ |
| 242 |
-e '/^.*upstream$/d' \ |
| 243 |
-e '/^.*fallthrough.*$/d' \ |
| 244 |
-e '/^.*forward . \/etc\/resolv.conf$/d' \ |
| 245 |
-e '/^.*loop$/d' \ |
| 246 |
) |
| 247 |
echo "Patched CoreDNS config:" |
| 248 |
echo "${fixed_coredns}" |
| 249 |
printf '%s' "${fixed_coredns}" | /usr/local/bin/kubectl apply -f - |
| 250 |
|
| 251 |
- name: Run tests |
| 252 |
run: | |
| 253 |
export KUBERNETES_CONFORMANCE_TEST='y' |
| 254 |
export E2E_REPORT_DIR=${PWD}/_artifacts |
| 255 |
|
| 256 |
# Run tests |
| 257 |
/usr/local/bin/ginkgo --nodes=25 \ |
| 258 |
--focus="\[Conformance\]" \ |
| 259 |
--skip="Feature|Federation|machinery|PerformanceDNS|DualStack|Disruptive|Serial|Slow|KubeProxy|LoadBalancer|GCE|Netpol|NetworkPolicy|NodeConformance" \ |
| 260 |
/usr/local/bin/e2e.test \ |
| 261 |
-- \ |
| 262 |
--kubeconfig=${PWD}/_artifacts/kubeconfig.conf \ |
| 263 |
--provider=local \ |
| 264 |
--dump-logs-on-failure=false \ |
| 265 |
--report-dir=${E2E_REPORT_DIR} \ |
| 266 |
--disable-log-dump=true |
| 267 |
|
| 268 |
- name: Upload Junit Reports |
| 269 |
if: always() |
| 270 |
uses: actions/upload-artifact@v4 |
| 271 |
with: |
| 272 |
name: kind-junit-${{ env.JOB_NAME }}-${{ matrix.k8s }}-${{ matrix.ipFamily }}-${{ github.run_id }} |
| 273 |
path: './_artifacts/*.xml' |
| 274 |
|
| 275 |
- name: Export logs |
| 276 |
if: always() |
| 277 |
run: | |
| 278 |
/usr/local/bin/kind export logs --name ${KIND_CLUSTER_NAME} -v 4 ./_artifacts/logs |
| 279 |
|
| 280 |
- name: Upload logs |
| 281 |
if: always() |
| 282 |
uses: actions/upload-artifact@v4 |
| 283 |
with: |
| 284 |
name: kind-logs-${{ env.JOB_NAME }}-${{ matrix.k8s }}-${{ matrix.ipFamily }}-${{ github.run_id }} |
| 285 |
path: ./_artifacts/logs |