From 1c529b91870d33bd91f58025c9f115f463fb8368 Mon Sep 17 00:00:00 2001 From: Ben Banfield-Zanin Date: Fri, 14 Nov 2025 16:14:52 +0000 Subject: [PATCH] Use k3d instead of kind --- .github/workflows/pytest.yml | 30 +++-- DEVELOPERS.md | 5 +- .../well-known-pytest-base-extras.yaml | 3 - ...t-matrix-rtc-synapse-wellknown-values.yaml | 2 - .../ci/pytest-well-known-values.yaml | 2 - .../matrix-stack/ci/test-cluster-mixin.yaml | 5 +- newsfragments/871.internal.md | 1 + poetry.lock | 39 ++++-- pyproject.toml | 1 + scripts/destroy_test_cluster.sh | 15 +-- scripts/setup_test_cluster.sh | 56 +++------ tests/integration/fixtures/__init__.py | 3 +- tests/integration/fixtures/cluster.py | 117 ++++++------------ .../fixtures/files/charts/ingress-nginx.yml | 20 --- .../fixtures/files/charts/metrics-server.yml | 7 -- .../fixtures/files/clusters/k3d.yml | 57 +++++++++ .../fixtures/files/clusters/kind.yml | 65 ---------- .../local_mirrors/localhost_5000_/hosts.toml | 7 -- tests/integration/fixtures/helm.py | 25 +++- tests/integration/fixtures/matrix_tools.py | 2 +- 20 files changed, 200 insertions(+), 262 deletions(-) create mode 100644 newsfragments/871.internal.md delete mode 100644 tests/integration/fixtures/files/charts/ingress-nginx.yml delete mode 100644 tests/integration/fixtures/files/charts/metrics-server.yml create mode 100644 tests/integration/fixtures/files/clusters/k3d.yml delete mode 100644 tests/integration/fixtures/files/clusters/kind.yml delete mode 100644 tests/integration/fixtures/files/clusters/local_mirrors/localhost_5000_/hosts.toml diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index a26f70f6..a28cbe2f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -79,12 +79,11 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - uses: engineerd/setup-kind@ecfad61750951586a9ef973db567df1d28671bdc # v0.6.2 + - name: Install k3d with asdf + uses: asdf-vm/actions/install@1902764435ca0dd2f3388eea723a4f92a4eb8302 # v4 with: - version: "v0.29.0" - name: "ess-helm" - skipClusterCreation: "true" - skipClusterLogsExport: "true" + tool_versions: | + k3d 5.8.3 - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 @@ -124,17 +123,24 @@ jobs: if: ${{ failure() }} shell: bash run: | - kind export logs --name ess-helm ./ess-helm-logs - kind export kubeconfig --name ess-helm - ns=$(kubectl --context kind-ess-helm get ns -l app.kubernetes.io/managed-by=pytest -o jsonpath='{.items[].metadata.name}') + mkdir ess-helm-logs + k3d kubeconfig merge ess-helm -ds + for ns in $(kubectl --context k3d-ess-helm get ns -o custom-columns=NS:.metadata.name --no-headers); do + mkdir -p "./ess-helm-logs/$ns" + for pod in $(kubectl --context k3d-ess-helm -n "$ns" get pod -o custom-columns=NS:.metadata.name --no-headers); do + kubectl --context k3d-ess-helm -n "$ns" logs --all-containers --prefix --timestamps --ignore-errors --previous "$pod" > "./ess-helm-logs/$ns/$pod.previous" + kubectl --context k3d-ess-helm -n "$ns" logs --all-containers --prefix --timestamps --ignore-errors "$pod" > "./ess-helm-logs/$ns/$pod.logs" + done + done + ess_ns=$(kubectl --context k3d-ess-helm get ns -l app.kubernetes.io/managed-by=pytest -o jsonpath='{.items[].metadata.name}') resources=("pods" "deployments" "statefulsets" "services" "configmaps" "ingresses" "persistentvolumes" "persistentvolumeclaims" "endpoints") for i in "${resources[@]}"; do - kubectl --context kind-ess-helm get "$i" -n "$ns" > "./ess-helm-logs/$i.txt" + kubectl --context k3d-ess-helm get "$i" -n "$ess_ns" > "./ess-helm-logs/$i.txt" echo "----" >> "./ess-helm-logs/$i.txt" - kubectl --context kind-ess-helm get "$i" -o yaml -n "$ns" >> "./ess-helm-logs/$i.txt" + kubectl --context k3d-ess-helm get "$i" -o yaml -n "$ess_ns" >> "./ess-helm-logs/$i.txt" done - kubectl --context kind-ess-helm get events --sort-by=.metadata.creationTimestamp -n "$ns" > ./ess-helm-logs/events.txt - kind delete cluster --name ess-helm + kubectl --context k3d-ess-helm get events --sort-by=.metadata.creationTimestamp -n "$ess_ns" > ./ess-helm-logs/events.txt + k3d cluster delete ess-helm - name: Upload logs if: ${{ failure() }} diff --git a/DEVELOPERS.md b/DEVELOPERS.md index 1199f63b..562e170a 100644 --- a/DEVELOPERS.md +++ b/DEVELOPERS.md @@ -16,6 +16,7 @@ Requirements for development: Optional Tools: * [`chart-testing`](https://github.com/helm/chart-testing) for Helm linting +* [`k3d`](https://k3d.io/stable/) for running the test cluster * [`kubeconform`](https://github.com/yannh/kubeconform) for Kubernetes manifest validation * [`shellcheck`](https://www.shellcheck.net/) * Managed via Poetry and so should be available after `poetry install` @@ -159,10 +160,10 @@ run `pytest test --cache-clear`. #### Special env variables - `PYTEST_KEEP_CLUSTER=1` : Do not destroy the cluster at the end of the test run. -You must delete it using `kind delete cluster --name ess-helm` manually before running any other test run. +You must delete it using `k3d cluster delete ess-helm` manually before running any other test run. #### Usage -Use `kind export kubeconfig --name ess-helm` to get access to the cluster. +Use `k3d kubeconfig merge ess-helm -ds` to get access to the cluster. The tests will use the cluster constructed by `scripts/setup_test_cluster.sh` if that is running. If the tests use an existing cluster, they won't destroy the cluster afterwards. diff --git a/charts/matrix-stack/ci/fragments/well-known-pytest-base-extras.yaml b/charts/matrix-stack/ci/fragments/well-known-pytest-base-extras.yaml index 460b92bb..7d151c37 100644 --- a/charts/matrix-stack/ci/fragments/well-known-pytest-base-extras.yaml +++ b/charts/matrix-stack/ci/fragments/well-known-pytest-base-extras.yaml @@ -3,9 +3,6 @@ # # SPDX-License-Identifier: AGPL-3.0-only -ingress: - controllerType: ingress-nginx - wellKnownDelegation: ingress: tlsSecret: "{{ $.Release.Name }}-well-known-web-tls" diff --git a/charts/matrix-stack/ci/pytest-matrix-rtc-synapse-wellknown-values.yaml b/charts/matrix-stack/ci/pytest-matrix-rtc-synapse-wellknown-values.yaml index 14ae703a..74878035 100644 --- a/charts/matrix-stack/ci/pytest-matrix-rtc-synapse-wellknown-values.yaml +++ b/charts/matrix-stack/ci/pytest-matrix-rtc-synapse-wellknown-values.yaml @@ -16,8 +16,6 @@ haproxy: podSecurityContext: runAsGroup: 0 replicas: 2 -ingress: - controllerType: ingress-nginx initSecrets: annotations: has-no-service-monitor: "true" diff --git a/charts/matrix-stack/ci/pytest-well-known-values.yaml b/charts/matrix-stack/ci/pytest-well-known-values.yaml index 9a3f1951..35002ae0 100644 --- a/charts/matrix-stack/ci/pytest-well-known-values.yaml +++ b/charts/matrix-stack/ci/pytest-well-known-values.yaml @@ -18,8 +18,6 @@ global: haproxy: podSecurityContext: runAsGroup: 0 -ingress: - controllerType: ingress-nginx matrixAuthenticationService: enabled: false matrixRTC: diff --git a/charts/matrix-stack/ci/test-cluster-mixin.yaml b/charts/matrix-stack/ci/test-cluster-mixin.yaml index b50e3743..6ae72fa4 100644 --- a/charts/matrix-stack/ci/test-cluster-mixin.yaml +++ b/charts/matrix-stack/ci/test-cluster-mixin.yaml @@ -10,9 +10,6 @@ certManager: clusterIssuer: ess-selfsigned -ingress: - controllerType: ingress-nginx - matrixRTC: # Because the authoriser service won't trust certificates issued by the above self-signed CA extraEnv: @@ -24,4 +21,4 @@ matrixRTC: - ess.localhost - mrtc.ess.localhost - synapse.ess.localhost - ip: '{{ ( (lookup "v1" "Service" "ingress-nginx" "ingress-nginx-controller") | default (dict "spec" (dict "clusterIP" "127.0.0.1")) ).spec.clusterIP }}' + ip: '{{ ( (lookup "v1" "Service" "kube-system" "traefik") | default (dict "spec" (dict "clusterIP" "127.0.0.1")) ).spec.clusterIP }}' diff --git a/newsfragments/871.internal.md b/newsfragments/871.internal.md new file mode 100644 index 00000000..1b48541c --- /dev/null +++ b/newsfragments/871.internal.md @@ -0,0 +1 @@ +CI: switch from kind to k3d for integration tests. diff --git a/poetry.lock b/poetry.lock index 2e5dafda..cc3476a8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "aiodns" @@ -1407,6 +1407,21 @@ http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "httpx-retries" +version = "0.4.5" +description = "A retry layer for HTTPX." +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "httpx_retries-0.4.5-py3-none-any.whl", hash = "sha256:ae22d6ef197a2da49242246a01d721474cbd6516b1fef155f6da694ee410bb37"}, + {file = "httpx_retries-0.4.5.tar.gz", hash = "sha256:acee306d7384eefad71ac12fefe8b13d7b41c19595c538e68d9bd7e40e59539d"}, +] + +[package.dependencies] +httpx = ">=0.20.0" + [[package]] name = "hyperframe" version = "6.1.0" @@ -2625,14 +2640,14 @@ files = [ [[package]] name = "pydantic" -version = "2.12.4" +version = "2.12.5" description = "Data validation using Python type hints" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e"}, - {file = "pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac"}, + {file = "pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d"}, + {file = "pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49"}, ] [package.dependencies] @@ -3035,6 +3050,13 @@ optional = false python-versions = ">=3.8" groups = ["dev"] files = [ + {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, + {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, + {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, + {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, @@ -3105,14 +3127,14 @@ files = [ [[package]] name = "rdflib" -version = "7.4.0" +version = "7.5.0" description = "RDFLib is a Python library for working with RDF, a simple yet powerful language for representing information." optional = false python-versions = ">=3.8.1" groups = ["dev"] files = [ - {file = "rdflib-7.4.0-py3-none-any.whl", hash = "sha256:0af003470404ff21bc0eb04077cc97ee96da581f2429bf42a8e163fc1c2797bc"}, - {file = "rdflib-7.4.0.tar.gz", hash = "sha256:c8ee16c31848c19c174aed96185327ea139ca3d392fac7fa882ddf5687f8f533"}, + {file = "rdflib-7.5.0-py3-none-any.whl", hash = "sha256:b011dfc40d0fc8a44252e906dcd8fc806a7859bc231be190c37e9568a31ac572"}, + {file = "rdflib-7.5.0.tar.gz", hash = "sha256:663083443908b1830e567350d72e74d9948b310f827966358d76eebdc92bf592"}, ] [package.dependencies] @@ -3124,6 +3146,7 @@ html = ["html5rdf (>=1.2,<2)"] lxml = ["lxml (>=4.3,<6.0)"] networkx = ["networkx (>=2,<4)"] orjson = ["orjson (>=3.9.14,<4)"] +rdf4j = ["httpx (>=0.28.1,<0.29.0)"] [[package]] name = "referencing" @@ -4228,4 +4251,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "f72508bff3762d1effecd30601cf33e9d2325435f71b070ce9274a730974ef03" +content-hash = "71fdf35ede388f44f52caa847f3e3757e907df3ea9571659ceca2fede5db7557" diff --git a/pyproject.toml b/pyproject.toml index 33973be1..af40e35d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ types-pyyaml = "^6.0.12.20250915" semver = "^3.0.4" prometheus-client = "^0.23.1" yamllint = "^1.37.1" +httpx-retries = "^0.4.5" [build-system] requires = ["poetry-core>=2.1.0"] diff --git a/scripts/destroy_test_cluster.sh b/scripts/destroy_test_cluster.sh index bf4e8f59..ede58eb3 100755 --- a/scripts/destroy_test_cluster.sh +++ b/scripts/destroy_test_cluster.sh @@ -7,17 +7,10 @@ set -e -kind_cluster_name="ess-helm" +k3d_cluster_name="ess-helm" -if kind get clusters 2> /dev/null| grep "$kind_cluster_name"; then - kind delete cluster --name $kind_cluster_name +if k3d cluster list 2> /dev/null | grep "$k3d_cluster_name"; then + k3d cluster delete $k3d_cluster_name else - echo "Kind cluster ${kind_cluster_name} already destoryed" -fi - -if docker ps -a | grep "${kind_cluster_name}-registry"; then - docker stop "${kind_cluster_name}-registry" || true - docker rm "${kind_cluster_name}-registry" || true -else - echo "Kind cluster's local registry already destroyed" + echo "k3d cluster ${k3d_cluster_name} already destoryed" fi diff --git a/scripts/setup_test_cluster.sh b/scripts/setup_test_cluster.sh index db3fa952..97fae800 100755 --- a/scripts/setup_test_cluster.sh +++ b/scripts/setup_test_cluster.sh @@ -7,8 +7,8 @@ set -e -kind_cluster_name="ess-helm" -kind_context_name="kind-$kind_cluster_name" +k3d_cluster_name="ess-helm" +k3d_context_name="k3d-$k3d_cluster_name" # Space separated list of namespaces to use ess_namespaces=${ESS_NAMESPACES:-ess} @@ -16,43 +16,25 @@ root_folder="$(git rev-parse --show-toplevel)" ca_folder="$root_folder/.ca" mkdir -p "$ca_folder" -if docker ps -a | grep "${kind_cluster_name}-registry"; then - docker stop "${kind_cluster_name}-registry" || true - docker rm "${kind_cluster_name}-registry" || true -fi -if kind get clusters 2>/dev/null | grep "$kind_cluster_name"; then - echo "Cluster '$kind_cluster_name' is already provisioned by Kind" +if k3d cluster list 2>/dev/null | grep "$k3d_cluster_name"; then + echo "Cluster '$k3d_cluster_name' is already provisioned by k3d" else - echo "Creating new Kind cluster '$kind_cluster_name'" - (cd "$root_folder/tests/integration/fixtures/files/clusters"; kind create cluster --name "$kind_cluster_name" --config "kind.yml") + echo "Creating new k3d cluster '$k3d_cluster_name'" + k3d cluster create "$k3d_cluster_name" --config "tests/integration/fixtures/files/clusters/k3d.yml" fi -network=$(docker inspect $kind_cluster_name-control-plane | jq '.[0].NetworkSettings.Networks | keys | .[0]' -r) -docker run \ - -d --restart=always -p "127.0.0.1:5000:5000" --network "$network" --network-alias "registry" --name "${kind_cluster_name}-registry" \ - registry:2 - -helm --kube-context $kind_context_name upgrade -i ingress-nginx --repo https://kubernetes.github.io/ingress-nginx ingress-nginx \ - --namespace ingress-nginx \ - --create-namespace \ - -f "$root_folder/tests/integration/fixtures/files/charts/ingress-nginx.yml" - -helm --kube-context $kind_context_name upgrade -i metrics-server --repo https://kubernetes-sigs.github.io/metrics-server metrics-server \ - --namespace kube-system \ - -f "$root_folder/tests/integration/fixtures/files/charts/metrics-server.yml" - -helm --kube-context $kind_context_name upgrade -i prometheus-operator-crds --repo https://prometheus-community.github.io/helm-charts prometheus-operator-crds \ +helm --kube-context $k3d_context_name upgrade -i prometheus-operator-crds --repo https://prometheus-community.github.io/helm-charts prometheus-operator-crds \ --namespace prometheus-operator \ --create-namespace -helm --kube-context $kind_context_name upgrade -i cert-manager --repo https://charts.jetstack.io cert-manager \ +helm --kube-context $k3d_context_name upgrade -i cert-manager --repo https://charts.jetstack.io cert-manager \ --namespace cert-manager \ --create-namespace \ -f "$root_folder/tests/integration/fixtures/files/charts/cert-manager.yml" # Create a new CA certificate if [[ ! -f "$ca_folder"/ca.crt || ! -f "$ca_folder"/ca.pem ]]; then - cat </dev/null || true - kubectl --context $kind_context_name -n cert-manager delete Certificate ess-ca 2>/dev/null || true - kubectl --context $kind_context_name -n cert-manager delete Secret ess-ca 2>/dev/null || true - kubectl --context $kind_context_name -n cert-manager create secret generic ess-ca \ + kubectl --context $k3d_context_name delete ClusterIssuer ess-ca 2>/dev/null || true + kubectl --context $k3d_context_name -n cert-manager delete Certificate ess-ca 2>/dev/null || true + kubectl --context $k3d_context_name -n cert-manager delete Secret ess-ca 2>/dev/null || true + kubectl --context $k3d_context_name -n cert-manager create secret generic ess-ca \ --type=kubernetes.io/tls \ --from-file=tls.crt="$ca_folder"/ca.crt \ --from-file=tls.key="$ca_folder"/ca.pem \ --from-file=ca.crt="$ca_folder"/ca.crt fi -cat < "$ca_folder"/ca.crt - kubectl --context $kind_context_name -n cert-manager get secret ess-ca -o jsonpath="{.data['tls\.key']}" | base64 -d > "$ca_folder"/ca.pem + kubectl --context $k3d_context_name -n cert-manager get secret ess-ca -o jsonpath="{.data['ca\.crt']}" | base64 -d > "$ca_folder"/ca.crt + kubectl --context $k3d_context_name -n cert-manager get secret ess-ca -o jsonpath="{.data['tls\.key']}" | base64 -d > "$ca_folder"/ca.pem fi for namespace in $ess_namespaces; do echo "Constructing ESS dependencies in $namespace" - server_version=$(kubectl --context $kind_context_name version | grep Server | sed 's/.*v/v/' | awk -F. '{print $1"."$2}') + server_version=$(kubectl --context $k3d_context_name version | grep Server | sed 's/.*v/v/' | awk -F. '{print $1"."$2}') # We don't turn on enforce here as people may be experimenting but we do turn on warn so people see the warnings when helm install/upgrade - cat <", str(cluster_options.kubeconfig_path), ] ) @@ -64,9 +64,9 @@ class PotentiallyExistingKindCluster(KindManagerBase): @pytest.fixture(autouse=True, scope="session") async def cluster(): # Both these names must match what `setup_test_cluster.sh` would create - this_cluster = PotentiallyExistingKindCluster("ess-helm") + this_cluster = PotentiallyExistingK3dCluster("ess-helm") this_cluster.create( - ClusterOptions(cluster_name="ess-helm", provider_config=Path(__file__).parent / Path("files/clusters/kind.yml")) + ClusterOptions(cluster_name="ess-helm", provider_config=Path(__file__).parent / Path("files/clusters/k3d.yml")) ) yield this_cluster @@ -82,78 +82,39 @@ async def helm_client(cluster): @pytest.fixture(scope="session") async def kube_client(cluster): kube_config = KubeConfig.from_file(cluster.kubeconfig) - return AsyncClient(config=kube_config) + config = kube_config.get() - -@pytest.fixture(autouse=True, scope="session") -async def ingress(cluster, kube_client, helm_client: pyhelm3.Client): - chart = await helm_client.get_chart("ingress-nginx", repo="https://kubernetes.github.io/ingress-nginx") - - values_file = Path(__file__).parent.resolve() / Path("files/charts/ingress-nginx.yml") - # Install or upgrade a release - await helm_client.install_or_upgrade_release( - "ingress-nginx", - chart, - yaml.safe_load(values_file.read_text("utf-8")), - namespace="ingress-nginx", - create_namespace=True, - atomic=True, - wait=True, + # We've seen 429 errors with storage is (re)initializing. Let's retry those + ssl_context = verify_cluster(config.cluster, config.user, config.abs_file) + wrapped_transport = httpx.AsyncHTTPTransport(verify=ssl_context) + transport = httpx_retries.RetryTransport( + transport=wrapped_transport, retry=httpx_retries.Retry(status_forcelist=[429]) ) - - await asyncio.to_thread( - cluster.wait, - name="endpoints/ingress-nginx-controller-admission", - waitfor="jsonpath='{.subsets[].addresses}'", - namespace="ingress-nginx", - ) - await asyncio.to_thread( - cluster.wait, - name="lease/ingress-nginx-leader", - waitfor="jsonpath='{.spec.holderIdentity}'", - namespace="ingress-nginx", - ) - return (await kube_client.get(Service, name="ingress-nginx-controller", namespace="ingress-nginx")).spec.clusterIP + return AsyncClient(config=kube_config, transport=transport) -@pytest.fixture(autouse=True, scope="session") -async def registry(cluster): - pytest_registry_container_name = "pytest-ess-helm-registry" - test_cluster_registry_container_name = "ess-helm-registry" - - # We have a registry created by `setup_test_cluster.sh` - if docker.container.exists(test_cluster_registry_container_name): - container_name = test_cluster_registry_container_name - # We have a registry created by a previous run of pytest - elif docker.container.exists(pytest_registry_container_name): - container_name = pytest_registry_container_name - # We have no registry, create one - else: - container_name = pytest_registry_container_name - container = docker.run( - name=container_name, - image="registry:2", - publish=[("127.0.0.1:5000", "5000")], - restart="always", - detach=True, - ) - - container = docker.container.inspect(container_name) - if not container.state.running: - container.start() - - kind_network = docker.network.inspect("kind") - if container.id not in kind_network.containers: - docker.network.connect(kind_network, container, alias="registry") - - yield - - if container_name == pytest_registry_container_name: - container.stop() - container.remove() +@pytest.fixture(scope="session") +async def ingress(cluster, kube_client): + attempt = 0 + while attempt < 120: + try: + # We can't just kubectl wait as that doesn't work with non-existent objects + # This can be setup before the LB port is accessible externally, so we do it afterwards + service = await kube_client.get(Service, name="traefik", namespace="kube-system") + await asyncio.to_thread( + cluster.wait, + name="service/traefik", + waitfor="jsonpath='{.status.loadBalancer.ingress[0].ip}'", + namespace="kube-system", + ) + return service.spec.clusterIP + except ApiError: + await asyncio.sleep(1) + attempt += 1 + raise Exception("Couldn't fetch Trafeik Service IP afrter 120s") -@pytest.fixture(autouse=True, scope="session") +@pytest.fixture(scope="session") async def prometheus_operator_crds(helm_client): if os.environ.get("SKIP_SERVICE_MONITORS_CRDS", "false") == "false": chart = await helm_client.get_chart( @@ -173,7 +134,7 @@ async def prometheus_operator_crds(helm_client): @pytest.fixture(scope="session") -async def ess_namespace(cluster: PotentiallyExistingKindCluster, kube_client: AsyncClient, generated_data: ESSData): +async def ess_namespace(cluster: PotentiallyExistingK3dCluster, kube_client: AsyncClient, generated_data: ESSData): (major_version, minor_version) = cluster.version() try: await kube_client.get(Namespace, name=generated_data.ess_namespace) diff --git a/tests/integration/fixtures/files/charts/ingress-nginx.yml b/tests/integration/fixtures/files/charts/ingress-nginx.yml deleted file mode 100644 index 618404ec..00000000 --- a/tests/integration/fixtures/files/charts/ingress-nginx.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2024 New Vector Ltd -# Copyright 2025 Element Creations Ltd -# -# SPDX-License-Identifier: AGPL-3.0-only - -controller: - ingressClassResource: - default: true - - config: - hsts: false - - hostPort: - enabled: true - - allowSnippetAnnotations: true - - service: - type: ClusterIP - enabled: true diff --git a/tests/integration/fixtures/files/charts/metrics-server.yml b/tests/integration/fixtures/files/charts/metrics-server.yml deleted file mode 100644 index 2458d0b7..00000000 --- a/tests/integration/fixtures/files/charts/metrics-server.yml +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2024 New Vector Ltd -# Copyright 2025 Element Creations Ltd -# -# SPDX-License-Identifier: AGPL-3.0-only - -args: -- --kubelet-insecure-tls diff --git a/tests/integration/fixtures/files/clusters/k3d.yml b/tests/integration/fixtures/files/clusters/k3d.yml new file mode 100644 index 00000000..97200c55 --- /dev/null +++ b/tests/integration/fixtures/files/clusters/k3d.yml @@ -0,0 +1,57 @@ +# Copyright 2024-2025 New Vector Ltd +# Copyright 2025 Element Creations Ltd +# +# SPDX-License-Identifier: AGPL-3.0-only + +apiVersion: k3d.io/v1alpha5 +kind: Simple +metadata: + name: ess-helm +# Until k3d comes with a recent k8s/k3s version +image: rancher/k3s:v1.34.2-k3s1 +agents: 0 +servers: 1 +ports: +- port: 127.0.0.1:80:80 + nodeFilters: [loadbalancer] +- port: 127.0.0.1:443:443 + nodeFilters: [loadbalancer] + # Matrix RTC SFU TCP and Muxed UDP +- port: 127.0.0.1:30881:30881 + nodeFilters: [loadbalancer] +- port: 127.0.0.1:30882:30882/UDP + nodeFilters: [loadbalancer] +options: + k3d: + wait: true + k3s: + extraArgs: + - arg: "--kubelet-arg=container-log-max-size=100Mi" + nodeFilters: [servers:*] + - arg: "--kubelet-arg=container-log-max-files=10" + nodeFilters: [servers:*] + - arg: "--kube-apiserver-arg=audit-log-path=/var/log/kubernetes/kube-apiserver-audit.log" + nodeFilters: [servers:*] + - arg: "--kube-apiserver-arg=audit-policy-file=/etc/kubernetes/policies/audit-policy.yaml" + nodeFilters: [servers:*] + runtime: + ulimits: + - name: nofile + soft: 100000 + hard: 100000 +files: +- source: audit-policy.yml + destination: /etc/kubernetes/policies/audit-policy.yaml + nodeFilters: [servers:*] +registries: + create: + name: k3d-ess-helm-registry + host: 127.0.0.1 + hostPort: "5000" + # The k3d created registry will be available at localhost:5000 externally and so that's what we'll tag imges with + # The below makes it available internally at the same address + config: | + mirrors: + "localhost:5000": + endpoint: + - http://k3d-ess-helm-registry:5000 diff --git a/tests/integration/fixtures/files/clusters/kind.yml b/tests/integration/fixtures/files/clusters/kind.yml deleted file mode 100644 index 90f4d89e..00000000 --- a/tests/integration/fixtures/files/clusters/kind.yml +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2024-2025 New Vector Ltd -# Copyright 2025 Element Creations Ltd -# -# SPDX-License-Identifier: AGPL-3.0-only - -apiVersion: kind.x-k8s.io/v1alpha4 -kind: Cluster -name: ess-helm -nodes: -- role: control-plane - kubeadmConfigPatches: - - |- - kind: InitConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: "ingress-ready=true" - - |- - apiVersion: kubelet.config.k8s.io/v1beta1 - kind: KubeletConfiguration - containerLogMaxSize: 100Mi - containerLogMaxFiles: 10 - - | - kind: ClusterConfiguration - apiServer: - # enable auditing flags on the API server - extraArgs: - audit-log-path: /var/log/kubernetes/kube-apiserver-audit.log - audit-policy-file: /etc/kubernetes/policies/audit-policy.yaml - # mount new files / directories on the control plane - extraVolumes: - - name: audit-policies - hostPath: /etc/kubernetes/policies - mountPath: /etc/kubernetes/policies - readOnly: true - pathType: "DirectoryOrCreate" - - name: "audit-logs" - hostPath: "/var/log/kubernetes" - mountPath: "/var/log/kubernetes" - readOnly: false - pathType: DirectoryOrCreate - extraMounts: - - hostPath: ./audit-policy.yml - containerPath: /etc/kubernetes/policies/audit-policy.yaml - readOnly: true - - hostPath: ./local_mirrors - containerPath: /etc/containerd/certs.d - readOnly: true - extraPortMappings: - - containerPort: 80 - hostPort: 80 - protocol: TCP - - containerPort: 443 - hostPort: 443 - protocol: TCP - # Matrix RTC SFU TCP and Muxed UDP - - containerPort: 30881 - hostPort: 30881 - protocol: TCP - - containerPort: 30882 - hostPort: 30882 - protocol: UDP -containerdConfigPatches: -- |- - [plugins."io.containerd.grpc.v1.cri".registry] - config_path = "/etc/containerd/certs.d" diff --git a/tests/integration/fixtures/files/clusters/local_mirrors/localhost_5000_/hosts.toml b/tests/integration/fixtures/files/clusters/local_mirrors/localhost_5000_/hosts.toml deleted file mode 100644 index e71b3794..00000000 --- a/tests/integration/fixtures/files/clusters/local_mirrors/localhost_5000_/hosts.toml +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2025 New Vector Ltd -# Copyright 2025 Element Creations Ltd -# -# SPDX-License-Identifier: AGPL-3.0-only - -[host."http://registry:5000"] - capabilities = ["pull", "resolve"] diff --git a/tests/integration/fixtures/helm.py b/tests/integration/fixtures/helm.py index ebb8fb22..a7db0041 100644 --- a/tests/integration/fixtures/helm.py +++ b/tests/integration/fixtures/helm.py @@ -7,6 +7,7 @@ import asyncio import base64 import os from collections.abc import Awaitable +from ssl import SSLCertVerificationError, SSLContext import pyhelm3 import pytest @@ -155,6 +156,7 @@ async def matrix_stack( helm_client: pyhelm3.Client, ingress, helm_prerequisites, + prometheus_operator_crds, ess_namespace: Namespace, generated_data: ESSData, loaded_matrix_tools: dict, @@ -197,7 +199,7 @@ async def matrix_stack( @pytest.fixture(scope="session") -def ingress_ready(cluster, kube_client: AsyncClient, matrix_stack, generated_data: ESSData): +def ingress_ready(cluster, kube_client: AsyncClient, matrix_stack, generated_data: ESSData, ssl_context: SSLContext): async def _ingress_ready(ingress_suffix): await asyncio.to_thread( cluster.wait, @@ -215,6 +217,27 @@ def ingress_ready(cluster, kube_client: AsyncClient, matrix_stack, generated_dat ) await wait_for_endpoint_ready(service.metadata.name, generated_data.ess_namespace, cluster, kube_client) + if rule.host: + attempt = 0 + while attempt < 20: + try: + # Wait for the port and certificate to be available + _, writer = await asyncio.wait_for( + asyncio.open_connection("127.0.0.1", 443, ssl=ssl_context, server_hostname=rule.host), + timeout=1, + ) + writer.close() + await writer.wait_closed() + break + except (ConnectionResetError, ConnectionRefusedError, SSLCertVerificationError, TimeoutError): + await asyncio.sleep(1) + attempt += 1 + else: + raise Exception( + f"Unable to connect to Ingress/{generated_data.release_name}-{ingress_suffix}" + " externally after 20s" + ) + return _ingress_ready diff --git a/tests/integration/fixtures/matrix_tools.py b/tests/integration/fixtures/matrix_tools.py index 91c19fea..eaa79c6e 100644 --- a/tests/integration/fixtures/matrix_tools.py +++ b/tests/integration/fixtures/matrix_tools.py @@ -25,7 +25,7 @@ async def build_matrix_tools(): @pytest.fixture(autouse=True, scope="session") -async def loaded_matrix_tools(registry, build_matrix_tools: Image): +async def loaded_matrix_tools(cluster, build_matrix_tools: Image): # Until the image is made publicly available # In local runs we always have to build it if os.environ.get("BUILD_MATRIX_TOOLS"):