From 28168c4168d75069714b8414454934636cffb576 Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Fri, 3 Oct 2025 11:16:53 +1300 Subject: [PATCH] Add a kubernetes provider for console container A new ``ironic.console.container`` provider is added called ``kubernetes`` which allows Ironic conductor to manage console containers as Kubernetes pods. The kubernetes resources are defined in the template file configured by ``[vnc]kubernetes_container_template`` and the default template creates one secret to store the app info, and one pod to run the console container. It is expected that Ironic conductor is deployed inside the kubernetes cluster. The associated service account will need roles and bindings which allow it to manage the required resources (with the default template this will be secrets and pods). This provider holds the assumption that ironic-novnc will be deployed in the same kubernetes cluster, and so can connect to the VNC servers via the pod's ``status.hostIP``. Assisted-By: gemini Change-Id: Ib91f7d7c15be51d68ebf886e44efaf191a14437b Signed-off-by: Steve Baker --- doc/source/install/graphical-console.rst | 67 +- ironic/conf/vnc.py | 23 +- .../ironic-console-pod.yaml.template | 45 ++ ironic/console/container/kubernetes.py | 307 +++++++ .../container/test_console_container.py | 764 +++++++++++++++++- pyproject.toml | 1 + .../notes/console-k8s-b4aee1bb1d3d0a65.yaml | 18 + 7 files changed, 1216 insertions(+), 9 deletions(-) create mode 100644 ironic/console/container/ironic-console-pod.yaml.template create mode 100644 ironic/console/container/kubernetes.py create mode 100644 releasenotes/notes/console-k8s-b4aee1bb1d3d0a65.yaml diff --git a/doc/source/install/graphical-console.rst b/doc/source/install/graphical-console.rst index 1875ffd2cd..07bb8ded88 100644 --- a/doc/source/install/graphical-console.rst +++ b/doc/source/install/graphical-console.rst @@ -70,16 +70,72 @@ especially when Ironic itself is deployed in a containerized environment. Systemd container provider ~~~~~~~~~~~~~~~~~~~~~~~~~~ -The only functional container provider included is the systemd provider which -manages containers as Systemd Quadlet containers. This provider is appropriate -to use when the Ironic services themselves are not containerised, and is also -a good match when ironic-conductor itself is managed as a Systemd unit. +The ``systemd`` provider manages containers as Systemd Quadlet containers. +This provider is appropriate to use when the Ironic services themselves are +not containerised, and is also a good match when ironic-conductor itself is +managed as a Systemd unit. To start a container, this provider writes ``.container`` files to ``/etc/containers/systemd/users/{uid}/containers/systemd`` then calls ``systemctl --user daemon-reload`` to generate a unit file which is then started with ``systemctl --user start {unit name}``. +Kubernetes container provider +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``kubernetes`` provider manages containers as kubernetes pods and allows +associated resources to also be managed. The provider requires the ``kubectl`` +command, and valid kubernetes credentials be available to the running +ironic-conductor. The current assumption with this driver is that +ironic-conductor, ironic-novncproxy, and the console containers are all +running in the same kubernetes cluster. Therefore, the credentials will be +provided by the service account mechanism supplied to the ironic-conductor +pod. + +``ironic.conf`` ``[vnc]kubernetes_container_template`` points to a template +file which defines the kubernetes resources including the pod running the +console container. The default template creates one Secret to store the app +info (including BMC credentials) and one Pod to run the actual console +container. This default template ``ironic-console-pod.yaml.template`` is +functional but will likely need to be replaced with a variant that +customises: + +* The namespace the resources are deployed to +* The labels to match the conventions of the deployment + +When ironic-conductor starts and stops it will stop any existing console +container associated with that ironic-conductor. For this delete-all +operation, the labels in the template are transformed into a kubectl selector, +so this needs to be a consideration when choosing the labels in the template. + +When ironic-conductor is using cluster service account credentials, a +RoleBinding to a Role which allows appropriate resource management is +required. For example, the default template would require at minimum the +following role rules:: + + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + # ... + rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + +The provider assumes that ironic-novnc is running in the cluster, and can +connect a VNC server using the console container's ``hostIP``. + Creating an external container provider ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -118,7 +174,8 @@ server exposed by these containers, and so does nova-novncproxy when Nova is using the Ironic driver. For the ``systemd`` container the VNC server will be published on a random -high port number. +high port number. For the ``kubernetes`` pod the VNC server is running on +port ``5900`` on the pod's ``hostIP``. Console containers need access to the management network to access the BMC web interface. If driver_info ``redfish_verify_ca=False`` then web requests will diff --git a/ironic/conf/vnc.py b/ironic/conf/vnc.py index 3dda69fa38..532574876f 100644 --- a/ironic/conf/vnc.py +++ b/ironic/conf/vnc.py @@ -98,11 +98,14 @@ opts = [ '"systemd" manages containers as systemd units via podman ' 'Quadlet support. The default is "fake" which returns an ' 'unusable VNC host and port. This needs to be changed if enabled ' - 'is True'), + 'is True. ' + '"kubernetes" manages containers as pods using template driven ' + 'resource creation.'), cfg.StrOpt( 'console_image', mutable=True, - help='Container image reference for the "systemd" console container ' + help='Container image reference for the "systemd" and ' + '"kubernetes" console container ' 'provider, and any other out-of-tree provider which requires a ' 'configurable image reference.'), cfg.StrOpt( @@ -126,6 +129,22 @@ opts = [ 'have no authentication or encryption so they also should not ' 'be exposed to public access. Additionally, the containers ' 'need to be able to access BMC management endpoints. '), + cfg.StrOpt( + 'kubernetes_container_template', + default=os.path.join( + '$pybasedir', + 'console/container/ironic-console-pod.yaml.template'), + mutable=True, + help='For the kubernetes provider, path to the template for defining ' + 'the console resources. The default template creates one Secret ' + 'to store the app info, and one Pod to run a console ' + 'container. A custom template must include namespace metadata, ' + 'and must define labels which can be used as a delete-all ' + 'selector.'), + cfg.IntOpt('kubernetes_pod_timeout', + default=120, + help='For the kubernetes provider, the time (in seconds) to ' + 'wait for the console pod to start.'), cfg.StrOpt( 'ssl_cert_file', help="Certificate file to use when starting the server securely."), diff --git a/ironic/console/container/ironic-console-pod.yaml.template b/ironic/console/container/ironic-console-pod.yaml.template new file mode 100644 index 0000000000..05770bcf69 --- /dev/null +++ b/ironic/console/container/ironic-console-pod.yaml.template @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "ironic-console-{{ uuid }}" + namespace: openstack + labels: + app: ironic + component: ironic-console + conductor: "{{ conductor }}" +stringData: + app-info: '{{ app_info }}' +--- +apiVersion: v1 +kind: Pod +metadata: + name: "ironic-console-{{ uuid }}" + namespace: openstack + labels: + app: ironic + component: ironic-console + conductor: "{{ conductor }}" +spec: + containers: + - name: x11vnc + image: "{{ image }}" + imagePullPolicy: Always + ports: + - containerPort: 5900 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 1024Mi + env: + - name: APP + value: "{{ app }}" + - name: READ_ONLY + value: "{{ read_only }}" + - name: APP_INFO + valueFrom: + secretKeyRef: + name: "ironic-console-{{ uuid }}" + key: app-info \ No newline at end of file diff --git a/ironic/console/container/kubernetes.py b/ironic/console/container/kubernetes.py new file mode 100644 index 0000000000..35dfab2611 --- /dev/null +++ b/ironic/console/container/kubernetes.py @@ -0,0 +1,307 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Kubernetes pod console container provider. +""" +import json +import re +import time +import yaml + +from oslo_concurrency import processutils +from oslo_log import log as logging + +from ironic.common import exception +from ironic.common import utils +from ironic.conf import CONF +from ironic.console.container import base + +LOG = logging.getLogger(__name__) + +# How often to check pod status +POD_READY_POLL_INTERVAL = 2 + + +class KubernetesConsoleContainer(base.BaseConsoleContainer): + """Console container provider which uses kubernetes pods.""" + + def __init__(self): + # confirm kubectl is available + try: + utils.execute("kubectl", "version") + except processutils.ProcessExecutionError as e: + LOG.exception( + "kubectl not available, " "this provider cannot be used." + ) + raise exception.ConsoleContainerError( + provider="kubernetes", reason=e + ) + if not CONF.vnc.console_image: + raise exception.ConsoleContainerError( + provider="kubernetes", + reason="[vnc]console_image must be set.", + ) + try: + self._render_template() + except Exception as e: + raise exception.ConsoleContainerError( + provider="kubernetes", + reason=f"Parsing {CONF.vnc.kubernetes_container_template} " + f"failed: {e}", + ) + + def _render_template(self, uuid="", app_name=None, app_info=None): + """Render the Kubernetes manifest template. + + :param uuid: Unique identifier for the node. + :param app_name: Name of the application to run in the container. + :param app_info: Dictionary of application-specific information. + :returns: A string containing the rendered Kubernetes YAML manifest. + """ + + # TODO(stevebaker) Support bind-mounting certificate files to + # handle verified BMC certificates + + if not uuid: + uuid = "" + if not app_name: + app_name = "fake" + if not app_info: + app_info = {} + + params = { + "uuid": uuid, + "image": CONF.vnc.console_image, + "app": app_name, + "app_info": json.dumps(app_info).strip(), + "read_only": CONF.vnc.read_only, + "conductor": CONF.host, + } + return utils.render_template( + CONF.vnc.kubernetes_container_template, params=params + ) + + def _apply(self, manifest): + try: + utils.execute( + "kubectl", "apply", "-f", "-", process_input=manifest + ) + except processutils.ProcessExecutionError as e: + LOG.exception("Problem calling kubectl apply") + raise exception.ConsoleContainerError( + provider="kubernetes", reason=e + ) + + def _delete( + self, resource_type, namespace, resource_name=None, selector=None + ): + args = [ + "kubectl", + "delete", + "-n", + namespace, + resource_type, + "--ignore-not-found=true", + ] + if resource_name: + args.append(resource_name) + elif selector: + args.append("-l") + args.append(selector) + else: + raise exception.ConsoleContainerError( + provider="kubernetes", + reason="Delete must be called with either a resource name " + "or selector.", + ) + try: + utils.execute(*args) + except processutils.ProcessExecutionError as e: + LOG.exception("Problem calling kubectl delete") + raise exception.ConsoleContainerError( + provider="kubernetes", reason=e + ) + + def _get_pod_node_ip(self, pod_name, namespace): + try: + out, _ = utils.execute( + "kubectl", + "get", + "pod", + pod_name, + "-n", + namespace, + "-o", + "jsonpath={.status.podIP}", + ) + return out.strip() + except processutils.ProcessExecutionError as e: + LOG.exception("Problem getting pod host IP for %s", pod_name) + raise exception.ConsoleContainerError( + provider="kubernetes", reason=e + ) + + def _wait_for_pod_ready(self, pod_name, namespace): + end_time = time.time() + CONF.vnc.kubernetes_pod_timeout + while time.time() < end_time: + try: + out, _ = utils.execute( + "kubectl", + "get", + "pod", + pod_name, + "-n", + namespace, + "-o", + "json", + ) + pod_status = json.loads(out) + if ( + "status" in pod_status + and "conditions" in pod_status["status"] + ): + for condition in pod_status["status"]["conditions"]: + if ( + condition["type"] == "Ready" + and condition["status"] == "True" + ): + LOG.debug("Pod %s is ready.", pod_name) + return + except ( + processutils.ProcessExecutionError, + json.JSONDecodeError, + ) as e: + LOG.warning( + "Could not get pod status for %s: %s", pod_name, e + ) + + time.sleep(POD_READY_POLL_INTERVAL) + + msg = ( + f"Pod {pod_name} did not become ready in " + f"{CONF.vnc.kubernetes_pod_timeout}s" + ) + + raise exception.ConsoleContainerError( + provider="kubernetes", reason=msg + ) + + def _get_resources_from_yaml(self, rendered, kind=None): + """Extracts Kubernetes resources from a YAML manifest. + + This method parses a multi-document YAML string and yields each + Kubernetes resource (dictionary) found. If `kind` is specified, + only resources of that specific kind are yielded. + + :param rendered: A string containing the rendered Kubernetes YAML + manifest. + :param kind: Optional string, the 'kind' of Kubernetes resource to + filter by (e.g., 'Pod', 'Service'). If None, all + resources are yielded. + :returns: A generator yielding Kubernetes resource dictionaries. + """ + # Split the YAML into individual documents + documents = re.split(r"^---\s*$", rendered, flags=re.MULTILINE) + for doc in documents: + if not doc.strip(): + continue + data = yaml.safe_load(doc) + if not data: + continue + if not kind or data.get("kind") == kind: + yield data + + def start_container(self, task, app_name, app_info): + """Start a console container for a node. + + Any existing running container for this node will be stopped. + + :param task: A TaskManager instance. + :raises: ConsoleContainerError + """ + node = task.node + uuid = node.uuid + + LOG.debug("Starting console container for node %s", uuid) + + rendered = self._render_template(uuid, app_name, app_info) + self._apply(rendered) + + pod = list(self._get_resources_from_yaml(rendered, kind="Pod"))[0] + pod_name = pod["metadata"]["name"] + namespace = pod["metadata"]["namespace"] + + try: + self._wait_for_pod_ready(pod_name, namespace) + host_ip = self._get_pod_node_ip(pod_name, namespace) + except Exception as e: + LOG.error( + "Failed to start container for node %s, cleaning up.", uuid + ) + try: + self._stop_container(uuid) + except Exception: + LOG.exception( + "Could not clean up resources for node %s", uuid + ) + raise e + + return host_ip, 5900 + + def _stop_container(self, uuid): + rendered = self._render_template(uuid) + resources = list(self._get_resources_from_yaml(rendered)) + resources.reverse() + for resource in resources: + kind = resource["kind"] + name = resource["metadata"]["name"] + namespace = resource["metadata"]["namespace"] + self._delete(kind, namespace, resource_name=name) + + def stop_container(self, task): + """Stop a console container for a node. + + Any existing running container for this node will be stopped. + + :param task: A TaskManager instance. + :raises: ConsoleContainerError + """ + node = task.node + uuid = node.uuid + self._stop_container(uuid) + + def _labels_to_selector(self, labels): + selector = [] + for key, value in labels.items(): + selector.append(f"{key}={value}") + return ",".join(selector) + + def stop_all_containers(self): + """Stops all running console containers + + This is run on conductor startup and graceful shutdown to ensure + no console containers are running. + :raises: ConsoleContainerError + """ + LOG.debug("Stopping all console containers") + rendered = self._render_template() + resources = list(self._get_resources_from_yaml(rendered)) + resources.reverse() + + for resource in resources: + kind = resource["kind"] + namespace = resource["metadata"]["namespace"] + labels = resource["metadata"]["labels"] + selector = self._labels_to_selector(labels) + self._delete(kind, namespace, selector=selector) diff --git a/ironic/tests/unit/console/container/test_console_container.py b/ironic/tests/unit/console/container/test_console_container.py index 4fa894eb03..a16dd2c562 100644 --- a/ironic/tests/unit/console/container/test_console_container.py +++ b/ironic/tests/unit/console/container/test_console_container.py @@ -11,10 +11,13 @@ # License for the specific language governing permissions and limitations # under the License. +import json import os import socket import tempfile +import time from unittest import mock +import yaml from oslo_concurrency import processutils from oslo_config import cfg @@ -23,6 +26,7 @@ from ironic.common import console_factory from ironic.common import exception from ironic.common import utils from ironic.console.container import fake +from ironic.console.container import kubernetes from ironic.console.container import systemd from ironic.tests import base @@ -152,8 +156,8 @@ class TestSystemdConsoleContainer(base.TestCase): @mock.patch.object(utils, 'execute', autospec=True) def test__host_port(self, mock_exec): - mock_exec.return_value = ('5900/tcp -> 192.0.2.1:33819', None) - container = self.provider._container_name('1234') + mock_exec.return_value = ("5900/tcp -> 192.0.2.1:33819", None) + container = self.provider._container_name("1234") self.assertEqual( ('192.0.2.1', 33819), self.provider._host_port(container) @@ -467,3 +471,759 @@ WantedBy=default.target""", f.read()) mock_exec.reset_mock() self.provider.stop_all_containers() mock_exec.assert_not_called() + + +class TestKubernetesConsoleContainer(base.TestCase): + + def setUp(self): + super(TestKubernetesConsoleContainer, self).setUp() + _reset_provider("kubernetes") + self.addCleanup(_reset_provider, "fake") + + CONF.set_override("console_image", "test-image", "vnc") + + # The __init__ of the provider calls _render_template, so we need to + # mock it here. + with mock.patch.object(utils, "render_template", autospec=True): + with mock.patch.object( + utils, "execute", autospec=True + ) as mock_exec: + self.provider = ( + console_factory.ConsoleContainerFactory().provider + ) + mock_exec.assert_has_calls( + [ + mock.call("kubectl", "version"), + ] + ) + + def test__render_template(self): + CONF.set_override("read_only", True, group="vnc") + + uuid = "1234" + app_name = "fake-app" + app_info = {"foo": "bar"} + + rendered = self.provider._render_template( + uuid=uuid, app_name=app_name, app_info=app_info + ) + + self.assertEqual( + """apiVersion: v1 +kind: Secret +metadata: + name: "ironic-console-1234" + namespace: openstack + labels: + app: ironic + component: ironic-console + conductor: "fake-mini" +stringData: + app-info: '{"foo": "bar"}' +--- +apiVersion: v1 +kind: Pod +metadata: + name: "ironic-console-1234" + namespace: openstack + labels: + app: ironic + component: ironic-console + conductor: "fake-mini" +spec: + containers: + - name: x11vnc + image: "test-image" + imagePullPolicy: Always + ports: + - containerPort: 5900 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 1024Mi + env: + - name: APP + value: "fake-app" + - name: READ_ONLY + value: "True" + - name: APP_INFO + valueFrom: + secretKeyRef: + name: "ironic-console-1234" + key: app-info""", + rendered, + ) + + @mock.patch.object(utils, "execute", autospec=True) + def test__apply(self, mock_exec): + manifest = "fake-manifest" + self.provider._apply(manifest) + + mock_exec.assert_called_once_with( + "kubectl", "apply", "-f", "-", process_input=manifest + ) + + @mock.patch.object(utils, "execute", autospec=True) + def test__apply_failure(self, mock_exec): + manifest = "fake-manifest" + mock_exec.side_effect = processutils.ProcessExecutionError( + stderr="ouch" + ) + + self.assertRaisesRegex( + exception.ConsoleContainerError, + "ouch", + self.provider._apply, + manifest, + ) + + @mock.patch.object(utils, "execute", autospec=True) + def test__delete_by_name(self, mock_exec): + self.provider._delete( + "pod", "test-namespace", resource_name="test-pod" + ) + mock_exec.assert_called_once_with( + "kubectl", + "delete", + "-n", + "test-namespace", + "pod", + "--ignore-not-found=true", + "test-pod", + ) + + @mock.patch.object(utils, "execute", autospec=True) + def test__delete_by_selector(self, mock_exec): + self.provider._delete("pod", "test-namespace", selector="app=ironic") + mock_exec.assert_called_once_with( + "kubectl", + "delete", + "-n", + "test-namespace", + "pod", + "--ignore-not-found=true", + "-l", + "app=ironic", + ) + + def test__delete_no_name_or_selector(self): + self.assertRaisesRegex( + exception.ConsoleContainerError, + "Delete must be called with either a resource name or selector", + self.provider._delete, + "pod", + "test-namespace", + ) + + @mock.patch.object(utils, "execute", autospec=True) + def test__delete_failure(self, mock_exec): + mock_exec.side_effect = processutils.ProcessExecutionError( + stderr="ouch" + ) + self.assertRaisesRegex( + exception.ConsoleContainerError, + "ouch", + self.provider._delete, + "pod", + "test-namespace", + resource_name="test-pod", + ) + + @mock.patch.object(utils, "execute", autospec=True) + def test__get_pod_node_ip(self, mock_exec): + mock_exec.return_value = ("192.168.1.100", "") + ip = self.provider._get_pod_node_ip("test-pod", "test-namespace") + self.assertEqual("192.168.1.100", ip) + mock_exec.assert_called_once_with( + "kubectl", + "get", + "pod", + "test-pod", + "-n", + "test-namespace", + "-o", + "jsonpath={.status.podIP}", + ) + + @mock.patch.object(utils, "execute", autospec=True) + def test__get_pod_node_ip_failure(self, mock_exec): + mock_exec.side_effect = processutils.ProcessExecutionError( + stderr="ouch" + ) + self.assertRaisesRegex( + exception.ConsoleContainerError, + "ouch", + self.provider._get_pod_node_ip, + "test-pod", + "test-namespace", + ) + + @mock.patch.object(utils, "execute", autospec=True) + @mock.patch.object(time, "sleep", autospec=True) + def test__wait_for_pod_ready(self, mock_sleep, mock_exec): + pod_ready_status = { + "status": {"conditions": [{"type": "Ready", "status": "True"}]} + } + mock_exec.return_value = (json.dumps(pod_ready_status), "") + + self.provider._wait_for_pod_ready("test-pod", "test-namespace") + + mock_exec.assert_called_once_with( + "kubectl", + "get", + "pod", + "test-pod", + "-n", + "test-namespace", + "-o", + "json", + ) + mock_sleep.assert_not_called() + + @mock.patch.object(utils, "execute", autospec=True) + @mock.patch.object(time, "sleep", autospec=True) + @mock.patch.object(time, "time", autospec=True, side_effect=[1, 2, 3, 4]) + def test__wait_for_pod_ready_polling( + self, mock_time, mock_sleep, mock_exec + ): + pod_not_ready_status = { + "status": {"conditions": [{"type": "Ready", "status": "False"}]} + } + pod_ready_status = { + "status": {"conditions": [{"type": "Ready", "status": "True"}]} + } + mock_exec.side_effect = [ + (json.dumps(pod_not_ready_status), ""), + (json.dumps(pod_ready_status), ""), + ] + + self.provider._wait_for_pod_ready("test-pod", "test-namespace") + + self.assertEqual(2, mock_exec.call_count) + mock_sleep.assert_called_once_with(kubernetes.POD_READY_POLL_INTERVAL) + + @mock.patch.object(time, "time", autospec=True, side_effect=[0, 121]) + @mock.patch.object(utils, "execute", autospec=True) + def test__wait_for_pod_ready_timeout(self, mock_exec, mock_time): + pod_not_ready_status = { + "status": {"conditions": [{"type": "Ready", "status": "False"}]} + } + mock_exec.return_value = (json.dumps(pod_not_ready_status), "") + + self.assertRaisesRegex( + exception.ConsoleContainerError, + "did not become ready", + self.provider._wait_for_pod_ready, + "test-pod", + "test-namespace", + ) + + @mock.patch.object(time, "time", autospec=True, side_effect=[0, 121]) + @mock.patch.object(utils, "execute", autospec=True) + def test__wait_for_pod_ready_exec_error(self, mock_exec, mock_time): + mock_exec.side_effect = processutils.ProcessExecutionError() + self.assertRaisesRegex( + exception.ConsoleContainerError, + "did not become ready", + self.provider._wait_for_pod_ready, + "test-pod", + "test-namespace", + ) + + def test__get_resources_from_yaml_single_doc_no_kind(self): + rendered_yaml = """ +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - name: my-container + image: nginx +""" + resources = list( + self.provider._get_resources_from_yaml(rendered_yaml) + ) + self.assertEqual( + [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "my-pod"}, + "spec": { + "containers": [ + {"image": "nginx", "name": "my-container"} + ] + }, + } + ], + resources, + ) + + def test__get_resources_from_yaml_multi_doc_no_kind(self): + rendered_yaml = """ +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +--- +apiVersion: v1 +kind: Service +metadata: + name: my-service +""" + resources = list( + self.provider._get_resources_from_yaml(rendered_yaml) + ) + self.assertEqual( + [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "my-pod"}, + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": "my-service"}, + }, + ], + resources, + ) + + def test__get_resources_from_yaml_single_doc_with_kind_match(self): + rendered_yaml = """ +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +""" + resources = list( + self.provider._get_resources_from_yaml(rendered_yaml, kind="Pod") + ) + self.assertEqual( + [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "my-pod"}, + } + ], + resources, + ) + + def test__get_resources_from_yaml_single_doc_with_kind_no_match(self): + rendered_yaml = """ +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +""" + resources = list( + self.provider._get_resources_from_yaml( + rendered_yaml, kind="Service" + ) + ) + self.assertEqual(0, len(resources)) + + def test__get_resources_from_yaml_multi_doc_with_kind_match_some(self): + rendered_yaml = """ +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +--- +apiVersion: v1 +kind: Service +metadata: + name: my-service +--- +apiVersion: v1 +kind: Pod +metadata: + name: another-pod +""" + resources = list( + self.provider._get_resources_from_yaml(rendered_yaml, kind="Pod") + ) + self.assertEqual( + [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "my-pod"}, + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "another-pod"}, + }, + ], + resources, + ) + + def test__get_resources_from_yaml_multi_doc_with_kind_no_match_all(self): + rendered_yaml = """ +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +--- +apiVersion: v1 +kind: Service +metadata: + name: my-service +""" + resources = list( + self.provider._get_resources_from_yaml( + rendered_yaml, kind="Deployment" + ) + ) + self.assertEqual(0, len(resources)) + + def test__get_resources_from_yaml_empty_documents(self): + rendered_yaml = """ +--- +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +--- + +--- +apiVersion: v1 +kind: Service +metadata: + name: my-service +--- +""" + resources = list( + self.provider._get_resources_from_yaml(rendered_yaml) + ) + self.assertEqual( + [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "my-pod"}, + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": "my-service"}, + }, + ], + resources, + ) + + def test__get_resources_from_yaml_invalid_yaml(self): + rendered_yaml = """ +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +--- + - bad: indent + - invalid: yaml + +""" + try: + list(self.provider._get_resources_from_yaml(rendered_yaml)) + raise Exception("Expected YAMLError") + except yaml.YAMLError: + pass + + def test__get_resources_from_yaml_document_safe_load_none(self): + # This can happen if a document is just whitespace or comments + rendered_yaml = """ +# This is a comment +--- +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +""" + resources = list( + self.provider._get_resources_from_yaml(rendered_yaml) + ) + self.assertEqual( + [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "my-pod"}, + } + ], + resources, + ) + + def test__get_resources_from_yaml_empty_string(self): + rendered_yaml = "" + resources = list( + self.provider._get_resources_from_yaml(rendered_yaml) + ) + self.assertEqual(0, len(resources)) + + def test__get_resources_from_yaml_whitespace_string(self): + rendered_yaml = " \n\n" + resources = list( + self.provider._get_resources_from_yaml(rendered_yaml) + ) + self.assertEqual(0, len(resources)) + + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_get_pod_node_ip", + autospec=True, + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_wait_for_pod_ready", + autospec=True, + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_get_resources_from_yaml", + autospec=True, + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, "_apply", autospec=True + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_render_template", + autospec=True, + ) + def test_start_container( + self, + mock_render, + mock_apply, + mock_get_resources, + mock_wait, + mock_get_ip, + ): + task = mock.Mock(node=mock.Mock(uuid="1234")) + app_name = "test-app" + app_info = {"foo": "bar"} + + mock_render.return_value = "fake-manifest" + mock_get_resources.return_value = [ + { + "kind": "Pod", + "metadata": { + "name": "test-pod", + "namespace": "test-namespace", + }, + } + ] + mock_get_ip.return_value = "192.168.1.100" + + host, port = self.provider.start_container(task, app_name, app_info) + + self.assertEqual(("192.168.1.100", 5900), (host, port)) + mock_render.assert_called_once_with( + self.provider, "1234", app_name, app_info + ) + mock_apply.assert_called_once_with(self.provider, "fake-manifest") + mock_get_resources.assert_called_once_with( + self.provider, "fake-manifest", kind="Pod" + ) + mock_wait.assert_called_once_with( + self.provider, "test-pod", "test-namespace" + ) + mock_get_ip.assert_called_once_with( + self.provider, "test-pod", "test-namespace" + ) + + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_stop_container", + autospec=True, + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_get_pod_node_ip", + autospec=True, + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_wait_for_pod_ready", + autospec=True, + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_get_resources_from_yaml", + autospec=True, + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, "_apply", autospec=True + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_render_template", + autospec=True, + ) + def test_start_container_failure( + self, + mock_render, + mock_apply, + mock_get_resources, + mock_wait, + mock_get_ip, + mock_stop, + ): + task = mock.Mock(node=mock.Mock(uuid="1234")) + mock_render.return_value = "fake-manifest" + mock_get_resources.return_value = [ + {"metadata": {"name": "test-pod", "namespace": "test-ns"}} + ] + mock_wait.side_effect = exception.ConsoleContainerError(reason="boom") + + self.assertRaises( + exception.ConsoleContainerError, + self.provider.start_container, + task, + "app", + {}, + ) + mock_stop.assert_called_once_with(self.provider, "1234") + mock_get_ip.assert_not_called() + + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_stop_container", + autospec=True, + ) + def test_stop_container(self, mock_stop_container): + task = mock.Mock(node=mock.Mock(uuid="1234")) + self.provider.stop_container(task) + mock_stop_container.assert_called_once_with(self.provider, "1234") + + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, "_delete", autospec=True + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_get_resources_from_yaml", + autospec=True, + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_render_template", + autospec=True, + ) + def test__stop_container( + self, mock_render, mock_get_resources, mock_delete + ): + uuid = "1234" + mock_render.return_value = "fake-manifest" + mock_get_resources.return_value = [ + { + "kind": "Secret", + "metadata": { + "name": "ironic-console-1234", + "namespace": "test-namespace", + }, + }, + { + "kind": "Pod", + "metadata": { + "name": "ironic-console-1234", + "namespace": "test-namespace", + }, + }, + ] + + self.provider._stop_container(uuid) + + mock_render.assert_called_once_with(self.provider, uuid) + mock_get_resources.assert_called_once_with( + self.provider, "fake-manifest" + ) + mock_delete.assert_has_calls( + [ + mock.call( + self.provider, + "Pod", + "test-namespace", + resource_name="ironic-console-1234", + ), + mock.call( + self.provider, + "Secret", + "test-namespace", + resource_name="ironic-console-1234", + ), + ] + ) + self.assertEqual(2, mock_delete.call_count) + + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, "_delete", autospec=True + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_labels_to_selector", + autospec=True, + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_get_resources_from_yaml", + autospec=True, + ) + @mock.patch.object( + kubernetes.KubernetesConsoleContainer, + "_render_template", + autospec=True, + ) + def test_stop_all_containers( + self, + mock_render, + mock_get_resources, + mock_labels_to_selector, + mock_delete, + ): + mock_render.return_value = "fake-manifest" + mock_get_resources.return_value = [ + { + "kind": "Secret", + "metadata": { + "namespace": "test-ns", + "labels": {"app": "ironic"}, + }, + }, + { + "kind": "Pod", + "metadata": { + "namespace": "test-ns", + "labels": {"app": "ironic"}, + }, + }, + ] + mock_labels_to_selector.return_value = "app=ironic" + + self.provider.stop_all_containers() + + mock_render.assert_called_once_with(self.provider) + mock_get_resources.assert_called_once_with( + self.provider, "fake-manifest" + ) + mock_labels_to_selector.assert_has_calls( + [ + mock.call(self.provider, {"app": "ironic"}), + mock.call(self.provider, {"app": "ironic"}), + ] + ) + mock_delete.assert_has_calls( + [ + mock.call( + self.provider, "Pod", "test-ns", selector="app=ironic" + ), + mock.call( + self.provider, "Secret", "test-ns", selector="app=ironic" + ), + ] + ) diff --git a/pyproject.toml b/pyproject.toml index 1a9c12c474..21ae24a24a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ Bugs = "https://bugs.launchpad.net/ironic" [project.entry-points."ironic.console.container"] systemd = "ironic.console.container.systemd:SystemdConsoleContainer" +kubernetes = "ironic.console.container.kubernetes:KubernetesConsoleContainer" fake = "ironic.console.container.fake:FakeConsoleContainer" # TODO(stephenfin): Remove the oslo.db call that requires this. It's unnecessary diff --git a/releasenotes/notes/console-k8s-b4aee1bb1d3d0a65.yaml b/releasenotes/notes/console-k8s-b4aee1bb1d3d0a65.yaml new file mode 100644 index 0000000000..57a14bf2b6 --- /dev/null +++ b/releasenotes/notes/console-k8s-b4aee1bb1d3d0a65.yaml @@ -0,0 +1,18 @@ +--- +features: + - | + A new ``ironic.console.container`` provider is added called ``kubernetes`` + which allows Ironic conductor to manage console containers as Kubernetes + pods. The kubernetes resources are defined in the template file + configured by ``[vnc]kubernetes_container_template`` and the default + template creates one secret to store the app info, and one pod to run + the console container. + + It is expected that Ironic conductor is deployed inside the kubernetes + cluster. The associated service account will need roles and bindings which + allow it to manage the required resources (with the default template this + will be secrets and pods). + + This provider holds the assumption that ironic-novnc will be deployed + in the same kubernetes cluster, and so can connect to the VNC servers + via the pod's ``status.hostIP``. \ No newline at end of file