diff --git a/kolla/common/config.py b/kolla/common/config.py index 7e06fd7f87..f6a2e62556 100644 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -162,7 +162,8 @@ _CLI_OPTS = [ cfg.BoolOpt('skip-parents', default=False, help='Do not rebuild parents of matched images'), cfg.BoolOpt('skip-existing', default=False, - help='Do not rebuild images present in the docker cache'), + help='Do not rebuild images present in the container engine ' + 'cache'), cfg.DictOpt('build-args', help='Set docker build time variables'), cfg.BoolOpt('keep', default=False, @@ -176,7 +177,7 @@ _CLI_OPTS = [ cfg.StrOpt('network_mode', default='host', help='The network mode for Docker build. Example: host'), cfg.BoolOpt('cache', default=True, - help='Use the Docker cache when building'), + help='Use the container engine cache when building'), cfg.MultiOpt('profile', types.String(), short='p', help=('Build a pre-defined set of images, see [profiles]' ' section in config. The default profiles are:' @@ -194,14 +195,14 @@ _CLI_OPTS = [ help=('Build only images matching regex and its' ' dependencies')), cfg.StrOpt('registry', - help=('The docker registry host. The default registry host' - ' is Docker Hub')), + help=('The container image registry host. The default registry' + ' host is Docker Hub')), cfg.StrOpt('save-dependency', help=('Path to the file to store the docker image' ' dependency in Graphviz dot format')), cfg.StrOpt('format', short='f', default='json', choices=['json', 'none'], - help='Format to write the final results in'), + help='Format to write the final results in.'), cfg.StrOpt('tarballs-base', default=TARBALLS_BASE, help='Base url to OpenStack tarballs'), # NOTE(hrw): deprecate argument in Zed, remove in A-cycle @@ -214,7 +215,7 @@ _CLI_OPTS = [ ' (Note: setting to one will allow real time' ' logging)')), cfg.StrOpt('tag', default=version.cached_version_string(), - help='The Docker tag'), + help='The container image tag'), cfg.BoolOpt('template-only', default=False, help="Don't build images. Generate Dockerfile only"), cfg.IntOpt('timeout', default=120, @@ -255,6 +256,8 @@ _CLI_OPTS = [ help='Prefix prepended to image names'), cfg.StrOpt('repos-yaml', default='', help='Path to alternative repos.yaml file'), + cfg.StrOpt('engine', default='docker', choices=['docker'], + help='Container engine to build images on.') ] _BASE_OPTS = [ diff --git a/kolla/engine_adapter/__init__.py b/kolla/engine_adapter/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kolla/engine_adapter/engine.py b/kolla/engine_adapter/engine.py new file mode 100644 index 0000000000..184bb3856c --- /dev/null +++ b/kolla/engine_adapter/engine.py @@ -0,0 +1,57 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from distutils.version import StrictVersion +from enum import Enum +from kolla.image.utils import LOG + +try: + import docker +except (ImportError): + LOG.debug("Docker python library was not found") + + +class Engine(Enum): + + DOCKER = "docker" + + +class UnsupportedEngineError(ValueError): + + def __init__(self, engine_name): + super().__init__() + self.engine_name = engine_name + + def __str__(self): + return f'Unsupported engine name given: "{self.engine_name}"' + + +def getEngineException(conf): + if conf.engine == Engine.DOCKER.value: + return (docker.errors.DockerException) + else: + raise UnsupportedEngineError(conf.engine) + + +def getEngineClient(conf): + if conf.engine == Engine.DOCKER.value: + kwargs_env = docker.utils.kwargs_from_env() + return docker.APIClient(version='auto', **kwargs_env) + else: + raise UnsupportedEngineError(conf.engine) + + +def getEngineVersion(conf): + if conf.engine == Engine.DOCKER.value: + return StrictVersion(docker.version) + else: + raise UnsupportedEngineError(conf.engine) diff --git a/kolla/image/build.py b/kolla/image/build.py index 6377ebe9d2..196fab4e08 100644 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -21,6 +21,7 @@ import time from kolla.common import config as common_config from kolla.common import utils +from kolla.engine_adapter import engine from kolla.image.kolla_worker import KollaWorker from kolla.image.utils import LOG from kolla.image.utils import Status @@ -104,10 +105,28 @@ def run_build(): if conf.debug: LOG.setLevel(logging.DEBUG) - if conf.squash: - squash_version = utils.get_docker_squash_version() - LOG.info('Image squash is enabled and "docker-squash" version is %s', - squash_version) + if conf.engine not in (engine.Engine.DOCKER.value,): + LOG.error(f'Unsupported engine name "{conf.engine}", exiting.') + sys.exit(1) + LOG.info(f'Using engine: {conf.engine}') + + if conf.engine == engine.Engine.DOCKER.value: + try: + import docker + docker.version + except ImportError: + LOG.error("Error, you have set Docker as container engine, " + "but the Python library is not found." + "Try running 'pip install docker'") + sys.exit(1) + except AttributeError: + LOG.error("Error, Docker Python library is too old, " + "Try running 'pip install docker --upgrade'") + + if conf.squash: + squash_version = utils.get_docker_squash_version() + LOG.info('Image squash is enabled and "docker-squash" version ' + 'is %s', squash_version) kolla = KollaWorker(conf) kolla.setup_working_dir() @@ -133,7 +152,7 @@ def run_build(): if conf.save_dependency: kolla.save_dependency(conf.save_dependency) - LOG.info('Docker images dependency are saved in %s', + LOG.info('Container images dependency are saved in %s', conf.save_dependency) return if conf.list_images: diff --git a/kolla/image/kolla_worker.py b/kolla/image/kolla_worker.py index b1e7ac9437..1a665ddd44 100644 --- a/kolla/image/kolla_worker.py +++ b/kolla/image/kolla_worker.py @@ -11,7 +11,6 @@ # limitations under the License. import datetime -import docker import json import os import queue @@ -24,6 +23,7 @@ import time import jinja2 from kolla.common import config as common_config from kolla.common import utils +from kolla.engine_adapter import engine from kolla import exception from kolla.image.tasks import BuildTask from kolla.image.unbuildable import UNBUILDABLE_IMAGES @@ -42,7 +42,7 @@ PROJECT_ROOT = os.path.abspath(os.path.join( class Image(object): def __init__(self, name, canonical_name, path, parent_name='', status=Status.UNPROCESSED, parent=None, - source=None, logger=None, docker_client=None): + source=None, logger=None, engine_client=None): self.name = name self.canonical_name = canonical_name self.path = path @@ -56,7 +56,7 @@ class Image(object): self.children = [] self.plugins = [] self.additions = [] - self.dc = docker_client + self.engine_client = engine_client def copy(self): c = Image(self.name, self.canonical_name, self.path, @@ -72,8 +72,9 @@ class Image(object): c.additions = list(self.additions) return c - def in_docker_cache(self): - return len(self.dc.images(name=self.canonical_name, quiet=True)) == 1 + def in_engine_cache(self): + return len(self.engine_client.images(name=self.canonical_name, + quiet=True)) == 1 def __repr__(self): return ("Image(%s, %s, %s, parent_name=%s," @@ -148,16 +149,16 @@ class KollaWorker(object): self.maintainer = conf.maintainer self.distro_python_version = conf.distro_python_version - docker_kwargs = docker.utils.kwargs_from_env() try: - self.dc = docker.APIClient(version='auto', **docker_kwargs) - except docker.errors.DockerException as e: - self.dc = None + self.engine_client = engine.getEngineClient(self.conf) + except engine.getEngineException(self.conf) as e: + self.engine_client = None if not (conf.template_only or conf.save_dependency or conf.list_images or conf.list_dependencies): - LOG.error("Unable to connect to Docker, exiting") + LOG.error("Unable to connect to container engine daemon, " + "exiting") LOG.info("Exception caught: {0}".format(e)) sys.exit(1) @@ -179,18 +180,18 @@ class KollaWorker(object): # this is the correct path # TODO(SamYaple): Improve this to make this safer if os.path.exists(os.path.join(image_path, 'base')): - LOG.info('Found the docker image folder at %s', image_path) + LOG.info('Found the container image folder at %s', image_path) return image_path else: raise exception.KollaDirNotFoundException('Image dir can not ' 'be found') def build_rpm_setup(self, rpm_setup_config): - """Generates a list of docker commands based on provided configuration. + """Generates a list of engine commands based on provided configuration :param rpm_setup_config: A list of .rpm or .repo paths or URLs (can be empty) - :return: A list of docker commands + :return: A list of engine commands """ rpm_setup = list() @@ -470,7 +471,7 @@ class KollaWorker(object): if image.status != Status.MATCHED: continue # Skip image if --skip-existing was given and image exists. - if (self.conf.skip_existing and image.in_docker_cache()): + if (self.conf.skip_existing and image.in_engine_cache()): LOG.debug('Skipping existing image %s', image.name) image.status = Status.SKIPPED # Skip image if --skip-parents was given and image has children. @@ -638,7 +639,7 @@ class KollaWorker(object): image = Image(image_name, canonical_name, path, parent_name=parent_name, logger=utils.make_a_logger(self.conf, image_name), - docker_client=self.dc) + engine_client=self.engine_client) # NOTE(jeffrey4l): register the opts if the section didn't # register in the kolla/common/config.py file @@ -683,7 +684,7 @@ class KollaWorker(object): except ImportError: LOG.error('"graphviz" is required for save dependency') raise - dot = graphviz.Digraph(comment='Docker Images Dependency') + dot = graphviz.Digraph(comment='Container Images Dependency') dot.body.extend(['rankdir=LR']) for image in self.images: if image.status not in [Status.MATCHED]: diff --git a/kolla/image/tasks.py b/kolla/image/tasks.py index cbfecce095..d15ef8d176 100644 --- a/kolla/image/tasks.py +++ b/kolla/image/tasks.py @@ -11,7 +11,6 @@ # limitations under the License. import datetime -import docker import errno import os import shutil @@ -23,6 +22,7 @@ from requests import exceptions as requests_exc from kolla.common import task # noqa from kolla.common import utils # noqa +from kolla.engine_adapter import engine from kolla.image.utils import Status from kolla.image.utils import STATUS_ERRORS @@ -31,21 +31,18 @@ class ArchivingError(Exception): pass -class DockerTask(task.Task): - - docker_kwargs = docker.utils.kwargs_from_env() - - def __init__(self): - super(DockerTask, self).__init__() - self._dc = None +class EngineTask(task.Task): + def __init__(self, conf): + super(EngineTask, self).__init__() + self._ec = None + self.conf = conf @property - def dc(self): - if self._dc is not None: - return self._dc - docker_kwargs = self.docker_kwargs.copy() - self._dc = docker.APIClient(version='auto', **docker_kwargs) - return self._dc + def engine_client(self): + if self._ec is not None: + return self._ec + self._ec = engine.getEngineClient(self.conf) + return self._ec class PushIntoQueueTask(task.Task): @@ -70,11 +67,11 @@ class PushError(Exception): pass -class PushTask(DockerTask): - """Task that pushes an image to a docker repository.""" +class PushTask(EngineTask): + """Task that pushes an image to a container image repository.""" def __init__(self, conf, image): - super(PushTask, self).__init__() + super(PushTask, self).__init__(conf) self.conf = conf self.image = image self.logger = image.logger @@ -89,9 +86,10 @@ class PushTask(DockerTask): try: self.push_image(image) except requests_exc.ConnectionError: - self.logger.exception('Make sure Docker is running and that you' - ' have the correct privileges to run Docker' - ' (root)') + self.logger.exception('Make sure container engine daemon is ' + 'running and that you have the correct ' + 'privileges to run the ' + 'container engine (root)') image.status = Status.CONNECTION_ERROR except PushError as exception: self.logger.error(exception) @@ -110,7 +108,8 @@ class PushTask(DockerTask): def push_image(self, image): kwargs = dict(stream=True, decode=True) - for response in self.dc.push(image.canonical_name, **kwargs): + for response in self.engine_client.push( + image.canonical_name, **kwargs): if 'stream' in response: self.logger.info(response['stream']) elif 'errorDetail' in response: @@ -120,11 +119,11 @@ class PushTask(DockerTask): image.status = Status.BUILT -class BuildTask(DockerTask): +class BuildTask(EngineTask): """Task that builds out an image.""" def __init__(self, conf, image, push_queue): - super(BuildTask, self).__init__() + super(BuildTask, self).__init__(conf) self.conf = conf self.image = image self.push_queue = push_queue @@ -145,8 +144,9 @@ class BuildTask(DockerTask): followups = [] if self.conf.push and self.success: followups.extend([ - # If we are supposed to push the image into a docker - # repository, then make sure we do that... + # If we are supposed to push the image into a + # container image repository, + # then make sure we do that... PushIntoQueueTask( PushTask(self.conf, self.image), self.push_queue), @@ -348,15 +348,16 @@ class BuildTask(DockerTask): buildargs = self.update_buildargs() try: - for stream in self.dc.build(path=image.path, - tag=image.canonical_name, - nocache=not self.conf.cache, - rm=True, - decode=True, - network_mode=self.conf.network_mode, - pull=pull, - forcerm=self.forcerm, - buildargs=buildargs): + for stream in \ + self.engine_client.build(path=image.path, + tag=image.canonical_name, + nocache=not self.conf.cache, + rm=True, + decode=True, + network_mode=self.conf.network_mode, + pull=pull, + forcerm=self.forcerm, + buildargs=buildargs): if 'stream' in stream: for line in stream['stream'].split('\n'): if line: @@ -369,11 +370,13 @@ class BuildTask(DockerTask): self.logger.error('%s', line) return - if image.status != Status.ERROR and self.conf.squash: + if image.status != Status.ERROR and self.conf.squash and \ + self.conf.engine == engine.Engine.DOCKER.value: self.squash() - except docker.errors.DockerException: + except engine.getEngineException(self.conf): image.status = Status.ERROR - self.logger.exception('Unknown docker error when building') + self.logger.exception('Unknown container engine ' + 'error when building') except Exception: image.status = Status.ERROR self.logger.exception('Unknown error when building') @@ -385,9 +388,9 @@ class BuildTask(DockerTask): def squash(self): image_tag = self.image.canonical_name - image_id = self.dc.inspect_image(image_tag)['Id'] + image_id = self.engine_client.inspect_image(image_tag)['Id'] - parent_history = self.dc.history(self.image.parent_name) + parent_history = self.engine_client.history(self.image.parent_name) parent_last_layer = parent_history[0]['Id'] self.logger.info('Parent lastest layer is: %s' % parent_last_layer) diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py index c4c560c2fd..a4784ba86b 100644 --- a/kolla/tests/test_build.py +++ b/kolla/tests/test_build.py @@ -493,7 +493,7 @@ class KollaWorkerTest(base.TestCase): self.assertEqual(utils.Status.SKIPPED, kolla.images[2].parent.status) self.assertEqual(utils.Status.SKIPPED, kolla.images[1].parent.status) - @mock.patch.object(Image, 'in_docker_cache') + @mock.patch.object(Image, 'in_engine_cache') def test_skip_existing(self, mock_in_cache): mock_in_cache.side_effect = [True, False] self.conf.set_override('skip_existing', True)