Merge pull request #185 from lachmanfrantisek/better-tests

Run tests for all target types

Author: TomasTomecek

Dockerfile.tests

@@ -3,7 +3,7 @@ FROM registry.fedoraproject.org/fedora:29
 ENV PYTHONDONTWRITEBYTECODE=yes-please
 
 # atomic in F28 can't mount: can't find the image
-RUN dnf install -y make python3-pytest python3-pyxattr \
+RUN dnf install -y make python3-pytest python3-pyxattr python3-pytest-cov \
     skopeo \
     https://kojipkgs.fedoraproject.org/packages/podman/0.9.1/3.gitaba58d1.fc29/x86_64/podman-0.9.1-3.gitaba58d1.fc29.x86_64.rpm \
     https://kojipkgs.fedoraproject.org/packages/containernetworking-plugins/0.7.3/2.fc29/x86_64/containernetworking-plugins-0.7.3-2.fc29.x86_64.rpm \

Makefile

@@ -15,10 +15,10 @@ test-in-container-now:
 	docker run --rm --privileged --security-opt label=disable --cap-add SYS_ADMIN -ti -v $(CURDIR):/src $(TEST_IMAGE_NAME) make exec-test TEST_TARGET="$(TEST_TARGET)"
 
 test-in-ci: setup-ci
-	PYTHONPATH=$(CURDIR) py.test $(TEST_TARGET)
+	PYTHONPATH=$(CURDIR) py.test --cov=colin $(TEST_TARGET)
 
 exec-test:
-	PYTHONPATH=$(CURDIR) py.test-3 $(TEST_TARGET)
+	PYTHONPATH=$(CURDIR) py.test-3 --cov=colin $(TEST_TARGET)
 
 setup-ci:
 	./setup-ci.sh

colin/core/target.py

@@ -70,6 +70,7 @@ class Target(object):
 
     def __init__(self):
         self._labels = None
+        self.target_name = None
 
     @property
     def labels(self):
@@ -115,9 +116,11 @@ def get_instance(target_type, **kwargs):
 
 
 class DockerfileTarget(Target):
+    target_type = "dockerfile"
 
     def __init__(self, target, **_):
         super().__init__()
+        self.target_name = target
         logger.debug("Target is a dockerfile.")
         if isinstance(target, io.IOBase):
             logger.debug("Target is a dockerfile loaded from the file-like object.")
@@ -224,13 +227,15 @@ class ImageTarget(AbstractImageTarget):
     """
     Represents the podman image as a target.
     """
+    target_type = "image"
 
     def __init__(self, target, pull, insecure=False, **_):
         super().__init__()
         logger.debug("Target is an image.")
         self.pull = pull
         self.insecure = insecure
-        self.image_name = ImageName.parse(target)
+        self.image_name_obj = ImageName.parse(target)
+        self.target_name = self.image_name_obj.name
 
         self._config_metadata = None
         self._mount_point = None
@@ -242,7 +247,7 @@ def __init__(self, target, pull, insecure=False, **_):
     @property
     def config_metadata(self):
         if not self._config_metadata:
-            cmd = ["podman", "inspect", self.image_name.name]
+            cmd = ["podman", "inspect", self.target_name]
             loaded_config = json.loads(subprocess.check_output(cmd))
             if loaded_config and isinstance(loaded_config, list):
                 self._config_metadata = loaded_config[0]
@@ -260,15 +265,15 @@ def labels(self):
     def mount_point(self):
         """ podman mount -- real filesystem """
         if self._mount_point is None:
-            cmd_create = ["podman", "create", self.image_name.name, "some-cmd"]
+            cmd_create = ["podman", "create", self.target_name, "some-cmd"]
             self._mounted_container_id = subprocess.check_output(cmd_create).decode().rstrip()
             cmd_mount = ["podman", "mount", self._mounted_container_id]
             self._mount_point = subprocess.check_output(cmd_mount).decode().rstrip()
         return self._mount_point
 
     def _try_image(self):
         logger.debug("Trying to find an image.")
-        cmd = ["podman", "images", "--quiet", self.image_name.name]
+        cmd = ["podman", "images", "--quiet", self.target_name]
         result = subprocess.run(cmd,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
@@ -279,7 +284,7 @@ def _try_image(self):
             if "unable to find" in result.stderr.decode():
                 if self.pull:
                     logger.debug("Pulling an image.")
-                    cmd_pull = ["podman", "pull", "--quiet", self.image_name.name]
+                    cmd_pull = ["podman", "pull", "--quiet", self.target_name]
                     result_pull = subprocess.run(cmd_pull,
                                                  stdout=subprocess.PIPE,
                                                  stderr=subprocess.PIPE)
@@ -288,10 +293,10 @@ def _try_image(self):
                         logger.debug("Image pulled with id: '{}'.".format(self.image_id))
                     else:
                         raise ColinException(
-                            "Cannot pull an image: '{}'.".format(self.image_name.name))
+                            "Cannot pull an image: '{}'.".format(self.target_name))
 
                 else:
-                    raise ColinException("Image '{}' not found.".format(self.image_name.name))
+                    raise ColinException("Image '{}' not found.".format(self.target_name))
             else:
                 raise ColinException("Podman error: {}".format(result.stderr))
 
@@ -313,17 +318,18 @@ class OstreeTarget(AbstractImageTarget):
     """
     Represents the ostree repository as an image target.
     """
+    target_type = "ostree"
 
     def __init__(self, target, **_):
         super().__init__()
         logger.debug("Target is an ostree repository.")
 
-        self.image_name = target
-        if self.image_name.startswith("ostree:"):
-            self.image_name = self.image_name[7:]
+        self.target_name = target
+        if self.target_name.startswith("ostree:"):
+            self.target_name = self.target_name[7:]
 
         try:
-            self.ref_image_name, self._ostree_path = self.image_name.split("@", 1)
+            self.ref_image_name, self._ostree_path = self.target_name.split("@", 1)
         except ValueError:
             raise RuntimeError("Invalid ostree target: should be 'image@path'.")
 

setup-ci.sh

@@ -10,7 +10,7 @@ echo "yum -y install make gcc python36 python36-devel python36-setuptools python
 yum -y install make gcc python36 python36-devel python36-setuptools python36-pip python36-cffi python36-pycparser
 
 echo "pip3 install pytest xattr"
-pip3.6 install pytest xattr
+pip3.6 install pytest pytest-cov xattr
 
 echo "yum copr enable -y baude/Upstream_CRIO_Family"
 yum copr enable -y baude/Upstream_CRIO_Family

tests/integration/conftest.py

@@ -1,17 +1,20 @@
 import logging
 import os
+import shutil
 import subprocess
+import tempfile
 
 import pytest
 
 from colin.core.colin import _set_logging
+from colin.core.target import ImageTarget, OstreeTarget, DockerfileTarget
 
 _set_logging(level=logging.DEBUG)
 
 BASH_IMAGE = "colin-test-bash"
 LS_IMAGE = "colin-test-ls"
-BUSYBOX_IMAGE = "docker.io/library/busybox"
-LABELS_IMAGE = "colin-labels:latest"
+BUSYBOX_IMAGE = "busybox"
+LABELS_IMAGE = "colin-labels"
 IMAGES = {
     BASH_IMAGE: {
         "dockerfile_path": "Dockerfile-bash"
@@ -25,29 +28,140 @@
 }
 
 
-def build_images():
-    """ build container images we need for testing """
-    this_dir = os.path.abspath(os.path.dirname(__file__))
-    data_dir = os.path.join(this_dir, os.path.pardir, "data")
-    for image_name, image_data in IMAGES.items():
-        cmd_create = ["podman", "build", "-t", image_name, "-f",
-                      image_data["dockerfile_path"], data_dir]
+def build_image_if_not_exists(image_name):
+    try:
+        subprocess.check_call(["podman", "image", "inspect", image_name],
+                              stdout=subprocess.PIPE)
+    except subprocess.CalledProcessError:
+        this_dir = os.path.abspath(os.path.dirname(__file__))
+        data_dir = os.path.join(this_dir, os.path.pardir, "data")
+
+        dockerfile_path = IMAGES[image_name]["dockerfile_path"]
+        cmd_create = ["podman", "build", "-t", image_name, "-f", dockerfile_path, data_dir]
         output = subprocess.check_output(cmd_create)
         assert output
 
 
-@pytest.fixture(autouse=True, scope='session')
-def setup_test_session():
-    """ set up environment before testing """
-    for image_name in IMAGES:
-        try:
-            subprocess.check_call(["podman", "image", "inspect", image_name],
-                                  stdout=subprocess.PIPE)
-        except subprocess.CalledProcessError:
-            break
-    # executed if break was not reached
-    else:
-        # all images were found, we're good
-        return
-    # the loop ended with break, create images
-    build_images()
+def pull_image_if_not_exists(image_name):
+    try:
+        subprocess.check_call(["podman", "image", "inspect", image_name],
+                              stdout=subprocess.PIPE)
+    except subprocess.CalledProcessError:
+        subprocess.check_call(["podman", "pull", image_name],
+                              stdout=subprocess.PIPE)
+
+
+def convert_image_to_ostree(image_name):
+    # /tmp is tmpfs and ostree can't do its magic there
+    tmpdir_path = tempfile.mkdtemp(prefix="pytest-", dir="/var/tmp")
+    ostree_path = os.path.join(tmpdir_path, "os3")
+    os.makedirs(ostree_path)
+    skopeo_target = get_skopeo_path(image_name=image_name, ostree_path=ostree_path)
+
+    subprocess.check_call(["ostree", "init", "--mode", "bare-user-only",
+                           "--repo", ostree_path])
+
+    cmd = ["podman", "push", image_name, skopeo_target]
+    subprocess.check_call(cmd)
+    return ostree_path
+
+
+def get_target(name, type):
+    if type == "image":
+
+        target = ImageTarget(target=name,
+                             pull=False)
+        yield target
+        target.clean_up()
+
+    elif type == "ostree":
+
+        ostree_path = convert_image_to_ostree(name)
+        skopeo_target = get_skopeo_path(image_name=name,
+                                        ostree_path=ostree_path)
+
+        ostree_target = OstreeTarget(target=skopeo_target)
+        yield ostree_target
+        ostree_target.clean_up()
+        shutil.rmtree(ostree_path)
+
+    elif type == "dockerfile":
+
+        this_dir = os.path.abspath(os.path.dirname(__file__))
+        data_dir = os.path.join(this_dir, os.path.pardir, "data")
+        dockerfile_path = os.path.join(data_dir, IMAGES[name]["dockerfile_path"])
+
+        yield DockerfileTarget(target=dockerfile_path)
+
+
+def get_skopeo_path(image_name, ostree_path):
+    return "ostree:%s@%s" % (image_name, ostree_path)
+
+
+@pytest.fixture(scope="session")
+def label_image():
+    build_image_if_not_exists(LABELS_IMAGE)
+
+
+@pytest.fixture(scope="session",
+                params=["image", "ostree", "dockerfile"])
+def target_label(request, label_image):
+    for t in get_target(name=LABELS_IMAGE,
+                        type=request.param):
+        yield t
+
+
+@pytest.fixture(scope="session",
+                params=["image", "ostree", "dockerfile"])
+def target_label_image_and_dockerfile(request, label_image):
+    for t in get_target(name=LABELS_IMAGE,
+                        type=request.param):
+        yield t
+
+
+@pytest.fixture(scope="session")
+def target_bash_image():
+    build_image_if_not_exists(BASH_IMAGE)
+
+
+@pytest.fixture(scope="session",
+                params=["image", "ostree"])
+def target_bash(request, target_bash_image):
+    for t in get_target(name=BASH_IMAGE,
+                        type=request.param):
+        yield t
+
+
+@pytest.fixture(scope="session")
+def target_ls_image():
+    build_image_if_not_exists(LS_IMAGE)
+
+
+@pytest.fixture(scope="session",
+                params=["image", "ostree"])
+def target_ls(request, target_ls_image):
+    for t in get_target(name=LS_IMAGE,
+                        type=request.param):
+        yield t
+
+
+@pytest.fixture(scope="session",
+                params=[LS_IMAGE, BASH_IMAGE])
+def target_help_file(request, target_ls, target_bash):
+    if request.param == LS_IMAGE:
+        return target_ls, False
+    if request.param == BASH_IMAGE:
+        return target_bash, True
+
+
+@pytest.fixture(scope="session")
+def target_busybox_image():
+    pull_image_if_not_exists(image_name=BUSYBOX_IMAGE)
+
+
+@pytest.fixture(scope="session",
+                params=["image", "ostree"])
+def target_busybox(request, target_busybox_image):
+    for t in get_target(name=BUSYBOX_IMAGE,
+                        type=request.param):
+        yield t

tests/integration/test_cont.py

@@ -3,18 +3,13 @@
 """
 import pytest
 
-from colin.core.target import ImageTarget
-from tests.integration.conftest import BUSYBOX_IMAGE, LABELS_IMAGE
 
-
-def test_file_is_present():
-    image = ImageTarget(target=BUSYBOX_IMAGE, pull=True)
-    assert image.file_is_present("/etc/passwd")
-    assert not image.file_is_present("/oglogoblogologlo")
+def test_file_is_present(target_busybox):
+    assert target_busybox.file_is_present("/etc/passwd")
+    assert not target_busybox.file_is_present("/oglogoblogologlo")
     with pytest.raises(IOError):
-        image.file_is_present("/etc")
+        target_busybox.file_is_present("/etc")
 
 
-def test_labels_are_present():
-    image = ImageTarget(target=LABELS_IMAGE, pull=False)
-    assert isinstance(image.labels, dict)
+def test_labels_are_present(target_label):
+    assert isinstance(target_label.labels, dict)

tests/integration/test_fs_checks.py

@@ -13,10 +13,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
-import colin
 import pytest
 
-from tests.integration.conftest import BASH_IMAGE, LS_IMAGE
+import colin
 
 
 @pytest.fixture()
@@ -35,13 +34,22 @@ def ruleset():
     }
 
 
-@pytest.mark.parametrize("image_name,should_pass", [
-    (LS_IMAGE, False),
-    (BASH_IMAGE, True),
-])
-def test_help_file_or_readme(ruleset, image_name, should_pass):
+def test_help_file_or_readme_bash(ruleset, target_bash):
+    help_file_or_readme_test(ruleset=ruleset,
+                             image=target_bash,
+                             should_pass=True)
+
+
+def test_help_file_or_readme_ls(ruleset, target_ls):
+    help_file_or_readme_test(ruleset=ruleset,
+                             image=target_ls,
+                             should_pass=False)
+
+
+def help_file_or_readme_test(ruleset, image, should_pass):
     """ verify that help_file_or_readme check works well """
-    results = colin.run(image_name, "image", ruleset=ruleset, logging_level=10, pull=False)
+    results = colin.run(target=image.target_name,
+                        target_type=image.target_type,
+                        ruleset=ruleset, logging_level=10, pull=False)
     assert results.ok
     assert results.fail is not should_pass
-