diff --git a/README.md b/README.md index 560545e..8418e5d 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,10 @@ in Firewall. For Fedora 21 this can be done so: firewall-cmd --permanent --add-port=8000/tcp ./manage.py runserver 0.0.0.0:8000 +For dockerfile_lint support, you'll need to install it: + + sudo npm install -g git+https://github.com/redhataccess/dockerfile_lint + Manipulate with the data ------------------------ diff --git a/dbs/admin.py b/dbs/admin.py index a18edef..756e31f 100644 --- a/dbs/admin.py +++ b/dbs/admin.py @@ -1,8 +1,9 @@ from __future__ import absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement from django.contrib import admin -from .models import TaskData, Task, Rpm, Registry, YumRepo, Image, ImageRegistryRelation +from .models import TaskData, TaskLint, Task, Rpm, Registry, YumRepo, Image, ImageRegistryRelation +admin.site.register(TaskLint) admin.site.register(TaskData) admin.site.register(Task) admin.site.register(Rpm) diff --git a/dbs/api/core.py b/dbs/api/core.py index 772746b..2697694 100644 --- a/dbs/api/core.py +++ b/dbs/api/core.py @@ -8,7 +8,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.shortcuts import get_object_or_404 -from dbs.models import Task, TaskData, Dockerfile, Image +from dbs.models import Task, TaskData, TaskLint, Dockerfile, Image from dbs.task_api import TaskApi from dbs.utils import chain_dict_get @@ -21,6 +21,13 @@ class ErrorDuringRequest(Exception): """ indicate that there was an error during processing request; e.g. 404, invalid sth... """ +def lint_output_callback(task_id, lint, **kwargs): + t = Task.objects.get(id=task_id) + tl = TaskLint(lint=lint["html_markup"]) + tl.save() + t.task_lint = tl + t.save() + def new_image_callback(task_id, response_tuple): try: response_hash, df, build_log = response_tuple @@ -59,6 +66,7 @@ def new_image_callback(task_id, response_tuple): t.status = Task.STATUS_SUCCESS else: + logger.debug("task failed: %s" % repr (response_tuple)) t.status = Task.STATUS_FAILED t.save() @@ -76,10 +84,12 @@ def build(post_args, **kwargs): type=Task.TYPE_BUILD, owner=owner, task_data=td) t.save() + lint_callback = partial(lint_output_callback, t.id) callback = partial(new_image_callback, t.id) post_args.update({'build_image': "buildroot-fedora", 'local_tag': local_tag, - 'callback': callback}) + 'callback': callback, + 'lint_callback': lint_callback}) task_id = builder_api.build_docker_image(**post_args) t.celery_id = task_id t.save() diff --git a/dbs/lint.py b/dbs/lint.py new file mode 100644 index 0000000..81598fa --- /dev/null +++ b/dbs/lint.py @@ -0,0 +1,171 @@ +from __future__ import absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement + +import git +import shutil +import subprocess +import tempfile +import os +import json + +def html_escape(s): + # In python3 we can use html.escape(s, quote=False) + return s.replace("&", "&").replace("<", "<").replace(">", ">") + +class DockerfileLint: + rules = "/usr/lib/node_modules/dockerfile_lint/sample_rules.yaml" + + PF_CLASSES = { 'error': + { 'alert': '
', + 'icon': """ + + + + """ }, + + 'warn': + { 'alert': '
', + 'icon': """ + + + + """ }, + + 'info': + { 'alert': '
', + 'icon': """ + """} + } + + def __init__ (self, git_url, git_path=None, git_commit=None): + self._git_url = git_url + self._git_path = git_path + self._git_commit = git_commit + self._temp_dir = None + + def __del__ (self): + if self._temp_dir: + try: + shutil.rmtree(self._temp_dir) + except (IOError, OSError, AttributeError) as exc: + pass + + def _get_dockerfile (self): + self._temp_dir = tempfile.mkdtemp () + git.Repo.clone_from (self._git_url, self._temp_dir) + if self._git_path: + if self._git_path.endswith('Dockerfile'): + git_df_dir = os.path.dirname(self._git_path) + df_path = os.path.abspath(os.path.join(self._temp_dir, + git_df_dir)) + else: + df_path = os.path.abspath(os.path.join(self._temp_dir, + self._git_path)) + else: + df_path = self._temp_dir + + self._Dockerfile = os.path.join(df_path, "Dockerfile") + + def _run_dockerfile_lint (self): + with open ("/dev/null", "rw") as devnull: + dfl = subprocess.Popen (["dockerfile_lint", + "-j", + "-r", self.rules, + "-f", self._Dockerfile], + stdin=devnull, + stdout=subprocess.PIPE, + stderr=devnull, + close_fds=True) + + (stdout, stderr) = dfl.communicate () + jsonstr = stdout.decode () + self._lint = json.loads (jsonstr) + + def _mark_dockerfile (self): + out = "" + with open(self._Dockerfile, "r") as df: + dflines = df.readlines () + dflines.append ("\n") # Extra line to bind 'absent' messages to + lastline = len (dflines) + msgs_by_linenum = {} + for severity in ["error", "warn", "info"]: + for msg in self._lint[severity]["data"]: + if "line" in msg: + linenum = msg["line"] + else: + linenum = lastline + + msgs = msgs_by_linenum.get (linenum, []) + msgs.append (msg) + msgs_by_linenum[linenum] = msgs + + linenum = 1 + for line in dflines: + msgs = msgs_by_linenum.get (linenum, []) + linenum += 1 + if not msgs: + continue + + display_line = False + msgout = "" + for msg in msgs: + if "line" in msg: + display_line = True + + level = msg["level"] + classes = self.PF_CLASSES.get (level) + if not classes: + continue + + msgout += classes["alert"] + classes["icon"] + "\n" + msgout += (" " + + html_escape (msg["message"]) + + " ") + description = msg.get ("description", "None") + if description != "None": + msgout += html_escape (description) + url = msg.get ("reference_url", "None") + if url != "None": + if type (url) == list: + url = reduce (lambda a, b: a + b, url) + + msgout += (' ' + '(more info)' % url) + msgout += "\n
\n" + + if display_line: + out += (("
%d:" % linenum) +
+                            html_escape (line).rstrip () + "
\n") + + out += msgout + + if out == "": + out = """ +
+ + Looks good! This Dockerfile has no output from dockerfile_lint. +
+""" + + self._html_markup = out + + def run (self): + self._get_dockerfile () + self._run_dockerfile_lint () + self._mark_dockerfile () + + if self._temp_dir: + try: + shutil.rmtree(self._temp_dir) + self._temp_dir = None + except (IOError, OSError) as exc: + pass + + return self._html_markup + + def get_json (self): + return self._lint + +if __name__ == "__main__": + git_url = "https://github.com/TomasTomecek/docker-hello-world.git" + lint = DockerfileLint (git_url) + print (lint.run ()) diff --git a/dbs/models.py b/dbs/models.py index cb8a473..38b0050 100644 --- a/dbs/models.py +++ b/dbs/models.py @@ -20,6 +20,12 @@ def __unicode__(self): return json.dumps(json.loads(self.json), indent=4) +class TaskLint(models.Model): + lint = models.TextField() + + def __unicode__(self): + return self.lint + class Task(models.Model): STATUS_PENDING = 1 @@ -48,6 +54,7 @@ class Task(models.Model): type = models.IntegerField(choices=_TYPE_NAMES.items()) owner = models.CharField(max_length=38) task_data = models.ForeignKey(TaskData) + task_lint = models.ForeignKey(TaskLint, null=True, blank=True) log = models.TextField(blank=True, null=True) class Meta: diff --git a/dbs/task_api.py b/dbs/task_api.py index b68cc76..f13827e 100644 --- a/dbs/task_api.py +++ b/dbs/task_api.py @@ -21,7 +21,11 @@ def watch_task(task, callback, kwargs=None): :return: None """ - response = task.wait() + try: + response = task.wait() + except Exception as exc: + response = exc + if kwargs: callback(response, **kwargs) else: @@ -33,7 +37,7 @@ class TaskApi(object): def build_docker_image(self, build_image, git_url, local_tag, git_dockerfile_path=None, git_commit=None, parent_registry=None, target_registries=None, tag=None, repos=None, - callback=None, kwargs=None): + callback=None, lint_callback=None, kwargs=None): """ build docker image from supplied git repo @@ -63,14 +67,43 @@ def build_docker_image(self, build_image, git_url, local_tag, git_dockerfile_pat 'git_commit': git_commit, 'git_dockerfile_path': git_dockerfile_path, 'repos': repos} - task_info = tasks.build_image.apply_async(args=args, kwargs=task_kwargs, - link=tasks.submit_results.s()) - task_id = task_info.task_id - if callback: - t = Thread(target=watch_task, args=(task_info, callback, kwargs)) - #w.daemon = True - t.start() - return task_id + + # The linter task, which runs dockerfile_lint + linter_task = tasks.linter.s(git_url, + git_dockerfile_path, + git_commit) + + # This task builds the image + build_image_task = tasks.build_image.subtask((build_image, + git_url, + local_tag), + **task_kwargs) + + # This task submits the results + submit_results_task = tasks.submit_results.s() + + # Chain the tasks together in the right order and start them + task_chain = (linter_task | + build_image_task | + submit_results_task).apply_async() + + # Call lint_callback when the linter task is done + linter = task_chain.parent.parent # 3rd from last task + lint_watcher = Thread(target=watch_task, + args=(linter, + lint_callback, + kwargs)) + lint_watcher.start() + + # Call callback when the entire chain is done + chain_watcher = Thread(target=watch_task, + args=(task_chain, + callback, + kwargs)) + chain_watcher.start() + + # Return the celery task ID of the chain + return task_chain.task_id def find_dockerfiles_in_git(self): raise NotImplemented() diff --git a/dbs/tasks.py b/dbs/tasks.py index e565c31..9d20c8c 100644 --- a/dbs/tasks.py +++ b/dbs/tasks.py @@ -3,7 +3,38 @@ from celery import shared_task from dock.core import DockerBuilder, DockerTasker from dock.outer import PrivilegedDockerBuilder +from dbs.lint import DockerfileLint +import time +class LintErrors(Exception): + """ + This exception indicates the build was not attempted due to + lint errors. + """ + +@shared_task +def linter(git_url, git_path=None, git_commit=None): + """ + run dockerfile_lint on the Dockerfile we want to build + + :param git_url: url to git repo + :param git_path: path to dockerfile within git repo (default is ./Dockerfile) + :param git_commit: which commit to checkout (master by default) + :return: HTML markup of Dockerfile with dockerfile_lint messages + """ + json = None + try: + lint = DockerfileLint (git_url, git_path, git_commit) + html_markup = lint.run () + json = lint.get_json () + except OSError as exc: + # Perhaps dockerfile_lint is not installed + html_markup = "Executing dockerfile_lint: %s" % exc.strerror + except ValueError as exc: + # Perhaps there was a problem parsing the JSON output + html_markup = "Internal error: %s" % exc.message + finally: + return { "json": json, "html_markup": html_markup } @shared_task def build_image_hostdocker( @@ -43,13 +74,14 @@ def build_image_hostdocker( # TODO: postbuild_data = run_postbuild_plugins(d, private_tag) return inspect_data -@shared_task -def build_image(build_image, git_url, local_tag, git_dockerfile_path=None, +@shared_task(throws=(LintErrors,)) +def build_image(lint, build_image, git_url, local_tag, git_dockerfile_path=None, git_commit=None, parent_registry=None, target_registries=None, tag=None, repos=None, store_results=True): """ build docker image from provided arguments inside privileged container + :param lint: output from linter task :param build_image: name of the build image (supplied docker image is built inside this image) :param git_url: url to git repo :param local_tag: image is known within the service with this tag @@ -63,6 +95,13 @@ def build_image(build_image, git_url, local_tag, git_dockerfile_path=None, in local docker registry :return: dict with data from docker inspect """ + if lint and lint["json"]: + count = lint["json"]["error"]["count"] + if count > 0: + time.sleep (1) # Shouldn't be needed but seems to be + raise LintErrors("Build aborted: %d dockerfile_lint errors" % + count) + db = PrivilegedDockerBuilder(build_image, { "git_url": git_url, "local_tag": local_tag, @@ -108,4 +147,4 @@ def submit_results(result): """ # 2 requests, one for 'finished', other for data print(result) - + return result diff --git a/dbs/web/templates/dbs/task_detail.html b/dbs/web/templates/dbs/task_detail.html index 6b5f2af..5ccf892 100644 --- a/dbs/web/templates/dbs/task_detail.html +++ b/dbs/web/templates/dbs/task_detail.html @@ -49,6 +49,10 @@

Task Detail

  • Logs

  • {{ task.log|linebreaks }}
    • {% endif %} + {% if task.task_lint != nil %} +

  • Dockerfile lint

    + {{ task.task_lint|safe }}
    • + {% endif %} {% endblock %}