diff --git a/apps/common/api.py b/apps/common/api.py index 58a5822c1..0e626cacd 100644 --- a/apps/common/api.py +++ b/apps/common/api.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- # +import os import json +import uuid +from django.core.cache import cache from rest_framework.views import APIView from rest_framework.views import Response from ldap3 import Server, Connection @@ -11,6 +14,7 @@ from django.conf import settings from .permissions import IsSuperUser, IsAppUser from .serializers import MailTestSerializer, LDAPTestSerializer +from .const import FILE_END_GUARD class MailTestingAPI(APIView): @@ -105,3 +109,30 @@ class DjangoSettingsAPI(APIView): if i.isupper(): configs[i] = str(getattr(settings, i)) return Response(configs) + + +class FileTailApi(APIView): + permission_classes = (IsSuperUser,) + default_buff_size = 1024 * 10 + end = False + buff_size = None + + def get(self, request, *args, **kwargs): + file_path = request.query_params.get("file") + self.buff_size = request.query_params.get('buffer') or self.default_buff_size + mark = request.query_params.get("mark") or str(uuid.uuid4()) + + if not os.path.isfile(file_path): + return Response({"data": _("Waiting ...")}, status=203) + + with open(file_path, 'r') as f: + offset = cache.get(mark, 0) + f.seek(offset) + data = f.read(self.buff_size).replace('\n', '\r\n') + mark = str(uuid.uuid4()) + cache.set(mark, f.tell(), 5) + + if FILE_END_GUARD in data: + data = data.replace(FILE_END_GUARD, '') + self.end = True + return Response({"data": data, 'end': self.end, 'mark': mark}) \ No newline at end of file diff --git a/apps/common/celery.py b/apps/common/celery.py index e79f1d434..352f1ff45 100644 --- a/apps/common/celery.py +++ b/apps/common/celery.py @@ -5,7 +5,7 @@ import json from functools import wraps from celery import Celery, subtask -from celery.signals import worker_ready, worker_shutdown +from celery.signals import worker_ready, worker_shutdown, task_prerun, task_postrun from django.db.utils import ProgrammingError, OperationalError from .utils import get_logger @@ -199,3 +199,16 @@ def after_app_shutdown(sender=None, headers=None, body=None, **kwargs): ', '.join(__AFTER_APP_SHUTDOWN_CLEAN_TASKS)) ) PeriodicTask.objects.filter(name__in=__AFTER_APP_SHUTDOWN_CLEAN_TASKS).delete() + + +@task_prerun.connect +def pre_run_task_signal_handler(sender, task, *args, **kwargs): + + print("Sender: {}".format(sender)) + print("Task: {}".format(task)) + + +@task_postrun.connect +def post_run_task_signal_handler(sender, task, *args, **kwargs): + print("Sender: {}".format(sender)) + print("Task: {}".format(task)) \ No newline at end of file diff --git a/apps/common/const.py b/apps/common/const.py index a28b0f1db..f3669c49d 100644 --- a/apps/common/const.py +++ b/apps/common/const.py @@ -4,4 +4,5 @@ from django.utils.translation import ugettext as _ create_success_msg = _("%(name)s was created successfully") -update_success_msg = _("%(name)s was updated successfully") \ No newline at end of file +update_success_msg = _("%(name)s was updated successfully") +FILE_END_GUARD = ">>> Content End <<<" diff --git a/apps/common/templates/common/tail_file.html b/apps/common/templates/common/tail_file.html new file mode 100644 index 000000000..8c24fa707 --- /dev/null +++ b/apps/common/templates/common/tail_file.html @@ -0,0 +1,96 @@ +{% load static %} + + term.js + + + +
+
+
+
+ + + + diff --git a/apps/common/urls/api_urls.py b/apps/common/urls/api_urls.py index a9075e66e..e1e6c19ee 100644 --- a/apps/common/urls/api_urls.py +++ b/apps/common/urls/api_urls.py @@ -10,4 +10,5 @@ urlpatterns = [ url(r'^v1/mail/testing/$', api.MailTestingAPI.as_view(), name='mail-testing'), url(r'^v1/ldap/testing/$', api.LDAPTestingAPI.as_view(), name='ldap-testing'), url(r'^v1/django-settings/$', api.DjangoSettingsAPI.as_view(), name='django-settings'), + url(r'^v1/tail-file/$', api.FileTailApi.as_view(), name='tail-file'), ] diff --git a/apps/common/urls/view_urls.py b/apps/common/urls/view_urls.py index 466f7c49c..d2135f2d4 100644 --- a/apps/common/urls/view_urls.py +++ b/apps/common/urls/view_urls.py @@ -11,4 +11,7 @@ urlpatterns = [ url(r'^email/$', views.EmailSettingView.as_view(), name='email-setting'), url(r'^ldap/$', views.LDAPSettingView.as_view(), name='ldap-setting'), url(r'^terminal/$', views.TerminalSettingView.as_view(), name='terminal-setting'), + + url(r'^tail-file/$', views.TailFileView.as_view(), name='tail-file'), + url(r'^celery/task/log/$', views.CeleryTaskLogView.as_view(), name='celery-task-log'), ] diff --git a/apps/common/views.py b/apps/common/views.py index 99b924423..a57834a77 100644 --- a/apps/common/views.py +++ b/apps/common/views.py @@ -1,5 +1,7 @@ -from django.views.generic import TemplateView -from django.shortcuts import render, redirect + +from django.core.cache import cache +from django.views.generic import TemplateView, View +from django.shortcuts import render, redirect, Http404, reverse from django.contrib import messages from django.utils.translation import ugettext as _ from django.conf import settings @@ -120,3 +122,34 @@ class TerminalSettingView(AdminUserRequiredMixin, TemplateView): context.update({"form": form}) return render(request, self.template_name, context) + +class TailFileView(AdminUserRequiredMixin, TemplateView): + template_name = 'common/tail_file.html' + + def get_context_data(self, **kwargs): + file_path = self.request.GET.get("file") + context = super().get_context_data(**kwargs) + context.update({"file_path": file_path}) + return context + + +class CeleryTaskLogView(AdminUserRequiredMixin, TemplateView): + template_name = 'common/tail_file.html' + task_log_path = None + + def get(self, request, *args, **kwargs): + task = self.request.GET.get('task') + if not task: + raise Http404("Not found task") + + self.task_log_path = cache.get(task) + if not self.task_log_path: + raise Http404("Not found task log file") + return super().get(request, *args, **kwargs) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context.update({ + 'file_path': self.task_log_path + }) + return context diff --git a/apps/ops/api.py b/apps/ops/api.py index 92933098c..db27653cb 100644 --- a/apps/ops/api.py +++ b/apps/ops/api.py @@ -1,19 +1,23 @@ # ~*~ coding: utf-8 ~*~ import uuid -import re +import os from django.core.cache import cache from django.shortcuts import get_object_or_404 +from django.utils.translation import ugettext as _ from rest_framework import viewsets, generics -from rest_framework.generics import RetrieveAPIView +from rest_framework.views import APIView from rest_framework.views import Response from .hands import IsSuperUser +from common.const import FILE_END_GUARD from .models import Task, AdHoc, AdHocRunHistory from .serializers import TaskSerializer, AdHocSerializer, AdHocRunHistorySerializer from .tasks import run_ansible_task + + class TaskViewSet(viewsets.ModelViewSet): queryset = Task.objects.all() serializer_class = TaskSerializer @@ -27,8 +31,8 @@ class TaskRun(generics.RetrieveAPIView): def retrieve(self, request, *args, **kwargs): task = self.get_object() - run_ansible_task.delay(str(task.id)) - return Response({"msg": "start"}) + t = run_ansible_task.delay(str(task.id)) + return Response({"task": t.id}) class AdHocViewSet(viewsets.ModelViewSet): @@ -63,24 +67,28 @@ class AdHocRunHistorySet(viewsets.ModelViewSet): return self.queryset -class AdHocHistoryOutputAPI(RetrieveAPIView): - queryset = AdHocRunHistory.objects.all() +class LogFileViewApi(APIView): permission_classes = (IsSuperUser,) buff_size = 1024 * 10 end = False - def retrieve(self, request, *args, **kwargs): - history = self.get_object() + def get(self, request, *args, **kwargs): + file_path = request.query_params.get("file") mark = request.query_params.get("mark") or str(uuid.uuid4()) - with open(history.log_path, 'r') as f: + if not os.path.isfile(file_path): + print(file_path) + return Response({"error": _("Log file not found")}, status=204) + + with open(file_path, 'r') as f: offset = cache.get(mark, 0) f.seek(offset) data = f.read(self.buff_size).replace('\n', '\r\n') - print(repr(data)) mark = str(uuid.uuid4()) cache.set(mark, f.tell(), 5) - if history.is_finished and data == '': + if FILE_END_GUARD in data: + data.replace(FILE_END_GUARD, '') self.end = True + return Response({"data": data, 'end': self.end, 'mark': mark}) diff --git a/apps/ops/models.py b/apps/ops/models.py index 8a34e125a..62bd7d084 100644 --- a/apps/ops/models.py +++ b/apps/ops/models.py @@ -6,6 +6,7 @@ import os import time import datetime +from celery import current_task from django.db import models from django.conf import settings from django.utils import timezone @@ -211,11 +212,20 @@ class AdHoc(models.Model): return self._run_only() def _run_and_record(self): - history = AdHocRunHistory(adhoc=self, task=self.task) - time_start = time.time() try: - with open(history.log_path, 'w') as f: - raw, summary = self._run_only(file_obj=f) + hid = current_task.request.id + except AttributeError: + hid = str(uuid.uuid4()) + history = AdHocRunHistory(id=hid, adhoc=self, task=self.task) + time_start = time.time() + # f = open(history.log_path, 'w') + try: + date_start = timezone.now().strftime('%Y-%m-%d %H:%M:%S') + # f.write("{} {}\r\n\r\n".format(date_start, self.task.name)) + raw, summary = self._run_only() + # raw, summary = self._run_only(file_obj=f) + date_end = timezone.now().strftime('%Y-%m-%d %H:%M:%S') + # f.write("\r\n{} Task finish\r\n".format(date_end)) history.is_finished = True if summary.get('dark'): history.is_success = False @@ -227,6 +237,7 @@ class AdHoc(models.Model): except Exception as e: return {}, {"dark": {"all": str(e)}, "contacted": []} finally: + # f.close() history.date_finished = timezone.now() history.timedelta = time.time() - time_start history.save() diff --git a/apps/ops/tasks.py b/apps/ops/tasks.py index 41f60f20c..5a2738cd9 100644 --- a/apps/ops/tasks.py +++ b/apps/ops/tasks.py @@ -1,6 +1,7 @@ # coding: utf-8 from celery import shared_task, subtask + from common.utils import get_logger, get_object_or_none from .models import Task @@ -12,14 +13,13 @@ def rerun_task(): @shared_task -def run_ansible_task(task_id, callback=None, **kwargs): +def run_ansible_task(tid, callback=None, **kwargs): """ - :param task_id: is the tasks serialized data + :param tid: is the tasks serialized data :param callback: callback function name :return: """ - - task = get_object_or_none(Task, id=task_id) + task = get_object_or_none(Task, id=tid) if task: result = task.run() if callback is not None: diff --git a/apps/ops/templates/ops/adhoc_history.html b/apps/ops/templates/ops/adhoc_history.html index 802e8e8f6..cab76f692 100644 --- a/apps/ops/templates/ops/adhoc_history.html +++ b/apps/ops/templates/ops/adhoc_history.html @@ -82,7 +82,8 @@ function initTable() { select: [], columnDefs: [ {targets: 1, createdCell: function (td, cellData, rowData) { - $(td).html(cellData); + var d = new Date(cellData); + $(td).html(d); }}, {targets: 2, createdCell: function (td, cellData) { var total = "" + cellData.total + ""; diff --git a/apps/ops/templates/ops/adhoc_history_detail.html b/apps/ops/templates/ops/adhoc_history_detail.html index f3d18f8ba..15c26376f 100644 --- a/apps/ops/templates/ops/adhoc_history_detail.html +++ b/apps/ops/templates/ops/adhoc_history_detail.html @@ -18,6 +18,9 @@
  • {% trans 'Run history detail' %}
    • +
  • + {% trans 'History output' %} +
    • diff --git a/apps/ops/templates/ops/adhoc_history_output.html b/apps/ops/templates/ops/adhoc_history_output.html index 34f8078bf..781c2adf5 100644 --- a/apps/ops/templates/ops/adhoc_history_output.html +++ b/apps/ops/templates/ops/adhoc_history_output.html @@ -1,93 +1,37 @@ +{% extends 'base.html' %} {% load static %} - - - - term.js - - +{% load i18n %} + +{% block custom_head_css_js %} + + + + - -
    -
    +{% endblock %} +{% block content %} +
    +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    -
    - +{% endblock %} - - - - diff --git a/apps/ops/templates/ops/celery_task_output.html b/apps/ops/templates/ops/celery_task_output.html new file mode 100644 index 000000000..d48b35dcf --- /dev/null +++ b/apps/ops/templates/ops/celery_task_output.html @@ -0,0 +1,94 @@ +{% load static %} + + term.js + + + +
    +
    +
    +
    + + + + diff --git a/apps/ops/templates/ops/task_adhoc.html b/apps/ops/templates/ops/task_adhoc.html index 2d2c4de5c..381048602 100644 --- a/apps/ops/templates/ops/task_adhoc.html +++ b/apps/ops/templates/ops/task_adhoc.html @@ -105,6 +105,10 @@ $(td).html(cellData.user) } }}, + {targets: 6, createdCell: function (td, cellData) { + var d = new Date(cellData); + $(td).html(d.toLocaleString()) + }}, {targets: 7, createdCell: function (td, cellData, rowData) { var detail_btn = '{% trans "Detail" %}'.replace('{{ DEFAULT_PK }}', cellData); if (cellData) { diff --git a/apps/ops/templates/ops/task_detail.html b/apps/ops/templates/ops/task_detail.html index 4da7e0f58..4aba9998a 100644 --- a/apps/ops/templates/ops/task_detail.html +++ b/apps/ops/templates/ops/task_detail.html @@ -24,6 +24,9 @@
  • {% trans 'Run history' %}
    • +
  • + {% trans 'Last run output' %} +
    • @@ -160,6 +163,5 @@
    - {% include 'users/_user_update_pk_modal.html' %} {% endblock %} diff --git a/apps/ops/templates/ops/task_history.html b/apps/ops/templates/ops/task_history.html index cc620237d..92401a0f9 100644 --- a/apps/ops/templates/ops/task_history.html +++ b/apps/ops/templates/ops/task_history.html @@ -30,7 +30,7 @@
    - {% trans 'History of ' %} {{ object.task.name }}:{{ object.short_id }} + {% trans 'History of ' %} {{ object.name }}:{{ object.short_id }}
    @@ -85,7 +85,8 @@ function initTable() { select: [], columnDefs: [ {targets: 1, createdCell: function (td, cellData, rowData) { - $(td).html(cellData); + var d = new Date(cellData); + $(td).html(d.toLocaleString()); }}, {targets: 2, createdCell: function (td, cellData) { var total = "" + cellData.total + ""; diff --git a/apps/ops/templates/ops/task_list.html b/apps/ops/templates/ops/task_list.html index f098a4d78..37a73b4c0 100644 --- a/apps/ops/templates/ops/task_list.html +++ b/apps/ops/templates/ops/task_list.html @@ -111,9 +111,9 @@ $(document).ready(function() { var error = function (data) { alert(data) }; - var success = function () { - alert("任务开始执行，重定向到任务详情页面，多刷新几次查看结果") - window.location = "{% url 'ops:task-detail' pk=DEFAULT_PK %}".replace('{{ DEFAULT_PK }}', uid); + var success = function(data) { + var task_id = data.task; + window.location = "{% url 'ops:adhoc-history-output' pk=DEFAULT_PK %}".replace("{{ DEFAULT_PK }}", task_id); }; APIUpdateAttr({ url: the_url, diff --git a/apps/ops/urls/api_urls.py b/apps/ops/urls/api_urls.py index 26c1c89a1..a5f94da29 100644 --- a/apps/ops/urls/api_urls.py +++ b/apps/ops/urls/api_urls.py @@ -15,7 +15,7 @@ router.register(r'v1/history', api.AdHocRunHistorySet, 'history') urlpatterns = [ url(r'^v1/tasks/(?P[0-9a-zA-Z\-]{36})/run/$', api.TaskRun.as_view(), name='task-run'), - url(r'^v1/history/(?P[0-9a-zA-Z\-]{36})/output/$', api.AdHocHistoryOutputAPI.as_view(), name='history-output'), + # url(r'^v1/history/(?P[0-9a-zA-Z\-]{36})/output/$', api.CeleryTaskOutputApi.as_view(), name='history-output'), ] urlpatterns += router.urls diff --git a/apps/ops/urls/view_urls.py b/apps/ops/urls/view_urls.py index 980f9d070..e6979bbf3 100644 --- a/apps/ops/urls/view_urls.py +++ b/apps/ops/urls/view_urls.py @@ -18,5 +18,6 @@ urlpatterns = [ url(r'^adhoc/(?P[0-9a-zA-Z\-]{36})/$', views.AdHocDetailView.as_view(), name='adhoc-detail'), url(r'^adhoc/(?P[0-9a-zA-Z\-]{36})/history/$', views.AdHocHistoryView.as_view(), name='adhoc-history'), url(r'^adhoc/history/(?P[0-9a-zA-Z\-]{36})/$', views.AdHocHistoryDetailView.as_view(), name='adhoc-history-detail'), + url(r'^adhoc/history/(?P[0-9a-zA-Z\-]{36})/_output/$', views.CeleryTaskOutputView.as_view(), name='adhoc-history-output-alone'), url(r'^adhoc/history/(?P[0-9a-zA-Z\-]{36})/output/$', views.AdHocHistoryOutputView.as_view(), name='adhoc-history-output'), ] diff --git a/apps/ops/views.py b/apps/ops/views.py index fbc942b56..5a294e7c7 100644 --- a/apps/ops/views.py +++ b/apps/ops/views.py @@ -2,7 +2,7 @@ from django.utils.translation import ugettext as _ from django.conf import settings -from django.views.generic import ListView, DetailView +from django.views.generic import ListView, DetailView, TemplateView from common.mixins import DatetimeSearchMixin from .models import Task, AdHoc, AdHocRunHistory @@ -121,6 +121,11 @@ class AdHocHistoryDetailView(AdminUserRequiredMixin, DetailView): return super().get_context_data(**kwargs) +class CeleryTaskOutputView(AdminUserRequiredMixin, TemplateView): + model = AdHocRunHistory + template_name = 'ops/celery_task_output.html' + + class AdHocHistoryOutputView(AdminUserRequiredMixin, DetailView): model = AdHocRunHistory template_name = 'ops/adhoc_history_output.html'