fix: 修正构建错误

Co-authored-by: ibuler <ibuler@qq.com>

.github/ISSUE_TEMPLATE.md

@@ -2,7 +2,7 @@
 
 
 ##### 使用版本
-[请提供你使用的Jumpserver版本 1.x.x 注: 0.3.x不再提供支持]
+[请提供你使用的JumpServer版本 如 2.0.1 注: 1.4及以下版本不再提供支持]
 
 ##### 问题复现步骤
 1. [步骤1]

.github/release-config.yml

@@ -0,0 +1,44 @@
+name-template: 'v$RESOLVED_VERSION'
+tag-template: 'v$RESOLVED_VERSION'
+categories:
+  - title: '🌱 新功能 Features'
+    labels:
+      - 'feature'
+      - 'enhancement'
+      - 'feat'
+      - '新功能'
+  - title: '🚀 性能优化 Optimization'
+    labels:
+      - 'perf'
+      - 'opt'
+      - 'refactor'
+      - 'Optimization'
+      - '优化'
+  - title: '🐛 Bug修复 Bug Fixes'
+    labels:
+      - 'fix'
+      - 'bugfix'
+      - 'bug'
+  - title: '🧰 其它 Maintenance'
+    labels:
+      - 'chore'
+      - 'docs'
+exclude-labels:
+  - 'no'
+  - '无需处理'
+  - 'wontfix'
+change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
+version-resolver:
+  major:
+    labels:
+      - 'major'
+  minor:
+    labels:
+      - 'minor'
+  patch:
+    labels:
+      - 'patch'
+  default: patch
+template: |
+  ## 版本变化  What’s Changed
+  $CHANGES

.github/workflows/jms-generic-action-handler.yml

@@ -0,0 +1,12 @@
+on: [push, pull_request, release]
+
+name: JumpServer repos generic handler
+
+jobs:
+  generic_handler:
+    name: Run generic handler
+    runs-on: ubuntu-latest
+    steps:
+      - uses: jumpserver/action-generic-handler@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.PRIVATE_TOKEN }}

.github/workflows/release-drafter.yml

@@ -0,0 +1,46 @@
+on:
+  push:
+    # Sequence of patterns matched against refs/tags
+    tags:
+      - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+
+name: Create Release And Upload assets
+
+jobs:
+  create-realese:
+    name: Create Release
+    runs-on: ubuntu-latest
+    outputs:
+      upload_url: ${{ steps.create_release.outputs.upload_url }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Get version
+        id: get_version
+        run: |
+          TAG=$(basename ${GITHUB_REF})
+          VERSION=${TAG/v/}
+          echo "::set-output name=TAG::$TAG"
+          echo "::set-output name=VERSION::$VERSION"
+      - name: Create Release
+        id: create_release
+        uses: release-drafter/release-drafter@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          config-name: release-config.yml
+          version: ${{ steps.get_version.outputs.TAG }}
+          tag: ${{ steps.get_version.outputs.TAG }}
+
+  build-and-release:
+    needs: create-realese
+    name: Build and Release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build it and upload
+        uses: jumpserver/action-build-upload-assets@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ needs.create-realese.outputs.upload_url }}

Dockerfile

@@ -1,15 +1,21 @@
 FROM registry.fit2cloud.com/public/python:v3
 MAINTAINER Jumpserver Team <ibuler@qq.com>
 
+ENV LANG=en_US.UTF-8
+
 WORKDIR /opt/jumpserver
 RUN useradd jumpserver
 
 COPY ./requirements /tmp/requirements
 
+RUN wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-6.repo \
+    && sed -i 's@/centos/@/centos-vault/@g' /etc/yum.repos.d/CentOS-Base.repo \
+    && sed -i 's@$releasever@6.10@g' /etc/yum.repos.d/CentOS-Base.repo
+
 RUN yum -y install epel-release && \
       echo -e "[mysql]\nname=mysql\nbaseurl=https://mirrors.tuna.tsinghua.edu.cn/mysql/yum/mysql57-community-el6/\ngpgcheck=0\nenabled=1" > /etc/yum.repos.d/mysql.repo
 RUN cd /tmp/requirements && yum -y install $(cat rpm_requirements.txt)
-RUN cd /tmp/requirements && pip install --upgrade pip setuptools && pip install wheel && \
+RUN cd /tmp/requirements && pip install --upgrade pip setuptools==49.6.0 && pip install wheel && \
     pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt || pip install -r requirements.txt
 RUN mkdir -p /root/.ssh/ && echo -e "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null" > /root/.ssh/config
 
@@ -18,9 +24,6 @@ RUN echo > config.yml
 VOLUME /opt/jumpserver/data
 VOLUME /opt/jumpserver/logs
 
-ENV LANG=zh_CN.UTF-8
-ENV LC_ALL=zh_CN.UTF-8
-
 EXPOSE 8070
 EXPOSE 8080
 ENTRYPOINT ["./entrypoint.sh"]

apps/assets/migrations/0050_auto_20200711_1740.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.10 on 2020-07-11 09:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0049_systemuser_sftp_root'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='asset',
+            name='created_by',
+            field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Created by'),
+        ),
+    ]

apps/assets/models/asset.py

@@ -221,7 +221,7 @@ class Asset(ProtocolsMixin, NodesRelationMixin, OrgModelMixin):
     hostname_raw = models.CharField(max_length=128, blank=True, null=True, verbose_name=_('Hostname raw'))
 
     labels = models.ManyToManyField('assets.Label', blank=True, related_name='assets', verbose_name=_("Labels"))
-    created_by = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Created by'))
+    created_by = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Created by'))
     date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True, verbose_name=_('Date created'))
     comment = models.TextField(max_length=128, default='', blank=True, verbose_name=_('Comment'))
 

apps/assets/models/authbook.py

@@ -3,7 +3,6 @@
 
 from django.db import models, transaction
 from django.db.models import Max
-from django.core.cache import cache
 from django.utils.translation import ugettext_lazy as _
 
 from orgs.mixins.models import OrgManager
@@ -59,19 +58,17 @@ class AuthBook(BaseUser):
         """
         username = kwargs['username']
         asset = kwargs['asset']
-        key_lock = 'KEY_LOCK_CREATE_AUTH_BOOK_{}_{}'.format(username, asset.id)
-        with cache.lock(key_lock):
-            with transaction.atomic():
-                cls.objects.filter(
-                    username=username, asset=asset, is_latest=True
-                ).update(is_latest=False)
-                max_version = cls.get_max_version(username, asset)
-                kwargs.update({
-                    'version': max_version + 1,
-                    'is_latest': True
-                })
-                obj = cls.objects.create(**kwargs)
-                return obj
+        with transaction.atomic():
+            # 使用select_for_update限制并发创建相同的username、asset条目
+            instances = cls.objects.select_for_update().filter(username=username, asset=asset)
+            instances.filter(is_latest=True).update(is_latest=False)
+            max_version = cls.get_max_version(username, asset)
+            kwargs.update({
+                'version': max_version + 1,
+                'is_latest': True
+            })
+            obj = cls.objects.create(**kwargs)
+            return obj
 
     @property
     def connectivity(self):

apps/assets/tasks/asset_user_connectivity.py

@@ -85,7 +85,7 @@ def test_asset_user_connectivity_util(asset_user, task_name):
         raw, summary = test_user_connectivity(
             task_name=task_name, asset=asset_user.asset,
             username=asset_user.username, password=asset_user.password,
-            private_key=asset_user.private_key
+            private_key=asset_user.private_key_file
         )
     except Exception as e:
         logger.warn("Failed run adhoc {}, {}".format(task_name, e))

apps/assets/tasks/const.py

@@ -64,7 +64,7 @@ GATHER_ASSET_USERS_TASKS = [
         "action": {
             "module": "shell",
             "args": "users=$(getent passwd | grep -v 'nologin' | "
-                    "grep -v 'shudown' | awk -F: '{ print $1 }');for i in $users;do last -F $i -1 | "
+                    "grep -v 'shudown' | awk -F: '{ print $1 }');for i in $users;do last -w -F $i -1 | "
                     "head -1 | grep -v '^$'  | awk '{ print $1\"@\"$3\"@\"$5,$6,$7,$8 }';done"
         }
     }

apps/assets/tasks/system_user_connectivity.py

@@ -31,7 +31,10 @@ def test_system_user_connectivity_util(system_user, assets, task_name):
     """
     from ops.utils import update_or_create_ansible_task
 
-    hosts = clean_ansible_task_hosts(assets, system_user=system_user)
+    # hosts = clean_ansible_task_hosts(assets, system_user=system_user)
+    # TODO: 这里不传递系统用户，因为clean_ansible_task_hosts会通过system_user来判断是否可以推送，
+    #  不符合测试可连接性逻辑， 后面需要优化此逻辑
+    hosts = clean_ansible_task_hosts(assets)
     if not hosts:
         return {}
     platform_hosts_map = {}

apps/authentication/api/auth.py

@@ -54,12 +54,3 @@ class UserConnectionTokenApi(RootOrgViewMixin, APIView):
             return Response(value)
         else:
             return Response({'user': value['user']})
-
-    def get_permissions(self):
-        if self.request.query_params.get('user-only', None):
-            self.permission_classes = (AllowAny,)
-        return super().get_permissions()
-
-
-
-

apps/authentication/backends/radius.py

@@ -11,7 +11,7 @@ User = get_user_model()
 
 
 class CreateUserMixin:
-    def get_django_user(self, username, password=None):
+    def get_django_user(self, username, password=None, *args, **kwargs):
         if isinstance(username, bytes):
             username = username.decode()
         try:
@@ -27,6 +27,12 @@ class CreateUserMixin:
             user.save()
         return user
 
+    def authenticate(self, *args, **kwargs):
+        # 校验用户时，会传入public_key参数，父类authentication中不接受public_key参数，所以要pop掉
+        # TODO:需要优化各backend的authenticate方法，django进行调用前会检测各authenticate的参数
+        kwargs.pop('public_key', None)
+        return super().authenticate(*args, **kwargs)
+
 
 class RadiusBackend(CreateUserMixin, RADIUSBackend):
     pass

apps/common/tasks.py

@@ -24,7 +24,7 @@ def send_mail_async(*args, **kwargs):
     """
     if len(args) == 3:
         args = list(args)
-        args[0] = settings.EMAIL_SUBJECT_PREFIX + args[0]
+        args[0] = (settings.EMAIL_SUBJECT_PREFIX or '') + args[0]
         email_from = settings.EMAIL_FROM or settings.EMAIL_HOST_USER
         args.insert(2, email_from)
         args = tuple(args)

apps/jumpserver/api.py

@@ -10,7 +10,7 @@ from users.models import User
 from assets.models import Asset
 from terminal.models import Session
 from orgs.utils import current_org
-from common.permissions import IsOrgAdmin
+from common.permissions import IsOrgAdmin, IsOrgAuditor
 from common.utils import lazyproperty
 
 __all__ = ['IndexApi']
@@ -224,7 +224,7 @@ class TotalCountMixin:
 
 
 class IndexApi(TotalCountMixin, WeekSessionMetricMixin, MonthLoginMetricMixin, APIView):
-    permission_classes = (IsOrgAdmin,)
+    permission_classes = [IsOrgAdmin | IsOrgAuditor]
     http_method_names = ['get']
 
     def get(self, request, *args, **kwargs):

apps/jumpserver/conf.py

@@ -240,7 +240,7 @@ class Config(dict):
         'HTTP_BIND_HOST': '0.0.0.0',
         'HTTP_LISTEN_PORT': 8080,
         'WS_LISTEN_PORT': 8070,
-        'LOGIN_LOG_KEEP_DAYS': 90,
+        'LOGIN_LOG_KEEP_DAYS': 9999,
         'TASK_LOG_KEEP_DAYS': 10,
         'ASSETS_PERM_CACHE_TIME': 3600 * 24,
         'SECURITY_MFA_VERIFY_TTL': 3600,

apps/ops/ws.py

@@ -15,7 +15,11 @@ class CeleryLogWebsocket(JsonWebsocketConsumer):
     disconnected = False
 
     def connect(self):
-        self.accept()
+        user = self.scope["user"]
+        if user.is_authenticated and user.is_org_admin:
+            self.accept()
+        else:
+            self.close()
 
     def receive(self, text_data=None, bytes_data=None, **kwargs):
         data = json.loads(text_data)

apps/perms/api/mixin.py

@@ -21,7 +21,7 @@ class UserPermissionMixin:
     obj = None
 
     def initial(self, *args, **kwargs):
-        super().initial(*args, *kwargs)
+        super().initial(*args, **kwargs)
         self.obj = self.get_obj()
 
     def get_obj(self):

apps/terminal/api/command.py

@@ -53,14 +53,15 @@ class CommandQueryMixin:
         q = self.request.query_params
         multi_command_storage = get_multi_command_storage()
         queryset = multi_command_storage.filter(
-            date_from=date_from, date_to=date_to, input=q.get("input"),
-            user=q.get("user"), asset=q.get("asset"),
-            system_user=q.get("system_user"),
+            date_from=date_from, date_to=date_to,
+            user=q.get("user"), asset=q.get("asset"), system_user=q.get("system_user"),
+            input=q.get("input"), session=q.get("session_id"),
             risk_level=self.get_query_risk_level(), org_id=self.get_org_id(),
         )
         return queryset
 
     def filter_queryset(self, queryset):
+        # 解决es存储命令时，父类根据filter_fields过滤出现异常的问题，返回的queryset类型list
         return queryset
 
     def get_filter_fields(self, request):

apps/terminal/backends/command/base.py

@@ -16,7 +16,7 @@ class CommandBase(object):
     @abc.abstractmethod
     def filter(self, date_from=None, date_to=None,
                user=None, asset=None, system_user=None,
-               input=None, session=None):
+               input=None, session=None, risk_level=None, org_id=None):
         pass
 
     @abc.abstractmethod

jms

@@ -217,7 +217,7 @@ def get_start_celery_ansible_kwargs():
 
 def get_start_celery_default_kwargs():
     print("\n- Start Celery as Distributed Task Queue: Celery")
-    return get_start_worker_kwargs('celery', 2)
+    return get_start_worker_kwargs('celery', 4)
 
 
 def get_start_worker_kwargs(queue, num):

requirements/requirements.txt

@@ -61,7 +61,7 @@ pytz==2018.3
 PyYAML==5.1
 redis==2.10.6
 requests==2.22.0
-jms-storage==0.0.29
+jms-storage==0.0.31
 s3transfer==0.3.3
 simplejson==3.13.2
 six==1.11.0