Skip to content
This repository was archived by the owner on Aug 22, 2022. It is now read-only.

Commit 1ffa6de

Browse files
authored
Merge pull request #856 from open-craft/gabor/add-grove
[SE-5023] Integrate with Grove
2 parents ff6286e + 4d9813f commit 1ffa6de

64 files changed

Lines changed: 1972 additions & 159 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

Makefile

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
API_CLIENT_PATH := frontend/packages/api_client
2323
.DEFAULT_GOAL := help
2424
HELP_SPACING ?= 30
25-
COVERAGE_THRESHOLD ?= 90
25+
# TODO: @gabor-boros set back to 90 after we are completely on Grove
26+
COVERAGE_THRESHOLD ?= 89
2627
WORKERS ?= 3
2728
WORKERS_LOW_PRIORITY ?= 3
2829
SHELL ?= /bin/bash
@@ -66,6 +67,7 @@ install_system_db_dependencies: apt_get_update ## Install system-level DB depend
6667

6768
install_system_dependencies: apt_get_update ## Install system-level dependencies from `debian_packages.lst`. Ignores comments.
6869
sudo -E apt-get install -y `grep -v '^#' debian_packages.lst | tr -d '\r'`
70+
curl https://sh.rustup.rs -sSf | sh -s -- -y
6971
if [ -z $$CI ] ; then \
7072
echo "Installing Firefox because we're not in a CI."; \
7173
sudo apt-get install -y libgtk3.0-cil-dev libasound2 libasound2 libdbus-glib-1-2 libdbus-1-3 --no-install-recommends; \

api/router.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from rest_framework import routers
2626

2727
from email_verification.api.v2.views import VerifyEmailViewset
28+
from grove.api.v1.views import GroveDeploymentAPIView
2829
from instance.api.appserver import StatusViewSet
2930
from instance.api.deployment import DeploymentTypeViewSet
3031
from instance.api.instance import InstanceViewSet, InstanceTagViewSet
@@ -45,6 +46,7 @@
4546

4647
v1_router.register(r'status', StatusViewSet, basename='status')
4748
v1_router.register(r'deployment_type', DeploymentTypeViewSet, basename='deployment_type')
49+
v1_router.register(r'grove/deployments', GroveDeploymentAPIView, basename='grove-deployments')
4850
v1_router.register(r'instance', InstanceViewSet, basename='instance')
4951
v1_router.register(r'instance_tag', InstanceTagViewSet, basename='instance_tag')
5052
v1_router.register(r'openedx_appserver', OpenEdXAppServerViewSet)

api/urls.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
from api.auth import JWTAuthToken, JwtTokenRefresh, JwtTokenVerify
3131
from api.router import v1_router, v2_router
3232
from opencraft.swagger import api_info
33+
from grove.api.v1.views import gitlab_webhook
3334

3435

3536
# URL Patterns ################################################################
@@ -61,4 +62,5 @@
6162
url(r'^swagger(?P<format>\.json|\.yaml)$', schema_view.without_ui(cache_timeout=10), name='schema-json'),
6263
url(r'^swagger/$', schema_view.with_ui('swagger', cache_timeout=10), name='schema-swagger-ui'),
6364
url(r'^redoc/$', schema_view.with_ui('redoc', cache_timeout=10), name='schema-redoc'),
65+
url(r'^webhook/$', gitlab_webhook, name='grove_webhook'),
6466
]

docker-compose.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ services:
4747
- mongodb
4848
env_file: .env
4949
ports:
50-
- "127.0.0.1:5000:5000"
50+
- "5000:5000"
5151
volumes:
5252
- ./:/usr/src/ocim/
5353

@@ -57,7 +57,7 @@ services:
5757
depends_on:
5858
- ocim
5959
ports:
60-
- "127.0.0.1:3000:3000"
60+
- "3000:3000"
6161
tty: true
6262
volumes:
6363
- ./frontend:/usr/src/ocim-frontend/

grove/__init__.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# OpenCraft -- tools to aid developing and hosting free software projects
4+
# Copyright (C) 2015-2021 OpenCraft <contact@opencraft.com>
5+
#
6+
# This program is free software: you can redistribute it and/or modify
7+
# it under the terms of the GNU Affero General Public License as
8+
# published by the Free Software Foundation, either version 3 of the
9+
# License, or (at your option) any later version.
10+
#
11+
# This program is distributed in the hope that it will be useful,
12+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
# GNU Affero General Public License for more details.
15+
#
16+
# You should have received a copy of the GNU Affero General Public License
17+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
18+
"""
19+
Grove app module.
20+
"""
21+
22+
default_app_config = 'grove.apps.GroveAppConfig'

grove/admin.py

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# OpenCraft -- tools to aid developing and hosting free software projects
4+
# Copyright (C) 2015-2021 OpenCraft <contact@opencraft.com>
5+
#
6+
# This program is free software: you can redistribute it and/or modify
7+
# it under the terms of the GNU Affero General Public License as
8+
# published by the Free Software Foundation, either version 3 of the
9+
# License, or (at your option) any later version.
10+
#
11+
# This program is distributed in the hope that it will be useful,
12+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
# GNU Affero General Public License for more details.
15+
#
16+
# You should have received a copy of the GNU Affero General Public License
17+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
18+
"""
19+
The admin site registration for the Grove app.
20+
"""
21+
22+
from django.contrib import admin
23+
24+
from grove.models.deployment import GroveDeployment
25+
from grove.models.instance import GroveInstance
26+
from grove.models.gitlabpipeline import GitlabPipeline
27+
from grove.models.repository import GroveClusterRepository
28+
from instance.admin import InlineInstanceReferenceAdmin
29+
30+
31+
class GitlabPipelineAdmin(admin.ModelAdmin): # pylint: disable=missing-docstring
32+
list_display = ("pipeline_id", "instance", "status")
33+
list_filter = ("status",)
34+
search_fields = ("instance__name",)
35+
36+
37+
class GroveClusterRepositoryAdmin(admin.ModelAdmin): # pylint: disable=missing-docstring
38+
list_display = ("name", "project_id", "unleash_instance_id", "git_ref",)
39+
list_filter = ("project_id",)
40+
search_fields = ("name",)
41+
42+
43+
class GroveDeploymentAdmin(admin.ModelAdmin): # pylint: disable=missing-docstring
44+
list_display = ("instance", "creator", "type", "has_overrides", "created")
45+
list_filter = ("creator", "type", "created")
46+
47+
def has_overrides(self, obj):
48+
"""
49+
Return whether deployment includes any changes.
50+
"""
51+
return bool(obj.overrides)
52+
53+
54+
class GroveInstanceAdmin(admin.ModelAdmin): # pylint: disable=missing-docstring
55+
list_display = ('repository', 'name', 'created', 'modified',)
56+
search_fields = ('internal_lms_domain',)
57+
inlines = (
58+
InlineInstanceReferenceAdmin,
59+
)
60+
61+
def get_inline_instances(self, request, obj=None):
62+
"""
63+
Hides inlines while creating ``GroveInstance``.
64+
"""
65+
# Doesn't show the inline instance for new objects since we have custom
66+
# logic for creating InstanceReference objects
67+
if obj is None or obj.pk is None:
68+
return []
69+
return super().get_inline_instances(request, obj)
70+
71+
72+
admin.site.register(GitlabPipeline, GitlabPipelineAdmin)
73+
admin.site.register(GroveClusterRepository, GroveClusterRepositoryAdmin)
74+
admin.site.register(GroveDeployment, GroveDeploymentAdmin)
75+
admin.site.register(GroveInstance, GroveInstanceAdmin)

grove/api/__init__.py

Whitespace-only changes.

grove/api/v1/__init__.py

Whitespace-only changes.

grove/api/v1/views.py

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# OpenCraft -- tools to aid developing and hosting free software projects
4+
# Copyright (C) 2015-2021 OpenCraft <contact@opencraft.com>
5+
#
6+
# This program is free software: you can redistribute it and/or modify
7+
# it under the terms of the GNU Affero General Public License as
8+
# published by the Free Software Foundation, either version 3 of the
9+
# License, or (at your option) any later version.
10+
#
11+
# This program is distributed in the hope that it will be useful,
12+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
# GNU Affero General Public License for more details.
15+
#
16+
# You should have received a copy of the GNU Affero General Public License
17+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
18+
"""
19+
API views for the Grove app.
20+
"""
21+
22+
import json
23+
import re
24+
from django.http import HttpResponse
25+
from django.views.decorators.csrf import csrf_exempt
26+
from django.views.decorators.http import require_POST
27+
from rest_framework.viewsets import GenericViewSet
28+
from rest_framework.mixins import RetrieveModelMixin, ListModelMixin, CreateModelMixin
29+
from rest_framework.permissions import IsAuthenticated
30+
31+
from grove.models.deployment import GroveDeployment
32+
from grove.models.gitlabpipeline import GitlabPipeline
33+
from grove.serializers import GroveDeploymentSerializer
34+
from instance.models.instance import InstanceReference
35+
from opencraft.swagger import viewset_swagger_helper
36+
37+
38+
@viewset_swagger_helper(
39+
retrieve="Return a deployment",
40+
list="Return all deployments",
41+
create="Trigger new deployment in Grove for an existing instance",
42+
public_actions=["create"],
43+
)
44+
class GroveDeploymentAPIView(RetrieveModelMixin, ListModelMixin, CreateModelMixin, GenericViewSet):
45+
"""
46+
GroveDeploymentAPI is used to create new and monitor existing deployments.
47+
48+
When an instance is created by a new customer or a deployment is needed to
49+
update the existing instance configuration, a new Deployment object is
50+
created. On creation GitLab will be notified and the appropriate GitLab
51+
pipeline is called to create a new or update an existing instance.
52+
"""
53+
54+
serializer_class = GroveDeploymentSerializer
55+
permission_classes = [IsAuthenticated]
56+
57+
def get_queryset(self):
58+
"""
59+
Filter the queryset based on user staff state.
60+
61+
If the user is a staff user, return all deployments. Otherwise, return
62+
only those deployments that are related to the given user.
63+
"""
64+
65+
user = self.request.user
66+
67+
if user.is_staff:
68+
return GroveDeployment.objects.all()
69+
70+
return GroveDeployment.objects.filter(instance__creator__user=user)
71+
72+
73+
def is_deployment_pipeline(body):
74+
"""
75+
Check if the pipeline is related to instance deployment
76+
"""
77+
return body.get('commit', '') and "deployment" in body['commit']['title']
78+
79+
80+
def get_instance(body):
81+
"""
82+
Fetch instance related to deployment pipeline
83+
"""
84+
instance_name = re.findall(r"deployment\/(\w+)\/", body['commit']['title'])
85+
if not instance_name:
86+
return None
87+
88+
return InstanceReference.objects.get(name=instance_name[0])
89+
90+
91+
@require_POST
92+
@csrf_exempt
93+
def gitlab_webhook(request):
94+
"""
95+
Webhook endpoint for gitlab pipeline
96+
"""
97+
payload = json.loads(request.body.decode('utf-8'))
98+
if is_deployment_pipeline(payload):
99+
pipeline_instance = get_instance(payload)
100+
if pipeline_instance:
101+
pipeline_id = payload['object_attributes']['id']
102+
if GitlabPipeline.objects.filter(pipeline_id=pipeline_id).exists():
103+
# pipeline already exists, call status update method
104+
pipeline_obj = GitlabPipeline.objects.get(pipeline_id=pipeline_id)
105+
new_status = payload['object_attributes']['status']
106+
pipeline_obj.update_status(new_status)
107+
pipeline_obj.save()
108+
else:
109+
# create new pipeline object for the deployment object
110+
deployment = pipeline_instance.instance.get_latest_deployment()
111+
new_pipeline = GitlabPipeline.objects.create(pipeline_id=pipeline_id, instance=pipeline_instance)
112+
deployment.pipeline = new_pipeline
113+
deployment.save()
114+
115+
return HttpResponse('Webhook received', status=200)

grove/apps.py

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# OpenCraft -- tools to aid developing and hosting free software projects
4+
# Copyright (C) 2015-2021 OpenCraft <contact@opencraft.com>
5+
#
6+
# This program is free software: you can redistribute it and/or modify
7+
# it under the terms of the GNU Affero General Public License as
8+
# published by the Free Software Foundation, either version 3 of the
9+
# License, or (at your option) any later version.
10+
#
11+
# This program is distributed in the hope that it will be useful,
12+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
# GNU Affero General Public License for more details.
15+
#
16+
# You should have received a copy of the GNU Affero General Public License
17+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
18+
"""
19+
Application configuration for the Grove app.
20+
"""
21+
22+
from django.apps import AppConfig
23+
24+
25+
class GroveAppConfig(AppConfig):
26+
"""
27+
Application configuration for the Grove app.
28+
"""
29+
name = "grove"
30+
31+
def ready(self):
32+
"""
33+
Connect signal handlers when the app is ready.
34+
"""

0 commit comments

Comments
 (0)