Merge pull request #1 from eduNEXT/cag/celery-queue

feat: allow to run multiple celery queues and workers

Author: Ian2012

.github/workflows/test.yml

@@ -0,0 +1,24 @@
+name: Run tests
+
+on:
+  pull_request:
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+         python-version: ['3.8', '3.12']
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+      - name: Install dependencies
+        run: |
+          pip install .[dev]
+      - name: Test lint, types, and format
+        run: make test

README.md

@@ -0,0 +1,66 @@
+# Celery plugin for [Tutor](https://docs.tutor.edly.io)
+
+A tutor plugin to extend the default LMS and CMS celery workers included in Tutor.
+It adds and configures extra deployments running LMS and CMS celery workers where
+every deployment will process async tasks routed to a specific queue. Having this
+workers separation per queue can help to define the scaling requirements for the Celery
+deployments, since having a single queue (the default one) with a single deployment can
+lead to unexpected behaviors when running large-scale sites.
+
+## Installation
+
+```shell
+pip install git+https://github.com/eduNEXT/tutor-contrib-celery
+```
+
+## Usage
+
+```shell
+tutor plugins enable celery
+```
+
+## Configuration
+
+### Celery queues
+
+By default, tutor-contrib-celery enables the following queues with independent deployments
+for each:
+
+- **CMS**: default, high, low (taken from CMS settings [here](https://github.com/openedx/edx-platform/blob/open-release/redwood.master/cms/envs/common.py#L1578-L1582))
+- **LMS**: default, high, high_mem (taken from LMS settings [here](https://github.com/openedx/edx-platform/blob/open-release/redwood.master/lms/envs/common.py#L2913-L2917))
+
+> [!NOTE]
+> We recommend using [tutor-contrib-pod-autoscaling](https://github.com/eduNEXT/tutor-contrib-pod-autoscaling)
+> to setup requested resources and limits.
+
+In case you are using different celery queues than the defaults from Open edX, you can
+extend the list by setting `CELERY_WORKER_VARIANTS` on your `config.yml`. The format is the following:
+
+```yaml
+CELERY_WORKER_VARIANTS:
+  lms:
+    - high
+    - high_mem
+    - lms_custom_queue
+  cms:
+    - high
+    - low
+    - cms_custom_queue
+```
+
+This plugin also provides a setting to directly route LMS/CMS tasks to an specific queue. It can extends/overrides
+the default `EXPLICIT_QUEUES` setting:
+
+```yaml
+CELERY_LMS_EXPLICIT_QUEUES:
+  lms.djangoapps.grades.tasks.compute_all_grades_for_course:
+    queue: edx.lms.core.high_mem
+CELERY_CMS_EXPLICIT_QUEUES:
+  cms.djangoapps.contentstore.tasks.import_olx:
+    queue: edx.cms.core.high
+```
+
+License
+*******
+
+This software is licensed under the terms of the AGPLv3.

README.rst

@@ -7,7 +7,7 @@
 
 
 def load_readme():
-    with io.open(os.path.join(HERE, "README.rst"), "rt", encoding="utf8") as f:
+    with io.open(os.path.join(HERE, "README.md"), "rt", encoding="utf8") as f:
         return f.read()
 
 
@@ -42,10 +42,10 @@ def load_about():
     packages=find_packages(exclude=["tests*"]),
     include_package_data=True,
     python_requires=">=3.7",
-    install_requires=["tutor>=17.0.0,<18.0.0"],
+    install_requires=["tutor>=18.0.0,<19.0.0"],
     extras_require={
         "dev": [
-            "tutor[dev]>=17.0.0,<18.0.0",
+            "tutor[dev]>=18.0.0,<19.0.0",
         ]
     },
     entry_points={

setup.py

@@ -1 +1 @@
-__version__ = "17.0.0"
+__version__ = "18.0.0"

tutorcelery/__about__.py

@@ -0,0 +1,54 @@
+{% for service, variants in CELERY_WORKER_VARIANTS.items() %}
+{% for variant in variants%}
+{% set deployment = service + "-" + "worker" + "-" + variant.replace("_", "-") %}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{deployment}}
+  labels:
+    app.kubernetes.io/name: {{deployment}}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{deployment}}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{deployment}}
+    spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+      containers:
+        - name: {{deployment}}
+          image: {{ DOCKER_IMAGE_OPENEDX }}
+          args:
+            - "celery"
+            - "--app={{service}}.celery"
+            - "worker"
+            - "--loglevel=info"
+            - "--hostname=edx.{{service}}.core.{{variant}}.%%h"
+            - "--max-tasks-per-child=100"
+            - "--queues=edx.{{service}}.core.{{variant}}"
+          env:
+          - name: SERVICE_VARIANT
+            value: {{service}}
+          - name: DJANGO_SETTINGS_MODULE
+            value: {{service}}.envs.tutor.production
+          volumeMounts:
+            - mountPath: /openedx/edx-platform/{{service}}/envs/tutor/
+              name: settings-{{service}}
+            - mountPath: /openedx/config
+              name: config
+          securityContext:
+            allowPrivilegeEscalation: false
+      volumes:
+        - name: settings-{{service}}
+          configMap:
+            name: openedx-settings-{{service}}
+        - name: config
+          configMap:
+            name: openedx-config
+{% endfor %}
+{% endfor %}

tutorcelery/patches/k8s-deployments

@@ -0,0 +1,20 @@
+{% for service in ["lms", "cms"] %}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{service}}-worker
+spec:
+  template:
+    spec:
+      containers:
+        - name: {{service}}-worker
+          args:
+            - "celery"
+            - "--app={{service}}.celery"
+            - "worker"
+            - "--loglevel=info"
+            - "--hostname=edx.{{service}}.core.default.%%h"
+            - "--max-tasks-per-child=100"
+            - "--queues=edx.{{service}}.core.default"
+{% endfor %}

tutorcelery/patches/k8s-override

@@ -0,0 +1,4 @@
+try:
+    EXPLICIT_QUEUES.update({{CELERY_CMS_EXPLICIT_QUEUES}})
+except NameError:
+    EXPLICIT_QUEUES = {{CELERY_CMS_EXPLICIT_QUEUES}}

tutorcelery/patches/openedx-cms-production-settings

@@ -0,0 +1,4 @@
+try:
+    EXPLICIT_QUEUES.update({{CELERY_LMS_EXPLICIT_QUEUES}})
+except NameError:
+    EXPLICIT_QUEUES = {{CELERY_LMS_EXPLICIT_QUEUES}}

tutorcelery/patches/openedx-lms-production-settings

@@ -19,6 +19,12 @@
         # Each new setting is a pair: (setting_name, default_value).
         # Prefix your setting names with 'CELERY_'.
         ("CELERY_VERSION", __version__),
+        (
+            "CELERY_WORKER_VARIANTS",
+            {"lms": ["high", "high_mem"], "cms": ["high", "low"]},
+        ),
+        ("CELERY_LMS_EXPLICIT_QUEUES", {}),
+        ("CELERY_CMS_EXPLICIT_QUEUES", {}),
     ]
 )