Skip to content

Commit 4185176

Browse files
GhesselinkGhesselink
committed
include release v0.8.1
2 parents 922f7c1 + 00b9df8 commit 4185176

15 files changed

Lines changed: 889 additions & 120 deletions

File tree

‎.github/workflows/playwright.yml‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
name: Playwright Tests
2+
on:
3+
push:
4+
branches: development
5+
pull_request:
6+
branches: development
7+
8+
permissions:
9+
contents: read
10+
actions: read
11+
checks: write
12+
pull-requests: write
13+
14+
jobs:
15+
playwright:
16+
name: 'Playwright Tests'
17+
runs-on: ubuntu-latest
18+
container:
19+
image: mcr.microsoft.com/playwright:v1.55.0-noble
20+
env:
21+
DJANGO_DB: postgresql
22+
POSTGRES_DB: postgres
23+
POSTGRES_HOST: postgres
24+
POSTGRES_NAME: postgres
25+
POSTGRES_PASSWORD: postgres
26+
POSTGRES_PORT: 5432
27+
MEDIA_ROOT: /tmp/files_storage
28+
CELERY_BROKER_URL: redis://redis:6379/0
29+
CELERY_RESULT_BACKEND: django-db
30+
CELERY_TASK_ALWAYS_EAGER: "false"
31+
CELERY_TASK_EAGER_PROPAGATES: "false"
32+
DJANGO_ALLOWED_HOSTS: "localhost 127.0.0.1"
33+
DJANGO_TRUSTED_ORIGINS: "http://localhost:3000 http://127.0.0.1:3000 http://localhost:8000 http://127.0.0.1:8000"
34+
POSTGRES_USER: postgres
35+
VIRTUAL_ENV: backend/.dev/venv
36+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
37+
38+
services:
39+
postgres:
40+
image: postgres:16.8-alpine
41+
env:
42+
POSTGRES_DB: postgres
43+
POSTGRES_USER: postgres
44+
POSTGRES_PASSWORD: postgres
45+
options: >-
46+
--health-cmd pg_isready
47+
--health-interval 10s
48+
--health-timeout 5s
49+
--health-retries 5
50+
--network-alias postgres
51+
ports:
52+
- 5432:5432
53+
redis:
54+
image: redis:7.2-alpine
55+
options: >-
56+
--health-cmd "redis-cli ping"
57+
--health-interval 10s
58+
--health-timeout 5s
59+
--health-retries 5
60+
ports:
61+
- 6379:6379
62+
volumes:
63+
- redis_data:/data
64+
65+
steps:
66+
- name: Checkout repository
67+
uses: actions/checkout@v4
68+
with:
69+
submodules: recursive
70+
71+
- name: Prepare MEDIA_ROOT
72+
run: |
73+
mkdir -p /tmp/files_storage
74+
chmod -R 777 /tmp/files_storage
75+
76+
- name: Setup Node.js
77+
uses: actions/setup-node@v4
78+
with:
79+
node-version: lts/*
80+
81+
- name: Setup Python
82+
uses: actions/setup-python@v5
83+
with:
84+
python-version: '3.11'
85+
86+
- name: Install system dependencies
87+
run: |
88+
apt-get update
89+
apt-get install -y make wget unzip libpq-dev
90+
91+
- name: Cache backend dependencies
92+
uses: actions/cache@v4
93+
with:
94+
path: backend/.dev/venv
95+
key: ${{ runner.os }}-backend-${{ hashFiles('backend/**/requirements.txt') }}
96+
restore-keys: |
97+
${{ runner.os }}-backend-
98+
99+
- name: Install backend dependencies
100+
run: |
101+
cd backend
102+
test -d .dev/venv || python3.11 -m venv .dev/venv
103+
.dev/venv/bin/pip install --upgrade pip
104+
find . -name 'requirements.txt' -exec .dev/venv/bin/pip install -r {} \;
105+
- name: Download and install ifcopenshell
106+
run: |
107+
wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.4-6924012-linux64.zip"
108+
mkdir -p .dev/venv/lib/python3.11/site-packages
109+
unzip -o -d .dev/venv/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip
110+
rm /tmp/ifcopenshell_python.zip
111+
# Verify installation
112+
ls -la .dev/venv/lib/python3.11/site-packages/
113+
echo "Checking for ifcopenshell installation:"
114+
find .dev/venv/lib/python3.11/site-packages -name "*ifcopenshell*" || echo "No ifcopenshell files found"
115+
working-directory: ./backend
116+
117+
- name: Cache frontend dependencies
118+
uses: actions/cache@v4
119+
with:
120+
path: frontend/node_modules
121+
key: ${{ runner.os }}-frontend-${{ hashFiles('frontend/package-lock.json') }}
122+
restore-keys: |
123+
${{ runner.os }}-frontend-
124+
125+
- name: Install frontend dependencies (required for webServer)
126+
run: npm install
127+
working-directory: ./frontend
128+
129+
- name: Cache e2e dependencies
130+
uses: actions/cache@v4
131+
with:
132+
path: e2e/node_modules
133+
key: ${{ runner.os }}-e2e-${{ hashFiles('e2e/package-lock.json') }}
134+
restore-keys: |
135+
${{ runner.os }}-e2e-
136+
137+
- name: Install e2e dependencies (replicating make e2e-test)
138+
run: npm install
139+
working-directory: ./e2e
140+
141+
- name: Cache Playwright browsers
142+
uses: actions/cache@v4
143+
with:
144+
path: ~/.cache/ms-playwright
145+
key: ${{ runner.os }}-playwright-${{ hashFiles('e2e/package-lock.json') }}
146+
restore-keys: |
147+
${{ runner.os }}-playwright-
148+
149+
- name: Install Playwright browsers
150+
run: npm run install-playwright
151+
working-directory: ./e2e
152+
153+
- name: Setup Django database, migrations, and superusers
154+
run: |
155+
cd backend
156+
.dev/venv/bin/python manage.py makemigrations
157+
.dev/venv/bin/python manage.py migrate
158+
.dev/venv/bin/python manage.py collectstatic --noinput
159+
DJANGO_SUPERUSER_USERNAME=root DJANGO_SUPERUSER_PASSWORD=root DJANGO_SUPERUSER_EMAIL=root@localhost .dev/venv/bin/python manage.py createsuperuser --noinput
160+
DJANGO_SUPERUSER_USERNAME=SYSTEM DJANGO_SUPERUSER_PASSWORD=system DJANGO_SUPERUSER_EMAIL=system@localhost .dev/venv/bin/python manage.py createsuperuser --noinput
161+
.dev/venv/bin/python manage.py seed_dummy_requests
162+
env:
163+
DJANGO_DB: postgresql
164+
POSTGRES_HOST: postgres
165+
POSTGRES_NAME: postgres
166+
POSTGRES_USER: postgres
167+
POSTGRES_PASSWORD: postgres
168+
POSTGRES_PORT: 5432
169+
VIRTUAL_ENV: backend/.dev/venv
170+
171+
- name: Run Playwright tests
172+
run: npx playwright test --workers="$(node -p "Math.max(1, Math.min(Math.floor((require('os').cpus()?.length||2)*0.75), 8))")"
173+
working-directory: ./e2e
174+
env:
175+
DJANGO_DB: postgresql
176+
POSTGRES_HOST: postgres
177+
POSTGRES_NAME: postgres
178+
POSTGRES_USER: postgres
179+
POSTGRES_PASSWORD: postgres
180+
POSTGRES_PORT: 5432
181+
182+
- name: Upload test results
183+
uses: actions/upload-artifact@v4
184+
if: always()
185+
with:
186+
name: playwright-report
187+
path: e2e/test-results/
188+
retention-days: 7

‎backend/Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,12 @@ test-syntax-task:
8383
test-schema-task:
8484
MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_schema_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
8585

86+
archive-dry-run:
87+
$(PYTHON) manage.py archive_requests --days 180 --all --dry-run
88+
89+
archive:
90+
$(PYTHON) manage.py archive_requests --days 180 --all --confirm
91+
8692
clean:
8793
rm -rf .dev
8894
rm -rf django_db.sqlite3

‎backend/apps/ifc_validation/management/commands/archive_requests.py‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
import os
2+
import gzip
3+
import shutil
4+
from datetime import timedelta
5+
6+
from django.core.management.base import BaseCommand
7+
from django.utils import timezone
8+
from django.db import transaction
9+
10+
from apps.ifc_validation_models.models import ValidationRequest
11+
from apps.ifc_validation.tasks.utils import get_absolute_file_path
12+
from apps.ifc_validation_models.decorators import requires_django_user_context
13+
from core.utils import format_human_readable_file_size
14+
15+
class Command(BaseCommand):
16+
17+
help = 'Archive ValidationRequest files matching certain pruning criteria (age, deletion status). Compresses *.ifc files to *.ifc.gz and updates database records accordingly.'
18+
19+
def add_arguments(self, parser):
20+
21+
# how many days to look back (default: 180)
22+
parser.add_argument(
23+
'--days', '-d',
24+
type=int,
25+
default=180,
26+
help='Number of days to look back for old Validation Requests (default: 180).'
27+
)
28+
29+
# whether to restrict to deleted requests only (default) or include non-deleted as well
30+
deleted_group = parser.add_mutually_exclusive_group()
31+
deleted_group.add_argument(
32+
'--deleted-only', '--deleted',
33+
dest='deleted_only',
34+
action='store_true',
35+
help='Archive only deleted Validation Requests (default).'
36+
)
37+
deleted_group.add_argument(
38+
'--include-non-deleted', '--all',
39+
dest='deleted_only',
40+
action='store_false',
41+
help='Include non-deleted Validation Requests as well.'
42+
)
43+
parser.set_defaults(deleted_only=True)
44+
45+
# dry-run mode: perform a simulation, do not modify any files or database records
46+
# just logs intended actions and outcomes to stdout
47+
dry_group = parser.add_mutually_exclusive_group()
48+
dry_group.add_argument(
49+
'--dry-run', '--simulate', '--recon',
50+
dest='dry_run',
51+
action='store_true',
52+
help='Dry run (default): show what would be archived without changing files or database records.'
53+
)
54+
dry_group.add_argument(
55+
'--confirm', '--apply',
56+
dest='dry_run',
57+
action='store_false',
58+
help='Confirm archiving: apply file archving and database record changes.'
59+
)
60+
parser.set_defaults(dry_run=True)
61+
62+
@requires_django_user_context
63+
def handle(self, *args, **options):
64+
days = options['days']
65+
deleted_only = options['deleted_only']
66+
dry_run = options['dry_run']
67+
68+
cutoff_date = timezone.now() - timedelta(days=days)
69+
70+
# query by age, deleted flag and file name ending with .ifc
71+
qs = ValidationRequest.objects.filter(created__lt=cutoff_date, file__iendswith='.ifc')
72+
73+
if deleted_only:
74+
qs = qs.filter(deleted=True)
75+
76+
total = qs.count()
77+
self.stdout.write(f"Found {total} Validation Request(s) older than {days} day(s){' (deleted only)' if deleted_only else ''}.")
78+
if dry_run:
79+
self.stdout.write(self.style.WARNING("NOTE: Running in DRY-RUN mode. No changes will be made. Use --confirm to apply changes."))
80+
81+
archived = 0
82+
skipped = 0
83+
total_savings = 0 # in MB
84+
85+
for request in qs.iterator():
86+
87+
# validate presence of file
88+
try:
89+
file_path = get_absolute_file_path(request.file.name)
90+
except FileNotFoundError:
91+
self.stdout.write(f"WARNING: File not found for Validation Request with id={request.id} - skipping...")
92+
skipped += 1
93+
continue
94+
95+
gz_filename = file_path + '.gz'
96+
gz_filename_only = request.file.name + '.gz'
97+
98+
# only report what would happen
99+
if dry_run:
100+
self.stdout.write(f"[DRY-RUN] Would archive and update ValidationRequest with id={request.id} from {request.file.name} to {gz_filename_only}")
101+
archived += 1
102+
total_savings += os.path.getsize(file_path) * 0.8
103+
continue
104+
105+
# create gzip archive
106+
total_savings += os.path.getsize(file_path)
107+
with open(file_path, 'rb') as f_in, gzip.open(gz_filename, 'wb') as f_out:
108+
shutil.copyfileobj(f_in, f_out)
109+
total_savings -= os.path.getsize(gz_filename)
110+
111+
# update database and remove original file
112+
try:
113+
with transaction.atomic():
114+
request.file.name = gz_filename_only
115+
request.save(update_fields=['file'])
116+
try:
117+
os.remove(file_path)
118+
except OSError as e:
119+
# Raise to trigger transaction rollback
120+
raise RuntimeError(f"Failed to remove original file: {e}")
121+
except Exception as e:
122+
# Ensure DB not updated and clean up the created gzip to keep state unchanged
123+
try:
124+
os.remove(gz_filename)
125+
except Exception:
126+
pass
127+
skipped += 1
128+
self.stdout.write(self.style.ERROR(f"Failed to archive Validation Request with id={request.id}: {e} - rolling back changes..."))
129+
continue
130+
131+
archived += 1
132+
self.stdout.write(f"Archived and updated Validation Request with id={request.id}: {gz_filename_only}")
133+
134+
# show summary
135+
total_savings = format_human_readable_file_size(total_savings)
136+
if dry_run:
137+
self.stdout.write(self.style.WARNING(f"DRY-RUN would have archived {archived}, skipped {skipped}, total considered {total}."))
138+
self.stdout.write(self.style.WARNING(f"DRY-RUN would free up approx. {total_savings} (compression ratio of 80%)."))
139+
else:
140+
self.stdout.write(self.style.SUCCESS(f"Archived {archived}, skipped {skipped}, total considered {total}."))
141+
self.stdout.write(self.style.SUCCESS(f"Freed up {total_savings}."))

‎backend/apps/ifc_validation/management/commands/seed_dummy_requests.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
from django.core.files.base import ContentFile
2+
from django.core.management.base import BaseCommand
3+
from django.contrib.auth.models import User
4+
5+
from apps.ifc_validation_models.models import set_user_context, Company, AuthoringTool, Model, ValidationRequest
6+
7+
class Command(BaseCommand):
8+
help = "Seed 51 dummy ValidationRequest rows for pagination tests"
9+
10+
def handle(self, *args, **opts):
11+
user, _ = User.objects.get_or_create(
12+
username="root",
13+
defaults={"is_staff": True, "is_superuser": True}
14+
)
15+
user.set_password("root")
16+
user.save()
17+
set_user_context(user)
18+
19+
company, _ = Company.objects.get_or_create(name="DummyCorp")
20+
tool, _ = AuthoringTool.objects.get_or_create(company=company, name="DummyTool", version="1.0")
21+
22+
payload = b"ISO-10303-21;\nEND-ISO-10303-21;"
23+
24+
for i in range(51):
25+
file_name = f"dummy_{i:03d}.ifc"
26+
27+
m = Model(
28+
produced_by=tool,
29+
file_name=file_name,
30+
file=f"uploads/{file_name}",
31+
size=len(payload),
32+
uploaded_by=user,
33+
schema="IFC4X3_ADD2",
34+
license=Model.License.UNKNOWN,
35+
)
36+
m.save()
37+
38+
vr = ValidationRequest(
39+
file_name=file_name,
40+
size=len(payload),
41+
status=ValidationRequest.Status.PENDING,
42+
model=m,
43+
channel=ValidationRequest.Channel.API,
44+
)
45+
vr.file.save(file_name, ContentFile(payload), save=False)
46+
vr.save()

0 commit comments

Comments
 (0)