Release 1.9.0 (#776)

* Generic object added to Dataset model

* Migration file; nullable fields

* force change to filename_pattern field (previous migration failed to apply correctly)

* Fix migrations in tests

* added subjectTrials fixture

* Fix REST tests

* timezone.utc -> datetime.utcnow

* Fix test

* bump version number

* GitHub Actions generated requirements_frozen.txt

* rephrase password differences explitely

* GitHub Actions generated requirements_frozen.txt

---------

Co-authored-by: Miles Wells <k1o0@5tk.co>
Co-authored-by: github-actions <github-actions@github.com>

Author: 3 people

alyx/alyx/__init__.py

@@ -1 +1 @@
-VERSION = __version__ = '1.8.0'
+VERSION = __version__ = '1.9.0'

alyx/data/fixtures/data.datasettype.json

@@ -1747,5 +1747,16 @@
       "description": "A list of acquisition devices and behaviour protocols, along with the data and sync file location.  The root keys are (devices, sync, task, procedures).",
       "filename_pattern": "_*_experiment.description.*"
     }
+  },
+  {
+    "model": "data.datasettype",
+    "pk": "e0614a49-8f18-46b1-a4d9-0710a080fd8d",
+    "fields": {
+      "json": null,
+      "name": "subjectTrials.table",
+      "created_by": null,
+      "description": "All trials data for a given subject, contains the same columns as trials.table, plus \"session\", \"session_start_time\" and \"session_number\"",
+      "filename_pattern": ""
+    }
   }
 ]

alyx/data/migrations/0013_dataset_content_type_dataset_object_id.py

@@ -0,0 +1,48 @@
+# Generated by Django 4.1.5 on 2023-01-31 12:34
+
+from django.db import migrations, transaction, models
+import django.db.models.deletion
+
+
+def forwards(apps, _):
+    """Go through the datasets and assign the session field to the content_object field"""
+    Dataset = apps.get_model('data', 'Dataset')
+    with transaction.atomic():
+        for dataset in Dataset.objects.filter(session__isnull=False).iterator():
+            if dataset.content_object is None:
+                dataset.content_object = dataset.session
+                dataset.save()
+
+
+def backwards(apps, _):
+    Dataset = apps.get_model('data', 'Dataset')
+    with transaction.atomic():
+        for dataset in Dataset.objects.filter(session__isnull=False).iterator():
+            if dataset.content_object is not None:
+                dataset.content_object = None
+                dataset.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('data', '0012_alter_datasettype_filename_pattern_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='dataset',
+            name='content_type',
+            field=models.ForeignKey(null=True, blank=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='dataset',
+            name='object_id',
+            field=models.UUIDField(
+                null=True, blank=True, help_text='UUID, an object of content_type with this ID must already exist to attach a note.'),
+            preserve_default=False,
+        ),
+#        migrations.RunPython(forwards, backwards)
+    ]

alyx/data/migrations/0014_alter_datasettype_filename_pattern.py

@@ -0,0 +1,44 @@
+# Generated by Django 4.1.5 on 2023-02-01 15:32
+# NB: The previous migrations (0011 and 0012) somehow failed to set the filename_pattern field as
+# nullable. Migrations 0014 and 0015 reverse and remake this change which apparently fixed this
+# issue.
+
+import alyx.base
+from django.db import migrations, transaction
+
+PATTERN = '$$$'
+
+
+def fix_null_fields(apps, _):
+    """Populate null filename_pattern fields before making column not null"""
+    DatasetType = apps.get_model('data', 'DatasetType')
+    assert not DatasetType.objects.filter(filename_pattern__startswith=PATTERN).count()
+    with transaction.atomic():
+        for dtype in DatasetType.objects.filter(filename_pattern__isnull=True).iterator():
+            dtype.filename_pattern = PATTERN + dtype.name
+            dtype.save()
+
+
+def null_fields(apps, _):
+    """Reset previously null filename_pattern fields"""
+    DatasetType = apps.get_model('data', 'DatasetType')
+    with transaction.atomic():
+        for dtype in DatasetType.objects.filter(filename_pattern__startswith=PATTERN).iterator():
+            dtype.filename_pattern = None
+            dtype.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0013_dataset_content_type_dataset_object_id'),
+    ]
+
+    operations = [
+        migrations.RunPython(fix_null_fields, null_fields),
+        migrations.AlterField(
+            model_name='datasettype',
+            name='filename_pattern',
+            field=alyx.base.CharNullField(blank=True, help_text="File name pattern (with wildcards) for this file in ALF naming convention. E.g. 'spikes.times.*' or '*.timestamps.*', or 'spikes.*.*' for a DataCollection, which would include all files starting with the word 'spikes'. NB: Case-insensitive matching.If null, the name field must match the object.attribute part of the filename.", max_length=255, unique=True),
+        ),
+    ]

alyx/data/migrations/0015_alter_datasettype_filename_pattern.py

@@ -0,0 +1,44 @@
+# Generated by Django 4.1.5 on 2023-02-01 15:33
+# NB: The previous migrations (0011 and 0012) somehow failed to set the filname_pattern field as
+# nullable. Migrations 0014 and 0015 reverse and remake this change which apparently fixed this
+# issue.
+
+import alyx.base
+from django.db import migrations, transaction
+
+PATTERN = '$$$'
+
+
+def fix_null_fields(apps, _):
+    """Populate null filename_pattern fields before making column not null"""
+    DatasetType = apps.get_model('data', 'DatasetType')
+    assert not DatasetType.objects.filter(filename_pattern__startswith=PATTERN).count()
+    with transaction.atomic():
+        for dtype in DatasetType.objects.filter(filename_pattern__isnull=True).iterator():
+            dtype.filename_pattern = PATTERN + dtype.name
+            dtype.save()
+
+
+def null_fields(apps, _):
+    """Reset previously null filename_pattern fields"""
+    DatasetType = apps.get_model('data', 'DatasetType')
+    with transaction.atomic():
+        for dtype in DatasetType.objects.filter(filename_pattern__startswith=PATTERN).iterator():
+            dtype.filename_pattern = None
+            dtype.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0014_alter_datasettype_filename_pattern'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datasettype',
+            name='filename_pattern',
+            field=alyx.base.CharNullField(blank=True, help_text="File name pattern (with wildcards) for this file in ALF naming convention. E.g. 'spikes.times.*' or '*.timestamps.*', or 'spikes.*.*' for a DataCollection, which would include all files starting with the word 'spikes'. NB: Case-insensitive matching.If null, the name field must match the object.attribute part of the filename.", max_length=255, null=True, unique=True),
+        ),
+        migrations.RunPython(null_fields, fix_null_fields),
+    ]

alyx/data/models.py

@@ -3,6 +3,8 @@
 from django.core.validators import RegexValidator
 from django.db import models
 from django.utils import timezone
+from django.contrib.contenttypes.fields import GenericForeignKey
+from django.contrib.contenttypes.models import ContentType
 
 from alyx.settings import TIME_ZONE, AUTH_USER_MODEL
 from actions.models import Session
@@ -294,6 +296,13 @@ class Dataset(BaseExperimentalData):
     """
     objects = DatasetManager()
 
+    # Generic foreign key to arbitrary model instances allows polymorphic relationships
+    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, null=True, blank=True)
+    object_id = models.UUIDField(help_text="UUID, an object of content_type with this "
+                                           "ID must already exist to attach a note.",
+                                 null=True, blank=True)
+    content_object = GenericForeignKey()
+
     file_size = models.BigIntegerField(blank=True, null=True, help_text="Size in bytes")
 
     md5 = models.UUIDField(blank=True, null=True,
@@ -334,31 +343,19 @@ class Dataset(BaseExperimentalData):
     @property
     def is_online(self):
         fr = self.file_records.filter(data_repository__globus_is_personal=False)
-        if fr:
-            return all(fr.values_list('exists', flat=True))
-        else:
-            return False
+        return bool(fr.count() and all(fr.values_list('exists', flat=True)))
 
     @property
     def is_protected(self):
-        tags = self.tags.filter(protected=True)
-        if tags.count() > 0:
-            return True
-        else:
-            return False
+        return bool(self.tags.filter(protected=True).count())
 
     @property
     def is_public(self):
-        tags = self.tags.filter(public=True)
-        if tags.count() > 0:
-            return True
-        else:
-            return False
+        return bool(self.tags.filter(public=True).count())
 
     @property
     def data_url(self):
-        records = self.file_records.filter(data_repository__data_url__isnull=False,
-                                           exists=True)
+        records = self.file_records.filter(data_repository__data_url__isnull=False, exists=True)
         # returns preferentially globus non-personal endpoint
         if records:
             order_keys = ('data_repository__globus_is_personal', '-data_repository__name')

alyx/data/tests.py

@@ -4,16 +4,26 @@
 from django.db.models import ProtectedError
 
 from data.models import Dataset, DatasetType, Tag
+from subjects.models import Subject
+from misc.models import Lab
 from data.transfers import get_dataset_type
 
 
 class TestModel(TestCase):
     def test_model_methods(self):
         (dset, _) = Dataset.objects.get_or_create(name='toto.npy')
 
-        assert dset.is_online is False
-        assert dset.is_public is False
-        assert dset.is_protected is False
+        self.assertIs(dset.is_online, False)
+        self.assertIs(dset.is_public, False)
+        self.assertIs(dset.is_protected, False)
+
+    def test_generic_foreign_key(self):
+        # Attempt to associate a dataset with a subject
+        self.lab = Lab.objects.create(name='test_lab')
+        subj = Subject.objects.create(nickname='foo', birth_date='2018-09-01', lab=self.lab)
+        dset = Dataset(name='toto.npy', content_object=subj)
+
+        self.assertIs(dset.content_object, subj)
 
     def test_delete(self):
         (dset, _) = Dataset.objects.get_or_create(name='foo.npy')

alyx/data/tests_rest.py

@@ -1,5 +1,5 @@
 import datetime
-import os.path as op
+from pathlib import PurePosixPath
 import uuid
 
 from django.contrib.auth import get_user_model
@@ -372,11 +372,11 @@ def _assert_registration(self, r, data):
 
         self.assertEqual(d0['file_records'][0]['data_repository'], 'dr')
         self.assertEqual(d0['file_records'][0]['relative_path'],
-                         op.join(data['path'], 'a.b.e1'))
+                         PurePosixPath(data['path'], 'a.b.e1').as_posix())
 
         self.assertEqual(d1['file_records'][0]['data_repository'], 'dr')
         self.assertEqual(d1['file_records'][0]['relative_path'],
-                         op.join(data['path'], 'a.c.e2'))
+                         PurePosixPath(data['path'], 'a.c.e2').as_posix())
 
     def test_register_existence_options(self):
 
@@ -509,8 +509,8 @@ def test_register_with_revision(self):
         self.assertTrue(not r['revision'])
         self.assertEqual(r['collection'], 'dir')
         # Check the revision relative path doesn't exist
-        self.assertTrue(r['file_records'][0]['relative_path'] ==
-                        op.join(data['path'], data['filenames']))
+        self.assertEqual(r['file_records'][0]['relative_path'],
+                         PurePosixPath(data['path'], data['filenames']).as_posix())
 
         # Now test specifying a revision in path
         data = {'path': '%s/2018-01-01/002/dir/#v1#' % self.subject,
@@ -523,7 +523,7 @@ def test_register_with_revision(self):
         self.assertTrue(r['revision'] == 'v1')
         self.assertEqual('dir', r['collection'])
         # Check file record relative path includes revision
-        self.assertTrue('#v1#' in r['file_records'][0]['relative_path'])
+        self.assertIn('#v1#', r['file_records'][0]['relative_path'])
 
         # Now test specifying a collection and a revision in filename
         data = {'path': '%s/2018-01-01/002/dir' % self.subject,
@@ -535,7 +535,7 @@ def test_register_with_revision(self):
         self.assertTrue(r['revision'] == 'v1')
         self.assertTrue(r['collection'] == 'dir/dir1')
         # Check file record relative path includes revision
-        self.assertTrue('#v1#' in r['file_records'][0]['relative_path'])
+        self.assertIn('#v1#', r['file_records'][0]['relative_path'])
 
         # Test that giving nested revision folders gives out an error
         data = {'path': '%s/2018-01-01/002/dir' % self.subject,

alyx/data/transfers.py

@@ -4,7 +4,7 @@
 import os.path as op
 import re
 import time
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 
 from django.db.models import Case, When, Count, Q, F
 import globus_sdk
@@ -88,7 +88,7 @@ def _get_absolute_path(file_record):
         path2 = path2[6:]
     if path2.startswith('/'):
         path2 = path2[1:]
-    path = op.join(path1, path2)
+    path = PurePosixPath(path1, path2).as_posix()
     return path
 
 
@@ -248,7 +248,7 @@ def _create_dataset_file_records(
 
     assert session is not None
     revision_name = f'#{revision.name}#' if revision else ''
-    relative_path = op.join(rel_dir_path, collection or '', revision_name, filename)
+    relative_path = PurePosixPath(rel_dir_path, collection or '', revision_name, filename)
     dataset_type = get_dataset_type(filename, DatasetType.objects.all())
     data_format = get_data_format(filename)
     assert dataset_type
@@ -261,8 +261,9 @@ def _create_dataset_file_records(
 
     # Get or create the dataset.
     dataset, is_new = Dataset.objects.get_or_create(
-        collection=collection, name=filename, session=session,
-        dataset_type=dataset_type, data_format=data_format, revision=revision)
+        collection=collection, name=filename, session=session,  # content_object=session,
+        dataset_type=dataset_type, data_format=data_format, revision=revision
+    )
     dataset.default_dataset = default is True
     dataset.save()
 
@@ -303,7 +304,7 @@ def _create_dataset_file_records(
         exists = repo in exists_in
         # Do not create a new file record if it already exists.
         fr, is_new = FileRecord.objects.get_or_create(
-            dataset=dataset, data_repository=repo, relative_path=relative_path)
+            dataset=dataset, data_repository=repo, relative_path=relative_path.as_posix())
         if is_new or is_patched:
             fr.exists = exists
             fr.json = None  # this is important if a dataset is patched during an ongoing transfer

alyx/subjects/models.py

@@ -263,7 +263,7 @@ def nicknamesafe(self):
     def age_days(self):
         if (self.death_date is None and self.birth_date is not None):
             # subject still alive
-            age = datetime.now(timezone.utc).date() - self.birth_date
+            age = datetime.utcnow().date() - self.birth_date
         elif (self.death_date is not None and self.birth_date is not None):
             # subject is dead
             age = self.death_date - self.birth_date