Merge pull request #135 from EGA-archive/schema-versions

New release v2.0-1d0a6ea

Author: costero-e

README.md

@@ -15,7 +15,8 @@ Please, go to [B2RI/B2PI docs website](https://b2ri-documentation-demo.ega-archi
 * Validation on the fly per framework and model(s).
 * Configuration of the entities of each entry type now done by .yml files.
 * Restart of the app when conf files or generic conf is modified (no need to rebuild).
-* Other bug fixes
+* OR Filters (in test approach, as it is still not approved officially by GA4GH).
+* Other bug fixes.
 * Unit tests expanded, with a total of 313 now.
 
 ## Main changes from B2RI

beacon/auth/tests.py

@@ -9,7 +9,7 @@
 from beacon.logs.logs import LOG
 
 # for keycloak, create aud in mappers, with custom, aud and beacon for audience
-mock_access_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJreS1tUXNxZ0ZYeHdSUVRfRUhuQlJJUGpmbVhfRXZuUTVEbzZWUTJCazdZIn0.eyJleHAiOjE3NjU5MDU0OTgsImlhdCI6MTc2NTkwNTE5OCwianRpIjoiMGRmYjhlNWYtMGQ4My00Njk3LTkzYTEtYWI4MTI3NzZhYjU4IiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL2F1dGgvcmVhbG1zL0JlYWNvbiIsImF1ZCI6ImJlYWNvbiIsInN1YiI6IjQ3ZWZmMWIxLTc2MjEtNDU3MC1hMGJiLTAxYTcxOWZiYTBhMiIsInR5cCI6IkJlYXJlciIsImF6cCI6ImJlYWNvbiIsInNlc3Npb25fc3RhdGUiOiI0YjUyMjI3Zi0wZjEyLTRmYjYtOWQ0YS0yY2E5OGU4MDg5NmYiLCJhY3IiOiIxIiwic2NvcGUiOiJvcGVuaWQgcHJvZmlsZSBlbWFpbCBtaWNyb3Byb2ZpbGUtand0Iiwic2lkIjoiNGI1MjIyN2YtMGYxMi00ZmI2LTlkNGEtMmNhOThlODA4OTZmIiwidXBuIjoiamFuZSIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwibmFtZSI6IkphbmUgU21pdGgiLCJncm91cHMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidW1hX2F1dGhvcml6YXRpb24iXSwicHJlZmVycmVkX3VzZXJuYW1lIjoiamFuZSIsImdpdmVuX25hbWUiOiJKYW5lIiwiZmFtaWx5X25hbWUiOiJTbWl0aCIsImVtYWlsIjoiamFuZS5zbWl0aEBiZWFjb24uZ2E0Z2gifQ.MfDxGD_a1IQoQZGJ2-1lDDmJGh2N6OM0b-UObdyF07tzfNZ3tFlCwO2_HJEEMb3E8BxT9jyhHkcGkZbEym8-mh8a7KAyS_Pxxaa_4TMCjbDnwJQdVCQb91Y15NxpZWSRDFZtj3tZO59btij1iiDoGVj8YhouYu3O8IHFgS9hlw4Cr5qiVOn_0_rBVg_k9YSRaTEM47PLmOMsJOd-xHW8kulrYR9CrDEhDL4Weu8GFdLJcsnleHLjggYoIq0hB4mJYf1fqn4P78oOH4sLL4l-ueIsg4AK3Hu_eJt810zdD27MUBM6IbCBSQCa4I6hCDg47Z_IrKKl5FSTeltxg8-lDA'
+mock_access_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJreS1tUXNxZ0ZYeHdSUVRfRUhuQlJJUGpmbVhfRXZuUTVEbzZWUTJCazdZIn0.eyJleHAiOjE3NjU5MTU2NzUsImlhdCI6MTc2NTkxNTM3NSwianRpIjoiMTkzZDU4ZjEtNzRiYS00MDJiLWJjNWQtYWU4NjAzZWIyYTliIiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL2F1dGgvcmVhbG1zL0JlYWNvbiIsImF1ZCI6ImJlYWNvbiIsInN1YiI6IjQ3ZWZmMWIxLTc2MjEtNDU3MC1hMGJiLTAxYTcxOWZiYTBhMiIsInR5cCI6IkJlYXJlciIsImF6cCI6ImJlYWNvbiIsInNlc3Npb25fc3RhdGUiOiJkYjk4YzE1OC1mOTAxLTQ5YTctODRkNS04MzQxNjVmYzMxYTMiLCJhY3IiOiIxIiwic2NvcGUiOiJvcGVuaWQgcHJvZmlsZSBlbWFpbCBtaWNyb3Byb2ZpbGUtand0Iiwic2lkIjoiZGI5OGMxNTgtZjkwMS00OWE3LTg0ZDUtODM0MTY1ZmMzMWEzIiwidXBuIjoiamFuZSIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwibmFtZSI6IkphbmUgU21pdGgiLCJncm91cHMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidW1hX2F1dGhvcml6YXRpb24iXSwicHJlZmVycmVkX3VzZXJuYW1lIjoiamFuZSIsImdpdmVuX25hbWUiOiJKYW5lIiwiZmFtaWx5X25hbWUiOiJTbWl0aCIsImVtYWlsIjoiamFuZS5zbWl0aEBiZWFjb24uZ2E0Z2gifQ.cboOLVmGnqR8BE8i71aVzwSite9aeTpujvGStx8yutosl80_4V_usLeqTI2pHRP9YEbgNLDFCBHXACJ0zL2SLGeuWYpj1OtOqk_49Vvh9wXVoIkv3DH0FrNMuWadtnZYtDViAOdvOyWK8dwsk8xptbcu4wS5EaiNKuOLXG8VneGYZ_GQ03gMocvSq4SORmzGjNo8hZ9NR5_JudoNOLbzftOXjY5smKItp7dsyrwWVOcFd09LKdkxgzf2OeHybjdso_S4U-Af47crrZ0XiLwfsVS535c2RNfJVHwImtrK_5li4A8n15mAGYRqXMqC2cRSl_cbqL8pYzjIBS_ECcAJ8g'
 mock_access_token_false = 'public'
 #dummy test anonymous
 #dummy test login

beacon/conf/api_version.yml

@@ -1 +1 @@
-api_version: v2.0-2aba4a4
+api_version: v2.0-1d0a6ea

beacon/conf/conf.py

@@ -14,7 +14,7 @@
 beacon_id = 'org.ega-archive.beacon-ri-demo'  # ID of the Beacon
 beacon_name = 'Beacon Production Implementation demo'  # Name of the Beacon service
 api_version = 'v2.2.0' # Version of the Beacon implementation
-uri = 'http://localhost:5050'
+uri = 'http://localhost:50510'
 uri_subpath = '/api'
 complete_url = uri + uri_subpath
 environment = 'test'

beacon/conf/models/models_conf.yml

@@ -2,4 +2,4 @@ ga4gh/beacon_v2_default_model:
   model_enabled: True
 
 EUCAIM:
-  model_enabled: True
+  model_enabled: False

beacon/models/ga4gh/beacon_v2_default_model/connections/mongo/non_collections.py

@@ -71,6 +71,7 @@ def get_phenotypic_endpoint_with_id(self, dataset: SingleDatasetResponse):
 def get_variants_of_phenotypic_endpoint(self, dataset: SingleDatasetResponse):
     analysis_confile=import_analysis_confile()
     run_confile=import_run_confile()
+    new_entry_id = RequestAttributes.entry_id
     # Check which is the queried initial entry type of the cross query and process and get the ids to convert to the final entry type response.
     if RequestAttributes.pre_entry_type == analysis_confile["analysis"]["endpoint_name"] or RequestAttributes.pre_entry_type == run_confile["run"]["endpoint_name"]:
         query = {"$and": [{"id": RequestAttributes.entry_id}]}
@@ -82,8 +83,8 @@ def get_variants_of_phenotypic_endpoint(self, dataset: SingleDatasetResponse):
             initial_ids = runs \
                 .find_one(query, {"biosampleId": 1, "_id": 0})  
         try:
-            RequestAttributes.entry_id = initial_ids["biosampleId"]
-        except Exception:
+            new_entry_id = initial_ids["biosampleId"]
+        except Exception as e:
             return dataset
     try:
         # Retrieve the ids per dataset.
@@ -95,7 +96,7 @@ def get_variants_of_phenotypic_endpoint(self, dataset: SingleDatasetResponse):
         return dataset
     # Map the ids and check which is the position fo the id found in the array of ids.
     for bioid in bioids:
-        if bioid == RequestAttributes.entry_id:
+        if bioid == new_entry_id:
             break
         position+=1
     if position == len(bioids):

beacon/utils/modules.py

@@ -25,6 +25,9 @@ def load_class(script_name, className):
     return klass
 
 def load_types_of_results(response_type):
+    with open("/beacon/conf/models/models_conf.yml", 'r') as pfile:
+        models_confile= yaml.safe_load(pfile)
+    pfile.close()
     list_of_results_classes_accepted=[]
     version_catch = re.search(r"(v\d+(\.\d+)*)", RequestAttributes.returned_schema[0]["schema"])
     if version_catch:
@@ -34,6 +37,9 @@ def load_types_of_results(response_type):
     dirs = os.listdir("/beacon/models")
     for folder in dirs:
         subdirs = os.listdir("/beacon/models/"+folder)
+        if folder in models_confile:
+            if models_confile[folder]["model_enabled"] == False:
+                continue
         if "validator" in subdirs:
             validatordirs = os.listdir("/beacon/models/"+folder+"/validator/"+response_type)
             for validatorfolder in validatordirs:
@@ -48,6 +54,9 @@ def load_types_of_results(response_type):
                         list_of_results_classes_accepted.append(klass)
         for subfolder in subdirs:
             underdirs = os.listdir("/beacon/models/"+folder+"/"+subfolder)
+            if  folder+'/'+subfolder in models_confile:
+                if models_confile[folder+'/'+subfolder ]["model_enabled"] == False:
+                    continue
             if "validator" in underdirs:
                 try:
                     validatordirs = os.listdir("/beacon/models/"+folder+"/"+subfolder+"/validator/"+response_type)
@@ -67,10 +76,16 @@ def load_types_of_results(response_type):
     return union_type
 
 def load_routes():
+    with open("/beacon/conf/models/models_conf.yml", 'r') as pfile:
+        models_confile= yaml.safe_load(pfile)
+    pfile.close()
     dirs = os.listdir("/beacon/models")
     routes_to_add={}
     for folder in dirs:
         subdirs = os.listdir("/beacon/models/"+folder)
+        if folder in models_confile:
+            if models_confile[folder]["model_enabled"] == False:
+                continue
         if "conf" in subdirs:
             confiles = os.listdir("/beacon/models/"+folder+"/conf/entry_types/")
             for confile in confiles:
@@ -90,6 +105,9 @@ def load_routes():
         else:
             for subfolder in subdirs:
                 underdirs = os.listdir("/beacon/models/"+folder+"/"+subfolder)
+                if folder+'/'+subfolder in models_confile:
+                    if models_confile[folder+'/'+subfolder ]["model_enabled"] == False:
+                        continue
                 if "conf" in underdirs:
                     confiles = os.listdir("/beacon/models/"+folder+"/"+subfolder+"/conf/entry_types/")
                     for confile in confiles:
@@ -191,12 +209,18 @@ def get_one_module_conf(entry_type):
                                 return entry_type_confile
 
 def get_modules_confiles():
+    with open("/beacon/conf/models/models_conf.yml", 'r') as pfile:
+        models_confile= yaml.safe_load(pfile)
+    pfile.close()
     dirs = os.listdir("/beacon/models")
     list_of_confiles=[]
     for folder in dirs:
         subdirs = os.listdir("/beacon/models/"+folder)
         if "conf" in subdirs:
             confiles = os.listdir("/beacon/models/"+folder+"/conf/entry_types/")
+            if folder in models_confile:
+                if models_confile[folder]["model_enabled"] == False:
+                    continue
             for confile in confiles:
                 if confile != '__pycache__':
                     with open("/beacon/models/"+folder+"/conf/entry_types/" + confile, 'r') as pfile:
@@ -206,6 +230,9 @@ def get_modules_confiles():
         else:
             for subfolder in subdirs:
                 underdirs = os.listdir("/beacon/models/"+folder+"/"+subfolder)
+                if folder+'/'+subfolder in models_confile:
+                    if models_confile[folder+'/'+subfolder ]["model_enabled"] == False:
+                        continue
                 if "conf" in underdirs:
                     confiles = os.listdir("/beacon/models/"+folder+"/"+subfolder+"/conf/entry_types/")
                     for confile in confiles:

beacon/utils/shutters.py

@@ -44,39 +44,67 @@ def graceful_shutdown_sigterm_handler():
 
 PATHS_TO_RESTART = [
     "/beacon/conf/conf.py",
-    "/beacon/models"
+    "/beacon/models",
+    "/beacon/conf/models"
 ]
+async def config_test_watcher(app):
+    pass
+
 
 async def config_watcher(app):
-    def snapshot():
-        times = {}
+    initial_times = {}
+
+    # Let's add all the snapshot times for the folders to restart when changed
+    for path in PATHS_TO_RESTART:
+        if os.path.isfile(path):
+            initial_times[path] = os.path.getmtime(path)
+        elif os.path.isdir(path):
+            for root, _, files in os.walk(path):
+                for f in files:
+                    full = os.path.join(root, f)
+                    initial_times[full] = os.path.getmtime(full)
+    # We give a time to the server to start
+    await asyncio.sleep(5) 
+
+    while True:
+        await asyncio.sleep(2)
+
+        new_initial_times = {}
+
+        # We check again for any change
         for path in PATHS_TO_RESTART:
             if os.path.isfile(path):
-                times[path] = os.path.getmtime(path)
+                new_initial_times[path] = os.path.getmtime(path)
             elif os.path.isdir(path):
                 for root, _, files in os.walk(path):
                     if "__pycache__" in root:
                         continue
+                    if not (f.endswith(".py") or f.endswith(".yml")):
+                        continue
                     for f in files:
-                        if not f.endswith(".py"):
-                            continue
+
+
                         full = os.path.join(root, f)
-                        times[full] = os.path.getmtime(full)
-        return times
+                        new_initial_times[full] = os.path.getmtime(full)
+
+
+
+
 
-    initial_times = snapshot()
-    await asyncio.sleep(5)
 
-    while True:
-        await asyncio.sleep(2)
-        new_times = snapshot()
 
-        for file_path, new_m in new_times.items():
+
+
+        # If there is a change then, restart again the app
+        for file_path, new_m in new_initial_times.items():
             old_m = initial_times.get(file_path)
-            if old_m is None or new_m > old_m:
+            if old_m is None or new_m != old_m:
                 os._exit(0)
 
-        initial_times = new_times
+        initial_times = new_initial_times
+
+async def on_startup(app):
+    app["config_watcher"] = asyncio.create_task(config_watcher(app))
 
 async def on_startup(app):
     app["config_watcher"] = asyncio.create_task(config_watcher(app))

docker-compose.yml

@@ -6,7 +6,7 @@ networks:
 
 services:
   beaconprod:
-    restart: always
+    restart: unless-stopped
     build:
       context: .
     container_name: beaconprod
@@ -18,6 +18,7 @@ services:
       - ./beacon/conf/conf.py:/beacon/conf/conf.py
       - ./beacon/permissions:/beacon/permissions
       - ./beacon/models:/beacon/models
+      - ./beacon/conf/models:/beacon/conf/models
     networks:
       - pub