feat: store metrics in local tables and refresh via scim actions

Author: aleks-iv

ckanext/scientometrics/cli.py

@@ -6,16 +6,16 @@
 from ckanext.scientometrics import config
 
 __all__ = [
-    "scientometrics",
+    "scim",
 ]
 
 
 @click.group()
-def scientometrics():
+def scim():
     pass
 
 
-@scientometrics.command()
+@scim.command()
 @click.option(
     "--user-ids",
     type=str,
@@ -43,8 +43,8 @@ def update_user_metrics(user_ids: tuple, requested_sources: tuple):
         requested_sources = config.enabled_metrics()
     with click.progressbar(user_ids, label="Updating user metrics") as bar:
         for user_id in bar:
-            tk.get_action("scientometrics_update_user_metrics")(
-                {}, {"user_id": user_id, "requested_sources": requested_sources}
+            tk.get_action("scim_update_user_metrics")(
+                {"ignore_auth": True}, {"user_id": user_id, "requested_sources": requested_sources}
             )
 
     click.echo("Metrics update complete!")

ckanext/scientometrics/const.py

@@ -7,16 +7,16 @@
 from ckanext.scientometrics import config
 
 
-def scientometrics_get_user_metrics(user_id: str) -> dict[str, Any]:
+def scim_get_user_metrics(user_id: str) -> dict[str, Any]:
     """Retrieve the metrics for a user."""
-    return tk.get_action("scientometrics_get_user_metrics")({}, {"user_id": user_id})
+    return tk.get_action("scim_get_user_metrics")({}, {"user_id": user_id})
 
 
-def scientometrics_get_enabled_metrics() -> list[str]:
+def scim_get_enabled_metrics() -> list[str]:
     """List of enabled metrics."""
     return config.enabled_metrics()
 
 
-def scientometrics_show_metrics_on_user_page() -> bool:
+def scim_show_metrics_on_user_page() -> bool:
     """Show metrics on user page in the info section."""
     return config.show_metrics_on_user_page()

ckanext/scientometrics/helpers.py

@@ -6,7 +6,7 @@
 
 
 class IScientometrics(Interface):
-    def get_metrics_extractors(self) -> dict[str, AuthorMetricsExtractor]:
+    def get_metrics_extractors(self) -> dict[str, type[AuthorMetricsExtractor]]:
         """Allows to redefine the default metrics extractors.
 
         Default:

ckanext/scientometrics/interfaces.py

@@ -1,22 +1,18 @@
 from __future__ import annotations
 
 import copy
+import logging
 from typing import Any
 
 import ckan.plugins.toolkit as tk
-from ckan import types
+from ckan import model, types
 from ckan.logic import validate
 
-from ckanext.scientometrics import utils
+from ckanext.scientometrics import config, utils
 from ckanext.scientometrics.logic import schema
+from ckanext.scientometrics.model import UserMetric
 
-
-@tk.chained_action
-def user_create(next_: Any, context: types.Context, data_dict: dict[str, Any]):
-    """Attach scientometrics extras to a user when it is created."""
-    user = next_(context, data_dict)
-    _attach_extras(context, data_dict, user["id"])
-    return user
+log = logging.getLogger(__name__)
 
 
 @tk.chained_action
@@ -36,10 +32,8 @@ def user_show(next_: Any, context: types.Context, data_dict: dict[str, Any]):
     userobj = context["model"].User.get(user_data["id"])
     if not userobj:
         raise tk.ObjectNotFound("user")
-    if userobj.plugin_extras and "scientometrics" in userobj.plugin_extras:
-        user_data["scientometrics"] = copy.deepcopy(
-            userobj.plugin_extras.get("scientometrics", {})
-        )
+    if userobj.plugin_extras and "scim" in userobj.plugin_extras:
+        user_data["scim"] = copy.deepcopy(userobj.plugin_extras.get("scim", {}))
     return user_data
 
 
@@ -52,73 +46,117 @@ def _attach_extras(context: types.Context, data_dict: dict[str, Any], user_id: s
         raise tk.ObjectNotFound("user")
     extras = copy.deepcopy(userobj.plugin_extras or {})
 
-    if "scientometrics" not in extras:
-        extras["scientometrics"] = {}
-    extras["scientometrics"].update(sm_details)
+    scim = extras.setdefault("scim", {})
+    for source in config.enabled_metrics():
+        key = f"{source}_author_id"
+        if key in data_dict and not data_dict.get(key):
+            scim.pop(key, None)
+    scim.update(sm_details)
     userobj.plugin_extras = extras
     userobj.save()
 
 
 @tk.side_effect_free
-@validate(schema.scientometrics_update_user_metrics)
-def scientometrics_update_user_metrics(
-        context: types.Context, data_dict: dict[str, Any]
-) -> dict[str, Any]:
-    """Update a user's scientometrics metrics.
+@validate(schema.scim_update_user_metrics)
+def scim_get_user_metrics(context: types.Context, data_dict: dict[str, Any]) -> dict[str, Any]:
+    """Retrieve user scientometrics metrics.
+
+    Args:
+        context (Dict[str, Any]): The CKAN action context.
+        data_dict (Dict[str, Any]): A dictionary containing:
+            - "user_id": The ID of the user whose metrics we want to retrieve.
+
+    Returns:
+        Dict[str, Any]: The user's scientometrics metrics keyed by source.
+    """
+    user_id_or_name = data_dict["user_id"]
+
+    user_dict = tk.get_action("user_show")({"ignore_auth": True}, {"id": user_id_or_name})
+    records = UserMetric.by_user_id(user_dict["id"])
+    return {record.source: record.dictize({}) for record in records}
+
+
+@validate(schema.scim_update_user_metrics)
+def scim_update_user_metrics(context: types.Context, data_dict: dict[str, Any]) -> dict[str, Any]:
+    """Create/update a user's scientometrics metrics using author ids.
 
     Args:
         context (Context): The CKAN action context.
         data_dict (dict[str, Any]): A dictionary containing:
             - "user_id": The ID of the user to update.
-            - "requested_sources": A list or dict indicating which metrics to update.
+            - "requested_sources": A list or dict indicating which metrics to update (subset of extras).
 
     Returns:
         Dict[str, Any]: A dictionary containing the updated metrics.
     """
+    tk.check_access("scim_update_user_metrics", context, data_dict)
     user_id_or_name = data_dict["user_id"]
-    user_dict = tk.get_action("user_show")(
-        {"ignore_auth": True}, {"id": user_id_or_name}
-    )
+    user_dict = tk.get_action("user_show")({"ignore_auth": True}, {"id": user_id_or_name})
     user_id = user_dict["id"]
 
-    requested_sources = data_dict["requested_sources"]
-    scientometrics_ids = user_dict.get("scientometrics", {})
-    updated_metrics = {}
+    requested_sources = set(data_dict["requested_sources"] or [])
+    records = UserMetric.by_user_id(user_id)
+    existing = {record.source: record for record in records}
+    authors = _collect_authors(user_id, existing)
+    sources = requested_sources & set(authors.keys()) if requested_sources else set(authors.keys())
+    updated_metrics: dict[str, dict[str, Any]] = {}
 
-    for metric_id_key, author_id in scientometrics_ids.items():
-        source = metric_id_key.replace("_author_id", "")
-        if not source or source not in requested_sources:
+    for source in sources:
+        author_id = authors.get(source)
+        if not author_id:
             continue
 
-        extracted_metrics = utils.fetch_author_metrics(source + "_author", author_id)
+        try:
+            extracted_metrics = utils.fetch_author_metrics(source + "_author", author_id)
+        except tk.ValidationError as exc:
+            log.warning("Failed to fetch metrics for user %s source %s: %s", user_id, source, exc, exc_info=True)
+            extracted_metrics = {"error": str(exc)}
         if not extracted_metrics:
             continue
 
-        utils.save_metrics_in_flake(user_id, source, author_id, extracted_metrics)
+        payload = dict(extracted_metrics)
+        payload["author_id"] = author_id
+        existing_record = existing.get(source)
+        external_id = payload.get("external_id") or (existing_record.external_id if existing_record else str(author_id))
+        external_url = (
+            payload.get("external_url")
+            or payload.get("url")
+            or (existing_record.external_url if existing_record else None)
+        )
+        status = existing_record.status if existing_record else "pending"
+        external = {"id": external_id, "url": external_url}
+
+        UserMetric.upsert(
+            user_id=user_id,
+            source=source,
+            metrics=payload,
+            external=external,
+            status=status,
+        )
         updated_metrics[source] = extracted_metrics
 
-    return updated_metrics
-
+    model.Session.commit()
 
-@tk.side_effect_free
-@validate(schema.scientometrics_update_user_metrics)
-def scientometrics_get_user_metrics(
-        context: types.Context, data_dict: dict[str, Any]
-) -> dict[str, Any]:
-    """Retrieve user scientometrics metrics.
-
-    Args:
-        context (Dict[str, Any]): The CKAN action context.
-        data_dict (Dict[str, Any]): A dictionary containing:
-            - "user_id": The ID of the user whose metrics we want to retrieve.
-
-    Returns:
-        Dict[str, Any]: The user's scientometrics metrics from the flake store.
-    """
-    user_id_or_name = data_dict["user_id"]
+    return updated_metrics
 
-    user_dict = tk.get_action("user_show")(
-        {"ignore_auth": True}, {"id": user_id_or_name}
-    )
 
-    return utils.get_metrics_from_flake(user_dict["id"])
+@validate(schema.scim_delete_user_metrics)
+def scim_delete_user_metrics(context: types.Context, data_dict: dict[str, Any]) -> int:
+    """Delete all scientometrics metrics for a user."""
+    tk.check_access("scim_delete_user_metrics", context, data_dict)
+    user_dict = tk.get_action("user_show")({"ignore_auth": True}, {"id": data_dict["user_id"]})
+    deleted = UserMetric.delete_by_user_id(user_dict["id"])
+    model.Session.commit()
+    return deleted
+
+
+def _collect_authors(user_id: str, existing: dict[str, UserMetric]) -> dict[str, str]:
+    """Collect authors strictly from user extras (scim or legacy scientometrics)."""
+    authors: dict[str, str] = {}
+    user_obj = model.User.get(user_id)
+    extras = (user_obj.plugin_extras or {}) if user_obj else {}
+    scim_extras = extras.get("scim") or extras.get("scientometrics") or {}
+    for key, val in scim_extras.items():
+        if key.endswith("_author_id") and val:
+            authors[key.removesuffix("_author_id")] = val
+    return authors

ckanext/scientometrics/logic/action.py

@@ -2,10 +2,16 @@
 
 from typing import Any
 
-from ckan import authz, types
+from ckan import types
 
 
-def scientometrics_update_user_metrics(
-    context: types.Context, data_dict: dict[str, Any]
-):
-    return authz.is_authorized("sysadmin", context, data_dict)
+def scim_update_user_metrics(context: types.Context, data_dict: dict[str, Any]):
+    return {"success": False}
+
+
+def scim_get_user_metrics(context: types.Context, data_dict: dict[str, Any]):
+    return {"success": True}
+
+
+def scim_delete_user_metrics(context: types.Context, data_dict: dict[str, Any]):
+    return {"success": False}

ckanext/scientometrics/logic/auth.py

@@ -1,22 +1,16 @@
-from typing import Any
-
 from ckan import types
 from ckan.logic.schema import validator_args
 
 from ckanext.scientometrics import config
 
-Schema = dict[str, Any]
-
 
 @validator_args
 def user_extras(ignore_empty: types.Validator) -> types.Schema:
-    return {
-        source + "_author_id": [ignore_empty] for source in config.enabled_metrics()
-    }
+    return {source + "_author_id": [ignore_empty] for source in config.enabled_metrics()}
 
 
 @validator_args
-def scientometrics_update_user_metrics(
+def scim_update_user_metrics(
     not_empty: types.Validator,
     default: types.Validator,
     convert_to_list_if_string: types.Validator,
@@ -28,3 +22,12 @@ def scientometrics_update_user_metrics(
             convert_to_list_if_string,
         ],
     }
+
+
+@validator_args
+def scim_delete_user_metrics(
+    not_empty: types.Validator,
+) -> types.Schema:
+    return {
+        "user_id": [not_empty],
+    }

ckanext/scientometrics/logic/schema.py

@@ -37,7 +37,7 @@ def include_object(
     compare_to: Any,
 ):
     if type_ == "table":
-        return object_name.startswith(name)
+        return object_name.startswith("scim_")
     return True
 
 

ckanext/scientometrics/migration/scientometrics/env.py

@@ -1,5 +1,6 @@
 import ckan.plugins.toolkit as tk
 from ckan import plugins as p
+from ckan.common import CKANConfig
 
 
 @tk.blanket.helpers
@@ -13,7 +14,7 @@ class ScientometricsPlugin(p.SingletonPlugin):
 
     # IConfigurer
 
-    def update_config(self, config_):
+    def update_config(self, config_: CKANConfig):
         tk.add_template_directory(config_, "templates")
         tk.add_public_directory(config_, "public")
         tk.add_resource("assets", "scientometrics")

ckanext/scientometrics/plugin.py

@@ -1,7 +1,7 @@
 {% ckan_extends %}
 
 {% block extra_fields %}
-    {% set enabled_metrics = h.scientometrics_get_enabled_metrics() %}
+    {% set enabled_metrics = h.scim_get_enabled_metrics() %}
     {% for metric in enabled_metrics %}
         {% snippet 'user/snippets/' + metric + '_field.html', form=form, data=data, errors=errors %}
     {% endfor %}