remove task json file, use the one from the webarena-verified library. Update task template to include revision number

Author: NicolasAG

browsergym/experiments/src/browsergym/experiments/benchmark/metadata/webarena_verified.csv

@@ -157,16 +157,16 @@ def prepare_backend(backend: str):
             )
             massage_tasks(
                 [
-                    f"webarena_verified.{intent_template_id}.{task_id}"
-                    for intent_template_id, task_id in [
-                        (23, 410),  # reddit
-                        (330, 533),  # gitlab
-                        (87, 561),  # gitlab wiki
-                        (88, 562),  # gitlab reddit
-                        (165, 574),  # shopping
-                        (16, 640),  # reddit
-                        (253, 680),  # shopping_admin
-                        (94, 740),  # wiki map
+                    f"webarena_verified.{intent_template_id}.{task_id}.{revision}"
+                    for intent_template_id, task_id, revision in [
+                        (23, 410, 2),  # reddit
+                        (330, 533, 2),  # gitlab
+                        (87, 561, 3),  # gitlab wiki
+                        (88, 562, 2),  # gitlab reddit
+                        (165, 574, 2),  # shopping
+                        (16, 640, 2),  # reddit
+                        (253, 680, 2),  # shopping_admin
+                        (94, 740, 2),  # wiki map
                     ]
                 ]
             )

browsergym/experiments/src/browsergym/experiments/benchmark/utils.py

@@ -16,6 +16,7 @@ authors = [
     {name = "Thibault Le Sellier De Chezelles"},
     {name = "Tom Marty"},
     {name = "Aman Jaiswal"},
+    {name = "Nicolas Gontier"},
 ]
 readme = "README.md"
 requires-python = ">3.10"

browsergym/pyproject.toml

@@ -49,4 +49,10 @@ benchmark = benchmark.subset_from_list(
 )
 ```
 
-**NOTE**: Tasks are registered with this template: `webarena_verified.{intent_template_id}.{task_id}`
+#### 3. Task gym ID format
+
+Tasks are registered to gym with this template: `webarena_verified.{intent_template_id}.{task_id}.{revision}`
+
+- the `intent_template_id` (int) refers to the template of the question. Multiple tasks can have the same template question but with different instantiations.
+- the `task_id` (int from 0 to 811) is unique to each question. This is the same task ID as in the original webarena benchmark.
+- the `revision` (int) is a version number to keep track of updates done to all webarena-verified tasks accross time.

browsergym/webarena_verified/README.md

@@ -29,11 +29,6 @@ path = "../core/src/browsergym/core/__init__.py"
 [tool.hatch.metadata.hooks.requirements_txt]
 files = ["requirements.txt"]
 
-[tool.hatch.build]
-include = [
-    "src/browsergym/webarena_verified/webarena_verified.json"
-]
-
 [tool.hatch.metadata]
 allow-direct-references = true
 

browsergym/webarena_verified/pyproject.toml

@@ -14,8 +14,10 @@
 ALL_WEBARENA_TASK_IDS = []
 
 # register all WebArena benchmark
-for task_id, intent_template_id in zip(config.TASK_IDS, config.INTENT_TEMPLATE_IDS):
-    gym_id = f"webarena_verified.{intent_template_id}.{task_id}"
+for task_id, intent_template_id, revision in zip(
+    config.TASK_IDS, config.INTENT_TEMPLATE_IDS, config.REVISIONS
+):
+    gym_id = f"webarena_verified.{intent_template_id}.{task_id}.{revision}"
     register_task(
         gym_id,
         task.WebArenaVerifiedTask,

browsergym/webarena_verified/src/browsergym/webarena_verified/__init__.py

@@ -5,34 +5,19 @@
 
 TASK_IDS = range(812)
 INTENT_TEMPLATE_IDS = []
+REVISIONS = []
 
-with open(Path(__file__).parent / "webarena_verified.json", "r") as f:
-    data = json.load(f)
-
-# Check if the json file is the same as the one in the webarena-verified repository
-library_json_string = (
+# Load the json file from the webarena-verified library
+data = json.loads(
     importlib.resources.files("webarena_verified")
     .joinpath("assets/dataset/webarena-verified.json")
     .read_text()
 )
-library_json = json.loads(library_json_string)
-
-if json.dumps(data, sort_keys=True, indent=2) != json.dumps(library_json, sort_keys=True, indent=2):
-    print(
-        "Warning: the json file is not the same as the one in the webarena-verified repository. Consider updating the library."
-    )
-    print("=" * 100)
-    print("Differences:")
-    for diff in difflib.unified_diff(
-        json.dumps(data, sort_keys=True, indent=2).splitlines(),
-        json.dumps(library_json, sort_keys=True, indent=2).splitlines(),
-    ):
-        print(diff)
-    print("=" * 100)
 
 for task in data:
     INTENT_TEMPLATE_IDS.append(task["intent_template_id"])
+    REVISIONS.append(task["revision"])
 
-assert len(INTENT_TEMPLATE_IDS) == len(
-    TASK_IDS
-), "Number of intent template IDs must match number of task IDs"
+assert (
+    len(INTENT_TEMPLATE_IDS) == len(TASK_IDS) == len(REVISIONS)
+), f"Number of intent template IDs ({len(INTENT_TEMPLATE_IDS)}), task IDs ({len(TASK_IDS)}), and revisions ({len(REVISIONS)}) must match"

browsergym/webarena_verified/src/browsergym/webarena_verified/config.py

@@ -3,6 +3,7 @@
 from platform-labs-agent-eval-harness.
 """
 
+import importlib.resources
 import json
 import logging
 import tempfile
@@ -46,7 +47,9 @@ def __init__(self, webarena_instance: WebArenaInstance):
         """
         # Create configuration for all sites and homepage from webarena_instance
         config = WebArenaVerifiedConfig(
-            test_data_file=Path(__file__).parent.joinpath("webarena_verified.json"),
+            test_data_file=importlib.resources.files("webarena_verified").joinpath(
+                "assets/dataset/webarena-verified.json"
+            ),
             environments={
                 **{
                     site: EnvironmentConfig(

browsergym/webarena_verified/src/browsergym/webarena_verified/evaluators.py

@@ -46,8 +46,8 @@ def __init__(
 
         # Load the webarena_verified.json file
         all_configs_str = (
-            importlib.resources.files("browsergym.webarena_verified")
-            .joinpath("webarena_verified.json")
+            importlib.resources.files("webarena_verified")
+            .joinpath("assets/dataset/webarena-verified.json")
             .read_text()
         )