Lab 3.0 (Newton)

.github/workflows/ci.yml

@@ -97,11 +97,11 @@ jobs:
   test:
     name: Run tests
     runs-on: [self-hosted, gpu]
-    timeout-minutes: 60
+    timeout-minutes: 80
     needs: [pre_commit]
 
     container:
-      image: nvcr.io/nvstaging/isaac-amr/isaaclab_arena:latest
+      image: nvcr.io/nvstaging/isaac-amr/isaaclab_arena:lab3
       credentials:
         username: $oauthtoken
         password: ${{ env.NGC_API_KEY }}
@@ -125,12 +125,36 @@ jobs:
           /isaac-sim/python.sh -m pip install --no-cache-dir -e .
           [ -e ./submodules/IsaacLab/_isaac_sim ] || ln -s /isaac-sim ./submodules/IsaacLab/_isaac_sim
 
-      # Run the tests (excluding the gr00t related tests)
-      - name: Run pytest excluding policy-related tests. First we run all tests without cameras.
-        run: /isaac-sim/python.sh -m pytest -sv -m "not with_cameras" isaaclab_arena/tests/
+      # Run the tests.
+      #
+      # Note(alexmillane): We have to run the tests in a specific way. To achieve a reasonable test speed,
+      # we don't close the SimulationApp between tests, where this is possible. We need, however, to restart the
+      # SimulationApp between tests that require cameras/rendering and those that don't. To make
+      # matters more complicated, since Isaac Sim 6.0, this persistent SimulationApp scheme is also not
+      # mixing well with subprocess-spawning tests. We get intermittent hangs if a subprocess-spawning
+      # test is run while a persistent SimulationApp is still open. For that reason we also need to
+      # separate those test that spawn subprocess. So in the end, we run
+      #
+      # - All tests not requiring cameras in a shared SimulationApp.
+      # - All tests requiring cameras in a shared SimulationApp.
+      # - All subprocess-spawning tests (each spawns their own SimulationApp)
+      #
+      # To restore sanity here is a goal for Isaac Lab Arena v0.3.
+
+      - name: Run in-process tests without cameras
+        run: |
+          /isaac-sim/python.sh -m pytest -sv -m "not with_cameras and not with_subprocess" \
+            isaaclab_arena/tests/
+
+      - name: Run in-process tests with cameras.
+        run: |
+          /isaac-sim/python.sh -m pytest -sv -m "with_cameras and not with_subprocess" \
+            isaaclab_arena/tests/
 
-      - name: Run pytest excluding policy-related tests. Now we run all tests with cameras.
-        run: /isaac-sim/python.sh -m pytest -sv -m with_cameras isaaclab_arena/tests/
+      - name: Run subprocess-spawning tests (before persistent SimApp is created)
+        run: |
+          /isaac-sim/python.sh -m pytest -sv -m with_subprocess \
+            isaaclab_arena/tests/
 
 
   test_policy:
@@ -140,10 +164,12 @@ jobs:
     needs: [pre_commit]
 
     container:
-      image: nvcr.io/nvstaging/isaac-amr/isaaclab_arena:cuda_gr00t_gn16
+      image: nvcr.io/nvstaging/isaac-amr/isaaclab_arena:lab3_cuda_gr00t_gn16
       credentials:
         username: $oauthtoken
         password: ${{ env.NGC_API_KEY }}
+    env:
+      GROOT_DEPS_DIR: /opt/groot_deps
     steps:
       # nvidia-smi
       - *nvidia_smi

.github/workflows/gh-pages.yml

@@ -7,7 +7,7 @@ name: GitHub Pages
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "feature/isaac_lab_3_newton" ]
 
 # Concurrency control to prevent parallel runs on the same PR
 concurrency:
@@ -32,7 +32,7 @@ jobs:
     timeout-minutes: 30
 
     container:
-      image: python:3.11-slim
+      image: python:3.12-slim
 
     steps:
       - name: Install git (and tools needed by hooks)

docker/Dockerfile.gr00t_server

@@ -22,6 +22,8 @@ RUN chmod +x /tmp/install_gr00t_deps.sh && \
     rm -f /tmp/install_gr00t_deps.sh
 
 RUN pip install --no-cache-dir --upgrade "opencv-python-headless==4.8.0.74"
+# GN1.6 uses termcolor 3.2.0
+RUN pip install --no-cache-dir termcolor==3.2.0
 
 COPY isaaclab_arena/remote_policy ${WORKDIR}/isaaclab_arena/remote_policy
 COPY isaaclab_arena_gr00t ${WORKDIR}/isaaclab_arena_gr00t
@@ -30,5 +32,7 @@ COPY isaaclab_arena_g1 ${WORKDIR}/isaaclab_arena_g1
 RUN pip install --no-cache-dir pyzmq msgpack
 
 ENV PYTHONPATH=${WORKDIR}
+# So gr00t_remote_policy loads transformers/tokenizers from GR00T deps (e.g. tokenizers 0.21.x) not system site-packages.
+ENV GROOT_DEPS_DIR=/opt/groot_deps
 
 ENTRYPOINT ["python", "-u", "-m", "isaaclab_arena.remote_policy.remote_policy_server_runner"]

docker/Dockerfile.isaaclab_arena

@@ -1,8 +1,7 @@
-ARG BASE_IMAGE=nvcr.io/nvidia/isaac-sim:5.1.0
+ARG BASE_IMAGE=nvcr.io/nvidia/isaac-sim:6.0.0-dev2
 
 FROM ${BASE_IMAGE}
 
-# Set user to root (Isaac Sim base image defaults to non-root user)
 USER root
 
 # GR00T Policy Build Arguments, these are only used if INSTALL_GROOT is true
@@ -43,12 +42,16 @@ RUN chmod 777 -R /isaac-sim/kit/
 # Make /isaac-sim directory traversable and readable by all users
 # This is needed when entrypoint switches to non-root user
 RUN chmod a+x /isaac-sim
-# NOTE(alexmillane, 2026-02-10): We started having issues with flatdict 4.0.1 installation
-# during IsaacLab install. We install here with build isolation which seems to fix the issue.
-RUN /isaac-sim/python.sh -m pip install flatdict==4.0.1 --no-build-isolation
+# Ensure isaaclab_visualizers is installed so --visualizer kit works.
+RUN /isaac-sim/python.sh -m pip install --no-deps -e ${WORKDIR}/submodules/IsaacLab/source/isaaclab_visualizers
+RUN /isaac-sim/python.sh -m pip install --no-deps -e ${WORKDIR}/submodules/IsaacLab/source/isaaclab_teleop
+
 # Install isaaclab
 RUN ${ISAACLAB_PATH}/isaaclab.sh -i
 
+# Install Isaac Teleop Python APIs (retargeters, device I/O, OpenXR bindings)
+RUN /isaac-sim/python.sh -m pip install isaacteleop~=1.0 --extra-index-url https://pypi.nvidia.com
+
 # Patch for osqp in IsaacLab. Downgrade qpsolvers
 # TODO(alexmillane): Watch the thread here: https://nvidia.slack.com/archives/C06HLQ6CB41/p1764680205807019
 #                    and remove this thread when IsaacLab has a fix.
@@ -80,11 +83,14 @@ RUN /isaac-sim/python.sh -m pip install --upgrade pip && \
 # Lightwheel server
 ENV LW_API_ENDPOINT="https://api-dev.lightwheel.net"
 
-# HuggingFace for downloading datasets and models.
-# NOTE(alexmillane, 2025-10-28): For some reason the CLI has issues when installed in the IsaacSim version of python.
-RUN pip install --break-system-packages huggingface-hub[cli]
-# Create alias for hf command to use the system-installed version
-RUN echo "alias hf='/usr/local/bin/hf'" >> /etc/bash.bashrc
+# HuggingFace CLI for downloading datasets and models.
+# Use pipx so the hf binary gets an isolated venv with all its deps (e.g. requests),
+# without touching system Python packages.
+RUN apt-get install -y pipx && \
+    PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin \
+    pipx install \
+    "huggingface-hub[cli]" \
+    pre-commit
 
 ###############################
 # Install GR00T and CUDA 12.8 #
@@ -104,14 +110,12 @@ COPY ./submodules/Isaac-GR00T ${WORKDIR}/submodules/Isaac-GR00T
 # Copy GR00T dependencies installation script
 COPY docker/setup/install_gr00t_deps.sh /tmp/install_gr00t_deps.sh
 RUN chmod +x /tmp/install_gr00t_deps.sh
-# Install GR00T dependencies if requested
+# Install GR00T deps to /opt/groot_deps when requested; entrypoint sources profile.d to set GROOT_DEPS_DIR
 RUN if [ "$INSTALL_GROOT" = "true" ]; then \
-    /tmp/install_gr00t_deps.sh; \
+    /tmp/install_gr00t_deps.sh && echo 'export GROOT_DEPS_DIR=/opt/groot_deps' > /etc/profile.d/groot_deps.sh; \
   else \
-    echo "Skipping GR00T installation"; \
-  fi && \
-  # Clean up installation scripts
-  rm -f /tmp/install_gr00t_deps.sh
+    echo "Skipping GR00T installation" && rm -f /etc/profile.d/groot_deps.sh; \
+  fi && rm -f /tmp/install_gr00t_deps.sh
 
 # Copy the rest of the files
 COPY *.* ${WORKDIR}/

docker/push_to_ngc.sh

@@ -2,7 +2,7 @@
 set -e
 
 ISAACLAB_ARENA_IMAGE_NAME='isaaclab_arena'
-TAG_NAME=latest
+TAG_NAME=lab3
 CONTAINER_ID=""
 PUSH_TO_NGC=false
 INSTALL_GROOT="false"
@@ -18,7 +18,7 @@ while getopts ":t:gn:vn:pn:Rn:hn:" OPTION; do
             ;;
         g)
             INSTALL_GROOT="true"
-            TAG_NAME='cuda_gr00t_gn16'
+            TAG_NAME='lab3_cuda_gr00t_gn16'
             echo "INSTALL_GROOT is ${INSTALL_GROOT}."
             ;;
         v)

docker/run_docker.sh

@@ -1,15 +1,12 @@
 #!/bin/bash
 set -e
 DOCKER_IMAGE_NAME='isaaclab_arena'
-DOCKER_VERSION_TAG='latest'
+DOCKER_VERSION_TAG='lab3'
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 WORKDIR="/workspaces/isaaclab_arena"
 
-# Default OpenXR directory shared with CloudXR runtime (lives in IsaacLab submodule)
-OPENXR_HOST_DIR="./submodules/IsaacLab/openxr"
-
 # Default mount directory on the host machine for the datasets
 DATASETS_HOST_MOUNT_DIRECTORY="$HOME/datasets"
 # Default mount directory on the host machine for the models
@@ -50,7 +47,7 @@ while getopts ":d:m:e:hn:rn:Rn:vn:gn:" OPTION; do
             ;;
         g)
             INSTALL_GROOT="true"
-            DOCKER_VERSION_TAG='cuda_gr00t_gn16'
+            DOCKER_VERSION_TAG='cuda_gr00t_gn16_lab3'
             ;;
         h)
             script_name=$(basename "$0")
@@ -140,21 +137,27 @@ else
                     "-v" "/tmp/.X11-unix:/tmp/.X11-unix:rw"
                     "-v" "/var/run/docker.sock:/var/run/docker.sock"
                     "-v" "$HOME/.Xauthority:/root/.Xauthority"
+                    # Mount host SSL certificate store so the container trusts CA certs
+                    "-v" "/etc/ssl/certs:/etc/ssl/certs:ro"
                     "--env" "DISPLAY"
                     "--env" "ACCEPT_EULA=Y"
                     "--env" "PRIVACY_CONSENT=Y"
                     "--env" "DOCKER_RUN_USER_ID=$(id -u)"
                     "--env" "DOCKER_RUN_USER_NAME=$(id -un)"
                     "--env" "DOCKER_RUN_GROUP_ID=$(id -g)"
                     "--env" "DOCKER_RUN_GROUP_NAME=$(id -gn)"
-                    # Setting envs for XR: https://isaac-sim.github.io/IsaacLab/v2.1.0/source/how-to/cloudxr_teleoperation.html#run-isaac-lab-with-the-cloudxr-runtime
-                    "--env" "XDG_RUNTIME_DIR=${WORKDIR}/submodules/IsaacLab/openxr/run"
-                    "--env" "XR_RUNTIME_JSON=${WORKDIR}/submodules/IsaacLab/openxr/share/openxr/1/openxr_cloudxr.json"
+                    # CloudXR shared volume: TeleopCore's run_cloudxr_via_docker.sh writes runtime
+                    # files to CXR_HOST_VOLUME_PATH (default ~/.cloudxr) on the host.
+                    "-v" "${CXR_HOST_VOLUME_PATH:-$HOME/.cloudxr}:/cloudxr"
+                    "--env" "XR_RUNTIME_JSON=/cloudxr/openxr_cloudxr.json"
+                    "--env" "NV_CXR_RUNTIME_DIR=/cloudxr/run"
                     # NOTE(alexmillane, 2025.07.23): This looks a bit suspect to me. We should be running
                     # as a user inside the container, not root. I've left it in for now, but we should
                     # remove it, if indeed it's not needed.
                     # "--env" "OMNI_KIT_ALLOW_ROOT=1"
                     "--env" "ISAACLAB_PATH=${WORKDIR}/submodules/IsaacLab"
+                    # Tell requests/urllib3 to use the system cert bundle
+                    "--env" "REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt"
                     )
 
     # map omniverse auth or config so we have connection to the dev nucleus

docker/run_gr00t_server.sh

@@ -238,6 +238,11 @@ if [ -d "${EVAL_DIR}" ]; then
   DOCKER_RUN_ARGS+=(-v "${EVAL_DIR}":/eval)
 fi
 
+# Pass through so gr00t_remote_policy can print path/debug info (e.g. GROOT_DEBUG_PATH=1).
+if [ -n "${GROOT_DEBUG_PATH:-}" ]; then
+  DOCKER_RUN_ARGS+=(-e "GROOT_DEBUG_PATH=${GROOT_DEBUG_PATH}")
+fi
+
 docker run "${DOCKER_RUN_ARGS[@]}" \
   "${IMAGE_TAG_FULL}" \
   "${SERVER_ARGS[@]}"

docker/setup/entrypoint.sh

@@ -49,6 +49,9 @@ if [ ! -e "$WORKDIR/submodules/IsaacLab/_isaac_sim" ]; then
     ln -s /isaac-sim/ "$WORKDIR/submodules/IsaacLab/_isaac_sim"
 fi
 
+# Export GROOT_DEPS_DIR when GR00T was installed (INSTALL_GROOT=true)
+[ -f /etc/profile.d/groot_deps.sh ] && set -a && source /etc/profile.d/groot_deps.sh && set +a
+
 # Run the passed command or just start the shell as the created user
 if [ $# -ge 1 ]; then
     echo "alias pytest='/isaac-sim/python.sh -m pytest'" >> /etc/aliasess.bashrc