Fix object type and remove legacy external tables

- Add save_template() method to Config for creating datajoint.json templates
- Add default_store setting to ObjectStorageSettings
- Fix get_store_backend() to use default object storage when no store specified
- Fix StorageBackend._full_path() to prepend location for all protocols
- Fix StorageBackend.open() to create parent directories for write mode
- Fix ObjectType to support tuple (extension, data) format for streams
- Fix ObjectType to pass through pre-computed metadata for staged inserts
- Fix staged_insert.py path handling (use relative paths consistently)
- Fix table.py __make_placeholder to handle None values for adapter types
- Update schema_object.py to use <object> syntax (angle brackets required)
- Remove legacy external table support (Table.external property)
- Remove legacy external tests (test_filepath, test_external_class, test_s3)
- Add tests for save_template() method

Test results: 471 passed, 2 skipped

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <[email protected]>

Author: dimitri-yatsenko

.gitignore

@@ -187,3 +187,4 @@ dj_local_conf.json
 !.vscode/launch.json
 # pixi environments
 .pixi
+_content/

src/datajoint/builtin_types.py

@@ -345,23 +345,42 @@ def encode(
         field = key.pop("_field", "data")
         primary_key = {k: v for k, v in key.items() if not k.startswith("_")}
 
+        # Check for pre-computed metadata (from staged insert)
+        if isinstance(value, dict) and "path" in value:
+            # Already encoded, pass through
+            return value
+
         # Determine content type and extension
         is_dir = False
         ext = None
         size = None
+        item_count = None
+        source_path = None
 
         if isinstance(value, bytes):
             content = value
             size = len(content)
+        elif isinstance(value, tuple) and len(value) == 2:
+            # Tuple format: (extension, data) where data is bytes or file-like
+            ext, data = value
+            if hasattr(data, "read"):
+                content = data.read()
+            else:
+                content = data
+            size = len(content)
         elif isinstance(value, (str, Path)):
             source_path = Path(value)
             if not source_path.exists():
-                raise FileNotFoundError(f"Source path does not exist: {source_path}")
+                from .errors import DataJointError
+
+                raise DataJointError(f"Source path not found: {source_path}")
             is_dir = source_path.is_dir()
             ext = source_path.suffix if not is_dir else None
             if is_dir:
                 # For directories, we'll upload later
                 content = None
+                # Count items in directory
+                item_count = sum(1 for _ in source_path.rglob("*") if _.is_file())
             else:
                 content = source_path.read_bytes()
                 size = len(content)
@@ -398,6 +417,7 @@ def encode(
             "size": size,
             "ext": ext,
             "is_dir": is_dir,
+            "item_count": item_count,
             "timestamp": timestamp.isoformat(),
         }
 
@@ -422,12 +442,19 @@ def decode(self, stored: dict, *, key: dict | None = None) -> Any:
         return ObjectRef.from_json(stored, backend=backend)
 
     def validate(self, value: Any) -> None:
-        """Validate that value is bytes or a valid path."""
+        """Validate that value is bytes, path, dict metadata, or (extension, data) tuple."""
         from pathlib import Path
 
         if isinstance(value, bytes):
             return
         if isinstance(value, (str, Path)):
+            # Could be a path or pre-encoded JSON string
+            return
+        if isinstance(value, tuple) and len(value) == 2:
+            # Tuple format: (extension, data)
+            return
+        if isinstance(value, dict) and "path" in value:
+            # Pre-computed metadata dict (from staged insert)
             return
         raise TypeError(f"<object> expects bytes or path, got {type(value).__name__}")
 

src/datajoint/content_registry.py

@@ -55,19 +55,17 @@ def get_store_backend(store_name: str | None = None) -> StorageBackend:
     Get a StorageBackend for content storage.
 
     Args:
-        store_name: Name of the store to use. If None, uses the default store.
+        store_name: Name of the store to use. If None, uses the default object storage
+            configuration or the configured default_store.
 
     Returns:
         StorageBackend instance
     """
-    if store_name is None:
-        # Use default store from object_storage settings
+    # If store_name is None, check for configured default_store
+    if store_name is None and config.object_storage.default_store:
         store_name = config.object_storage.default_store
-        if store_name is None:
-            raise DataJointError(
-                "No default store configured. Set object_storage.default_store " "or specify a store name explicitly."
-            )
 
+    # get_object_store_spec handles None by returning default object_storage config
     spec = config.get_object_store_spec(store_name)
     return StorageBackend(spec)
 

src/datajoint/settings.py

@@ -212,6 +212,7 @@ class ObjectStorageSettings(BaseSettings):
     secure: bool = Field(default=True, description="Use HTTPS")
 
     # Optional settings
+    default_store: str | None = Field(default=None, description="Default store name when not specified")
     partition_pattern: str | None = Field(default=None, description="Path pattern with {attribute} placeholders")
     token_length: int = Field(default=8, ge=4, le=16, description="Random suffix length for filenames")
 
@@ -647,6 +648,118 @@ def override(self, **kwargs: Any) -> Iterator["Config"]:
                     group, attr = key_parts
                     setattr(getattr(self, group), attr, original)
 
+    @staticmethod
+    def save_template(
+        path: str | Path = "datajoint.json",
+        minimal: bool = True,
+        create_secrets_dir: bool = True,
+    ) -> Path:
+        """
+        Create a template datajoint.json configuration file.
+
+        Credentials should NOT be stored in datajoint.json. Instead, use either:
+        - Environment variables (DJ_USER, DJ_PASS, DJ_HOST, etc.)
+        - The .secrets/ directory (created alongside datajoint.json)
+
+        Args:
+            path: Where to save the template. Defaults to 'datajoint.json' in current directory.
+            minimal: If True (default), create a minimal template with just database settings.
+                    If False, create a full template with all available settings.
+            create_secrets_dir: If True (default), also create a .secrets/ directory
+                               with template files for credentials.
+
+        Returns:
+            Path to the created config file.
+
+        Raises:
+            FileExistsError: If config file already exists (won't overwrite).
+
+        Example:
+            >>> import datajoint as dj
+            >>> dj.config.save_template()  # Creates minimal template + .secrets/
+            >>> dj.config.save_template("full-config.json", minimal=False)
+        """
+        filepath = Path(path)
+        if filepath.exists():
+            raise FileExistsError(f"File already exists: {filepath}. Remove it first or choose a different path.")
+
+        if minimal:
+            template = {
+                "database": {
+                    "host": "localhost",
+                    "port": 3306,
+                },
+            }
+        else:
+            template = {
+                "database": {
+                    "host": "localhost",
+                    "port": 3306,
+                    "reconnect": True,
+                    "use_tls": None,
+                },
+                "connection": {
+                    "init_function": None,
+                    "charset": "",
+                },
+                "display": {
+                    "limit": 12,
+                    "width": 14,
+                    "show_tuple_count": True,
+                },
+                "object_storage": {
+                    "project_name": None,
+                    "protocol": None,
+                    "location": None,
+                    "bucket": None,
+                    "endpoint": None,
+                    "secure": True,
+                    "partition_pattern": None,
+                    "token_length": 8,
+                },
+                "stores": {},
+                "loglevel": "INFO",
+                "safemode": True,
+                "fetch_format": "array",
+                "enable_python_native_blobs": True,
+                "cache": None,
+                "query_cache": None,
+                "download_path": ".",
+            }
+
+        with open(filepath, "w") as f:
+            json.dump(template, f, indent=2)
+            f.write("\n")
+
+        logger.info(f"Created template configuration at {filepath.absolute()}")
+
+        # Create .secrets/ directory with template files
+        if create_secrets_dir:
+            secrets_dir = filepath.parent / SECRETS_DIRNAME
+            secrets_dir.mkdir(exist_ok=True)
+
+            # Create placeholder secret files
+            secret_templates = {
+                "database.user": "your_username",
+                "database.password": "your_password",
+            }
+            for secret_name, placeholder in secret_templates.items():
+                secret_file = secrets_dir / secret_name
+                if not secret_file.exists():
+                    secret_file.write_text(placeholder)
+
+            # Create .gitignore to prevent committing secrets
+            gitignore_path = secrets_dir / ".gitignore"
+            if not gitignore_path.exists():
+                gitignore_path.write_text("# Never commit secrets\n*\n!.gitignore\n")
+
+            logger.info(
+                f"Created {SECRETS_DIRNAME}/ directory with credential templates. "
+                f"Edit the files in {secrets_dir.absolute()}/ to set your credentials."
+            )
+
+        return filepath.absolute()
+
     # Dict-like access for convenience
     def __getitem__(self, key: str) -> Any:
         """Get setting by dot-notation key (e.g., 'database.host')."""
@@ -724,7 +837,7 @@ def _create_config() -> Config:
     else:
         warnings.warn(
             f"No {CONFIG_FILENAME} found. Using defaults and environment variables. "
-            f"Create {CONFIG_FILENAME} in your project root to configure DataJoint.",
+            f"Run `dj.config.save_template()` to create a template configuration.",
             stacklevel=2,
         )
 

src/datajoint/staged_insert.py

@@ -114,9 +114,8 @@ def _get_storage_path(self, field: str, ext: str = "") -> str:
         spec = config.get_object_storage_spec()
         partition_pattern = spec.get("partition_pattern")
         token_length = spec.get("token_length", 8)
-        location = spec.get("location", "")
 
-        # Build storage path
+        # Build storage path (relative - StorageBackend will add location prefix)
         relative_path, token = build_object_path(
             schema=self._table.database,
             table=self._table.class_name,
@@ -127,18 +126,14 @@ def _get_storage_path(self, field: str, ext: str = "") -> str:
             token_length=token_length,
         )
 
-        # Full path with location prefix
-        full_path = f"{location}/{relative_path}" if location else relative_path
-
-        # Store staged object info
+        # Store staged object info (all paths are relative, backend adds location)
         self._staged_objects[field] = {
             "relative_path": relative_path,
-            "full_path": full_path,
             "ext": ext if ext else None,
             "token": token,
         }
 
-        return full_path
+        return relative_path
 
     def store(self, field: str, ext: str = "") -> fsspec.FSMap:
         """
@@ -180,11 +175,12 @@ def _compute_metadata(self, field: str) -> dict:
             JSON-serializable metadata dict
         """
         info = self._staged_objects[field]
-        full_path = info["full_path"]
+        relative_path = info["relative_path"]
         ext = info["ext"]
 
         # Check if it's a directory (multiple files) or single file
-        full_remote_path = self._backend._full_path(full_path)
+        # _full_path adds the location prefix
+        full_remote_path = self._backend._full_path(relative_path)
 
         try:
             is_dir = self._backend.fs.isdir(full_remote_path)
@@ -218,11 +214,11 @@ def _compute_metadata(self, field: str) -> dict:
             }
 
             # Write manifest alongside folder
-            manifest_path = f"{full_path}.manifest.json"
+            manifest_path = f"{relative_path}.manifest.json"
             self._backend.put_buffer(json.dumps(manifest, indent=2).encode(), manifest_path)
 
             metadata = {
-                "path": info["relative_path"],
+                "path": relative_path,
                 "size": total_size,
                 "hash": None,
                 "ext": ext,
@@ -233,12 +229,12 @@ def _compute_metadata(self, field: str) -> dict:
         else:
             # Single file
             try:
-                size = self._backend.size(full_path)
+                size = self._backend.size(relative_path)
             except Exception:
                 size = 0
 
             metadata = {
-                "path": info["relative_path"],
+                "path": relative_path,
                 "size": size,
                 "hash": None,
                 "ext": ext,
@@ -261,8 +257,8 @@ def _finalize(self):
         # Process each staged object
         for field in list(self._staged_objects.keys()):
             metadata = self._compute_metadata(field)
-            # Store JSON metadata in the record
-            self._rec[field] = json.dumps(metadata)
+            # Store metadata dict in the record (ObjectType.encode handles it)
+            self._rec[field] = metadata
 
         # Insert the record
         self._table.insert1(self._rec)
@@ -275,15 +271,15 @@ def _cleanup(self):
             return
 
         for field, info in self._staged_objects.items():
-            full_path = info["full_path"]
+            relative_path = info["relative_path"]
             try:
                 # Check if it's a directory
-                full_remote_path = self._backend._full_path(full_path)
+                full_remote_path = self._backend._full_path(relative_path)
                 if self._backend.fs.exists(full_remote_path):
                     if self._backend.fs.isdir(full_remote_path):
-                        self._backend.remove_folder(full_path)
+                        self._backend.remove_folder(relative_path)
                     else:
-                        self._backend.remove(full_path)
+                        self._backend.remove(relative_path)
             except Exception:
                 pass  # Best effort cleanup
 

src/datajoint/storage.py

@@ -279,7 +279,7 @@ def _create_filesystem(self) -> fsspec.AbstractFileSystem:
 
     def _full_path(self, path: str | PurePosixPath) -> str:
         """
-        Construct full path including bucket for cloud storage.
+        Construct full path including location/bucket prefix.
 
         Args:
             path: Relative path within the storage location
@@ -290,12 +290,21 @@ def _full_path(self, path: str | PurePosixPath) -> str:
         path = str(path)
         if self.protocol == "s3":
             bucket = self.spec["bucket"]
+            location = self.spec.get("location", "")
+            if location:
+                return f"{bucket}/{location}/{path}"
             return f"{bucket}/{path}"
         elif self.protocol in ("gcs", "azure"):
             bucket = self.spec.get("bucket") or self.spec.get("container")
+            location = self.spec.get("location", "")
+            if location:
+                return f"{bucket}/{location}/{path}"
             return f"{bucket}/{path}"
         else:
-            # Local filesystem - path is already absolute or relative to cwd
+            # Local filesystem - prepend location if specified
+            location = self.spec.get("location", "")
+            if location:
+                return str(Path(location) / path)
             return path
 
     def put_file(self, local_path: str | Path, remote_path: str | PurePosixPath, metadata: dict | None = None):
@@ -446,6 +455,11 @@ def open(self, remote_path: str | PurePosixPath, mode: str = "rb"):
             File-like object
         """
         full_path = self._full_path(remote_path)
+
+        # For write modes on local filesystem, ensure parent directory exists
+        if self.protocol == "file" and "w" in mode:
+            Path(full_path).parent.mkdir(parents=True, exist_ok=True)
+
         return self.fs.open(full_path, mode)
 
     def put_folder(self, local_path: str | Path, remote_path: str | PurePosixPath) -> dict: