Merge pull request #149 from DunklesArchipel/fix-dialect

Fix dialect

Author: DunklesArchipel

doc/conf.py

@@ -1,5 +1,5 @@
 project = 'unitpackage'
-copyright = '2022-2023, the unitpackage authors'
+copyright = '2022-2026, the unitpackage authors'
 author = 'the unitpackage authors'
 
 release = '0.12.0'
@@ -43,6 +43,10 @@
 
 # Ignore the link to the GNU General Public License v3.0
 # This is because checking results in a timeout.
+# Zenodo badge and record URLs are excluded because Zenodo's servers
+# block automated link checkers with 403 responses.
 linkcheck_ignore = [
     "https://www.gnu.org/licenses/gpl-3.0.html*",
+    "https://zenodo.org/badge/*",
+    "https://zenodo.org/records/*",
 ]

doc/news/fix-dialect.rst

@@ -0,0 +1,5 @@
+**Fixed:**
+
+* Fixed CSV-to-dataframe reconstruction from tabular resources to honor frictionless descriptor dialect and encoding metadata, avoiding silent misparsing for non-default delimiters.
+* Fixed the CSV loader API by replacing the ambiguous `delimiters` argument with explicit `delimiter` and `candidate_delimiters` parameters.
+

doc/usage/load_and_save.md

@@ -97,7 +97,8 @@ For CSV files with more complex structures, additional arguments can be provided
 - `header_lines` — number of header lines to skip before the data
 - `column_header_lines` — number of lines containing column headers (multiple lines are flattened and separated by ` / `)
 - `decimal` — decimal separator (e.g., `','` for European-style numbers)
-- `delimiters` — column delimiter (auto-detected if not specified)
+- `delimiter` — explicit column delimiter
+- `candidate_delimiters` — candidate delimiters used during autodetection
 - `encoding` — file encoding
 
 For example, a CSV with multiple header lines:
@@ -129,7 +130,7 @@ The loader automatically detects headers and delimiters. The resulting entry con
 entry.fields
 ```
 
-Information on the file structure is stored in the entry's metadata under `dsvDescription`:
+Information on detected file structure is available in entry metadata (for example under `dsvDescription`):
 
 ```{code-cell} ipython3
 entry.metadata['dsvDescription']['loader']

doc/usage/loaders.md

@@ -73,11 +73,15 @@ Multiple column headers will be flattened.
 from unitpackage.loaders.baseloader import BaseLoader
 csv = BaseLoader(file, header_lines=6,
                  column_header_lines=2,
-                 delimiters=None,
+                 delimiter="\t",
+                 candidate_delimiters=None,
                  decimal=None)
 csv.df
 ```
 
+To pin a known delimiter, pass `delimiter='\t'` (or any other separator).
+To restrict autodetection, pass `candidate_delimiters=['\t', ';']`.
+
 All parts of the file are accessible from the API for further use. For example the extraction of metadata from the header.
 
 ```{code-cell} ipython3

unitpackage/entry.py

@@ -1092,7 +1092,8 @@ def from_csv(  # pylint: disable=too-many-locals
         header_lines=None,
         column_header_lines=None,
         decimal=None,
-        delimiters=None,
+        delimiter=None,
+        candidate_delimiters=None,
         device=None,
     ):
         r"""
@@ -1105,6 +1106,9 @@ def from_csv(  # pylint: disable=too-many-locals
         A ``device`` can be specified to select a device-specific loader
         (e.g., ``'eclab'`` or ``'gamry'``).
 
+        ``candidate_delimiters`` can be used to restrict delimiter sniffing to
+        a known set of separators.
+
         EXAMPLES::
 
             >>> from unitpackage.entry import Entry
@@ -1135,6 +1139,18 @@ def from_csv(  # pylint: disable=too-many-locals
             [{'name': 'E / V', 'type': 'integer'},
             {'name': 'j / A / cm2', 'type': 'integer'}]
 
+        Candidate delimiters can be provided explicitly when parsing a file::
+
+            >>> import os
+            >>> import tempfile
+            >>> with tempfile.TemporaryDirectory() as tmpdir:
+            ...     filename = os.path.join(tmpdir, 'candidate_delimiters.csv')
+            ...     with open(filename, 'w', encoding='utf-8') as handle:
+            ...         _ = handle.write('a\tb\n1\t2\n')
+            ...     entry = Entry.from_csv(csvname=filename, candidate_delimiters=[';', '\t'])
+            >>> entry.metadata['dsvDescription']['delimiter']
+            '\t'
+
         A device-specific loader can be used to parse instrument files::
 
             >>> entry = Entry.from_csv(csvname='test/loader_data/eclab_cv.mpt', device='eclab')
@@ -1149,6 +1165,7 @@ def from_csv(  # pylint: disable=too-many-locals
 
             >>> entry.metadata['dsvDescription']['loader']
             'ECLabLoader'
+
             >>> entry.metadata['dsvDescription']['delimiter']
             '\t'
 
@@ -1162,7 +1179,8 @@ def from_csv(  # pylint: disable=too-many-locals
             "header_lines": header_lines,
             "column_header_lines": column_header_lines,
             "decimal": decimal,
-            "delimiters": delimiters,
+            "delimiter": delimiter,
+            "candidate_delimiters": candidate_delimiters,
         }
 
         loader_cls = BaseLoader.create(device) if device else BaseLoader

unitpackage/loaders/baseloader.py

@@ -48,6 +48,7 @@
 
 
 import logging
+from collections.abc import Iterable
 
 logger = logging.getLogger("loader")
 
@@ -94,21 +95,90 @@ class BaseLoader:
         0     2       0    0.1       0          0
         1     2       1    1.4       5          1
 
+    Candidate delimiters can be provided explicitly for autodetection.::
+
+        >>> from io import StringIO
+        >>> file = StringIO('''a\tb
+        ... 0\t0
+        ... 1\t1''')
+        >>> csv = BaseLoader(file, candidate_delimiters=[';', '\t'])
+        >>> csv.delimiter
+        '\t'
+
     """
 
+    DEFAULT_CANDIDATE_DELIMITERS = ("\t", ";", ",")
+    DELIMITER_SNIFF_SAMPLE_LINES = 25
+    _warned_default_candidate_delimiters = False
+
     def __init__(
         self,
         file,
         header_lines=None,
         column_header_lines=None,
         decimal=None,
-        delimiters=None,
+        delimiter=None,
+        candidate_delimiters=None,
     ):  # pylint: disable=dangerous-default-value
         self._file = file.read()
         self._header_lines = header_lines
         self._column_header_lines = column_header_lines
         self._decimal = decimal
-        self.delimiters = delimiters or ["\t", ";", ","]
+        self._delimiter = delimiter
+        self._candidate_delimiters = self._normalize_delimiter_candidates(
+            delimiter=delimiter,
+            candidate_delimiters=candidate_delimiters,
+        )
+
+    @staticmethod
+    def _normalize_delimiter_candidates(delimiter=None, candidate_delimiters=None):
+        r"""Return delimiter candidates normalized to a list of strings.
+
+        The public API separates the explicit delimiter from sniffing candidates:
+
+        - ``delimiter=","`` fixes the delimiter to a single value.
+        - ``candidate_delimiters=["\t", ";", ","]`` provides candidates for sniffing.
+
+        If ``delimiter`` is provided, ``candidate_delimiters`` must not be provided.
+
+        EXAMPLES::
+
+            >>> BaseLoader._normalize_delimiter_candidates(delimiter=',')
+            [',']
+
+            >>> BaseLoader._normalize_delimiter_candidates(candidate_delimiters=['\t', ';'])
+            ['\t', ';']
+
+            >>> BaseLoader._normalize_delimiter_candidates(delimiter=',', candidate_delimiters=[';'])
+            Traceback (most recent call last):
+            ...
+            ValueError: Use either 'delimiter' or 'candidate_delimiters', not both.
+        """
+        if delimiter is not None and candidate_delimiters is not None:
+            raise ValueError(
+                "Use either 'delimiter' or 'candidate_delimiters', not both."
+            )
+
+        if delimiter is not None:
+            return [delimiter]
+
+        if candidate_delimiters is None:
+            if not BaseLoader._warned_default_candidate_delimiters:
+                logger.warning(
+                    "No delimiter or candidate_delimiters were provided; using default candidate delimiters for sniffing."
+                )
+                BaseLoader._warned_default_candidate_delimiters = True
+            return list(BaseLoader.DEFAULT_CANDIDATE_DELIMITERS)
+
+        if isinstance(candidate_delimiters, str):
+            return [candidate_delimiters]
+
+        if isinstance(candidate_delimiters, Iterable):
+            return list(candidate_delimiters)
+
+        raise TypeError(
+            "'candidate_delimiters' must be a string or an iterable of strings."
+        )
 
     @property
     def file(self):
@@ -576,20 +646,102 @@ def delimiter(self):
             >>> csv.delimiter
             '\t'
 
+        Candidate delimiters are considered for sniffing even if the correct
+        delimiter is not the first candidate::
+
+            >>> from io import StringIO
+            >>> file = StringIO('''a\tb\n0\t0\n1\t1''')
+            >>> csv = BaseLoader(file, candidate_delimiters=[';', '\t', ','])
+            >>> csv.delimiter
+            '\t'
+
+        Inconsistent field counts between column headers and data rows are
+        reported early::
+
+            >>> from io import StringIO
+            >>> file = StringIO('''a,b\n0,0\n1,1,1''')
+            >>> csv = BaseLoader(file, delimiter=',')
+            >>> csv.delimiter
+            Traceback (most recent call last):
+            ...
+            ValueError: Inconsistent number of fields detected in data line 2: expected 2 based on column headers but found 3.
+
         """
-        # TODO:: Validate that the number of delimiters in the data lines
-        # matches those in the column header line.
-        # This will otherwise likely lead to erroneous loading of pandas dataframes
-        # and requires setting the column names specifically.
-        if len(self.delimiters) == 1:
-            return self.delimiters[0]
+        if self._delimiter is not None:
+            self._validate_delimiter_consistency(self._delimiter)
+            return self._delimiter
+
+        if len(self._candidate_delimiters) == 1:
+            delimiter = self._candidate_delimiters[0]
+            self._validate_delimiter_consistency(delimiter)
+            return delimiter
 
         import csv
         from io import StringIO
 
         combined = StringIO(self.column_headers.getvalue() + self.data.getvalue())
+        sample_lines = []
+        for _ in range(self.DELIMITER_SNIFF_SAMPLE_LINES):
+            line = combined.readline()
+            if not line:
+                break
+            sample_lines.append(line)
+
+        sample = "".join(sample_lines)
+        if not sample:
+            raise ValueError("Delimiter could not be determined from an empty sample.")
+
+        delimiter = csv.Sniffer().sniff(sample, self._candidate_delimiters).delimiter
+        self._validate_delimiter_consistency(delimiter)
+        return delimiter
+
+    def _validate_delimiter_consistency(self, delimiter):
+        r"""Validate that sampled data rows have the same field count as the
+        column headers. Returns ``True`` if all sampled rows are consistent.
+
+        EXAMPLES::
+
+            >>> from io import StringIO
+            >>> file = StringIO('''a,b\n0,0\n1,1''')
+            >>> csv = BaseLoader(file, delimiter=',')
+            >>> csv._validate_delimiter_consistency(',')
+            True
+
+            >>> from io import StringIO
+            >>> file = StringIO('''a,b\n0,0\n1,1,1''')
+            >>> csv = BaseLoader(file, delimiter=',')
+            >>> csv._validate_delimiter_consistency(',')
+            Traceback (most recent call last):
+            ...
+            ValueError: Inconsistent number of fields detected in data line 2: expected 2 based on column headers but found 3.
+
+        """
+        import csv
+
+        column_header_lines = self.column_headers.getvalue().splitlines()
+        if not column_header_lines:
+            return True
+
+        expected_fields = len(
+            next(csv.reader([column_header_lines[0]], delimiter=delimiter))
+        )
 
-        return csv.Sniffer().sniff(combined.readline(), self.delimiters).delimiter
+        for line_number, line in enumerate(
+            self.data.getvalue().splitlines()[: self.DELIMITER_SNIFF_SAMPLE_LINES],
+            start=1,
+        ):
+            if not line.strip():
+                continue
+
+            actual_fields = len(next(csv.reader([line], delimiter=delimiter)))
+            if actual_fields != expected_fields:
+                raise ValueError(
+                    "Inconsistent number of fields detected in data line "
+                    f"{line_number}: expected {expected_fields} based on "
+                    f"column headers but found {actual_fields}."
+                )
+
+        return True
 
     @property
     def decimal(self):