Merge pull request #55 from BehindTheMusicTree/feature/include-all-metadata-in-get-full-metadata

feature: include all metadata in get full metadata

Author: Andreas-Garcia

.cursor/rules/mastodon-posts.mdc

@@ -0,0 +1,16 @@
+---
+description: Mastodon and social announcement post length (494 char max)
+alwaysApply: true
+---
+
+# Mastodon / Social Announcement Posts
+
+When drafting **Mastodon posts** or **release/social announcement text** for this project:
+
+- **Maximum length: 494 characters** (including spaces, links, and hashtags).
+- Keep the post under this limit so it fits common instance limits and stays readable.
+- Prefer a short, scannable version with links (e.g. PyPI, GitHub) over a long post; link to changelog/README for details.
+
+If asked for "with links", include:
+- PyPI: https://pypi.org/project/audiometa-python/
+- GitHub: https://github.com/BehindTheMusicTree/audiometa

CHANGELOG.md

@@ -48,6 +48,10 @@ All contributors (including maintainers) should update `CHANGELOG.md` when creat
 
 ## [Unreleased]
 
+### Added
+
+- **get_full_metadata raw_metadata returns all tags**: `raw_metadata` now consistently includes every tag present in the file per format. RIFF INFO extractor and \_RiffManager no longer filter by `RiffTagKey`—every INFO chunk FourCC (known or custom) is returned in `raw_metadata["riff"]["parsed_fields"]`, with a parsing guard that only accepts valid 4-byte printable-ASCII FourCCs. Integration coverage for custom ID3v2 TXXX, custom Vorbis comments, ID3v1 parsed fields, custom RIFF FourCCs, and BWF bext; test helpers for custom TXXX and custom RIFF INFO fields. Documentation updated for raw-metadata guarantees.
+
 ## [1.1.0] - 2025-02-23
 
 ### Added

audiometa/__init__.py

@@ -1277,6 +1277,11 @@ def get_full_metadata(
     - Raw metadata details from each format. When include_raw_binary_data is False (default),
       binary/opaque frames (e.g. APIC, PRIV, TRAKTOR4) are summarized as size placeholders.
 
+    The result's raw_metadata contains every tag present in the file for each format:
+    ID3v2 (all frames including custom TXXX/PRIV), Vorbis (all comments), ID3v1 (fixed set),
+    RIFF (all INFO chunk FourCCs including custom), and BWF bext in chunk_structure.
+    Per-format keys: id3v2.frames, vorbis.comments, id3v1/riff.parsed_fields, riff.chunk_structure.bext.
+
     Args:
         file: Audio file path (str or Path)
         include_headers: Whether to include format-specific header information (default: True)

audiometa/manager/_rating_supporting/riff/_RiffManager.py

@@ -95,6 +95,10 @@ class _RiffManager(_RatingSupportingMetadataManager):
     Note: This manager is the preferred way to handle WAV metadata, as it uses the format's native metadata system
     rather than non-standard alternatives like ID3v2 tags. The custom implementation ensures proper handling of RIFF
     chunk structures, maintaining word alignment and size fields according to the specification.
+
+    Raw metadata: get_raw_metadata_info() (and thus get_full_metadata()["raw_metadata"]["riff"]) includes every
+    INFO chunk FourCC present in the file in parsed_fields (no filtering by RiffTagKey). Only subchunks with
+    a valid 4-byte printable-ASCII FourCC are included; custom and known FourCCs are both returned.
     """
 
     class RiffTagKey(RawMetadataKey):
@@ -193,7 +197,7 @@ def _extract_riff_metadata_directly(self, file_data: bytes) -> dict[str, list[st
 
         This method directly parses the RIFF structure to extract metadata from the INFO chunk.
         """
-        return extract_riff_metadata_directly(file_data, self._skip_id3v2_tags, self.RiffTagKey)
+        return extract_riff_metadata_directly(file_data, self._skip_id3v2_tags)
 
     def _extract_bext_chunk(self, file_data: bytes) -> dict[str, Any] | None:
         """Extract and parse the bext chunk from BWF files."""
@@ -253,10 +257,8 @@ def _convert_raw_mutagen_metadata_to_dict_with_potential_duplicate_keys(
         if hasattr(raw_mutagen_metadata_wav, "info") and raw_mutagen_metadata_wav.info is not None:
             info_tags = raw_mutagen_metadata_wav.info
             for key, value in info_tags.items():
-                # key is a FourCC string; check against enum member values
-                if any(key == member.value for member in self.RiffTagKey.__members__.values()):
-                    # info_tags now contains lists of values, so we can pass them directly
-                    raw_metadata_dict[key] = value
+                # info_tags contains lists of values; include every FourCC (known and custom)
+                raw_metadata_dict[key] = value
 
         return raw_metadata_dict
 

audiometa/manager/_rating_supporting/riff/_riff_info_chunk.py

@@ -5,78 +5,66 @@
 """
 
 from collections.abc import Callable
-from typing import TYPE_CHECKING, cast
 
 from ....utils.types import RawMetadataKey
+from ._riff_constants import RIFF_CHUNK_ID_SIZE, RIFF_HEADER_SIZE, RIFF_WAVE_FORMAT_POSITION
 
-if TYPE_CHECKING:
-    pass
+_FOURCC_MIN = 0x20
+_FOURCC_MAX = 0x7E
 
-from ._riff_constants import RIFF_CHUNK_ID_SIZE, RIFF_HEADER_SIZE, RIFF_WAVE_FORMAT_POSITION
+
+def _is_valid_fourcc(fourcc_bytes: bytes) -> bool:
+    """Return True if the 4 bytes form a valid INFO chunk FourCC (printable ASCII)."""
+    if len(fourcc_bytes) != RIFF_CHUNK_ID_SIZE:
+        return False
+    return all(_FOURCC_MIN <= b <= _FOURCC_MAX for b in fourcc_bytes)
 
 
 def extract_riff_metadata_directly(
-    file_data: bytes, skip_id3v2_tags_func: Callable[[bytes], bytes], riff_tag_key_class: type[object]
+    file_data: bytes, skip_id3v2_tags_func: Callable[[bytes], bytes]
 ) -> dict[str, list[str]]:
     """Manually extract metadata from RIFF chunks without relying on external libraries.
 
-    This function directly parses the RIFF structure to extract metadata from the INFO chunk.
-
-    Args:
-        file_data: Full file data (may include ID3v2 tags)
-        skip_id3v2_tags_func: Function to skip ID3v2 tags from file data
-        riff_tag_key_class: RiffTagKey class for validation
-
-    Returns:
-        Dictionary mapping RIFF tag IDs to lists of values
+    Parses the RIFF structure and returns every INFO chunk field. No filtering by
+    RiffTagKey—known and custom FourCCs are included. Only subchunks with a valid
+    4-byte printable-ASCII FourCC are accepted.
     """
     info_tags: dict[str, list[str]] = {}
 
-    # Skip ID3v2 if present
     file_data = skip_id3v2_tags_func(file_data)
 
-    # Validate RIFF header
     if (
         len(file_data) < RIFF_HEADER_SIZE
         or file_data[:RIFF_CHUNK_ID_SIZE] != b"RIFF"
         or file_data[RIFF_WAVE_FORMAT_POSITION:RIFF_HEADER_SIZE] != b"WAVE"
     ):
         return info_tags
 
-    pos = 12  # Start after RIFF header
+    pos = 12
     while pos < len(file_data) - 8:
         chunk_id = file_data[pos : pos + 4]
         chunk_size = int.from_bytes(file_data[pos + 4 : pos + 8], "little")
 
         if chunk_id == b"LIST" and pos + 12 <= len(file_data) and file_data[pos + 8 : pos + 12] == b"INFO":
-            # Process INFO chunk
             info_pos = pos + 12
             info_end = pos + 8 + chunk_size
 
             while info_pos < info_end - 8:
-                # Extract each metadata field
-                field_id = file_data[info_pos : info_pos + 4].decode("ascii", errors="ignore")
+                field_id_bytes = file_data[info_pos : info_pos + 4]
                 field_size = int.from_bytes(file_data[info_pos + 4 : info_pos + 8], "little")
 
-                if field_size > 0 and info_pos + 8 + field_size <= info_end:
-                    # -1 to exclude null terminator
+                if field_size > 0 and info_pos + 8 + field_size <= info_end and _is_valid_fourcc(field_id_bytes):
+                    field_id = field_id_bytes.decode("ascii")
                     field_data = file_data[info_pos + 8 : info_pos + 8 + field_size - 1]
                     try:
-                        # Decode and handle null-terminated strings
-                        field_value = field_data.decode("utf-8", errors="ignore")
-                        # Split on null byte and take first part if exists
-                        field_value = field_value.split("\x00")[0].strip()
-                        # Compare field_id with enum member values (FourCC strings)
-                        # Use getattr to safely access __members__ for type checking
-                        members = getattr(riff_tag_key_class, "__members__", {})
-                        if any(field_id == member.value for member in cast(dict, members).values()) and field_value:
+                        field_value = field_data.decode("utf-8", errors="ignore").split("\x00")[0].strip()
+                        if field_value:
                             if field_id not in info_tags:
                                 info_tags[field_id] = []
                             info_tags[field_id].append(field_value)
                     except UnicodeDecodeError:
                         pass
 
-                # Move to next field, maintaining alignment
                 info_pos += 8 + ((field_size + 1) & ~1)
             break
 

audiometa/test/helpers/id3v2/id3v2_metadata_setter.py

@@ -667,3 +667,14 @@ def set_ufid_with_owner(file_path: Path, owner: str, data: str) -> None:
 
         data_bytes = data.encode("utf-8")
         ManualID3v2FrameCreator.create_ufid_frame(file_path, owner, data_bytes, version="2.4")
+
+    @staticmethod
+    def set_custom_txxx(file_path: Path, description: str, value: str) -> None:
+        """Set a custom TXXX frame (for testing raw_metadata includes unsupported frames)."""
+        command = [
+            get_tool_path("mid3v2"),
+            "--TXXX",
+            f"{description}:{value}",
+            str(file_path),
+        ]
+        run_external_tool(command, "mid3v2")

audiometa/test/helpers/riff/riff_manual_metadata_creator.py

@@ -342,3 +342,10 @@ def _remove_existing_info_chunk(data: bytes) -> bytes:
                 pos = chunk_end
 
         return bytes(result)
+
+    @staticmethod
+    def add_custom_info_field(file_path: Path, fourcc: str, value: str) -> None:
+        """Append a custom INFO chunk field (FourCC + value) for testing raw_metadata includes all tags."""
+        existing_fields = ManualRIFFMetadataCreator._read_existing_info_fields(file_path)
+        new_field = ManualRIFFMetadataCreator._create_info_field(fourcc, value)
+        ManualRIFFMetadataCreator._write_riff_info_chunk(file_path, [*existing_fields, new_field])

audiometa/test/tests/integration/get_full_metadata/test_raw_metadata_includes_unsupported_tags.py

@@ -0,0 +1,80 @@
+"""Tests that get_full_metadata raw_metadata includes tags per format.
+
+ID3v2 and Vorbis expose all frames/comments (including custom/unsupported).
+ID3v1 has no extensible tags. RIFF INFO exposes all INFO chunk FourCCs (known and custom).
+BWF (bext) is exposed under RIFF chunk_structure.
+"""
+
+import pytest
+
+from audiometa import get_full_metadata
+from audiometa.test.helpers.id3v2.id3v2_metadata_setter import ID3v2MetadataSetter
+from audiometa.test.helpers.riff.riff_manual_metadata_creator import ManualRIFFMetadataCreator
+from audiometa.test.helpers.riff.riff_metadata_setter import RIFFMetadataSetter
+from audiometa.test.helpers.temp_file_with_metadata import temp_file_with_metadata
+from audiometa.test.helpers.vorbis.vorbis_metadata_setter import VorbisMetadataSetter
+from audiometa.utils.unified_metadata_key import UnifiedMetadataKey
+
+
+@pytest.mark.integration
+class TestGetFullMetadataRawMetadataIncludesUnsupportedTags:
+    def test_id3v2_raw_metadata_includes_custom_txxx_frame(self):
+        with temp_file_with_metadata({"title": "Track"}, "mp3") as test_file:
+            ID3v2MetadataSetter.set_custom_txxx(test_file, "MYCUSTOMKEY", "custom value")
+            result = get_full_metadata(test_file)
+            frames = result.get("raw_metadata", {}).get("id3v2", {}).get("frames", {})
+            assert "TXXX:MYCUSTOMKEY" in frames
+            frame_data = frames["TXXX:MYCUSTOMKEY"]
+            assert "text" in frame_data
+            assert "custom value" in frame_data["text"]
+
+    def test_vorbis_raw_metadata_includes_custom_comment(self):
+        with temp_file_with_metadata({"title": "Track"}, "flac") as test_file:
+            VorbisMetadataSetter.set_tag(test_file, "MYCUSTOMKEY", "custom value")
+            result = get_full_metadata(test_file)
+            comments = result.get("raw_metadata", {}).get("vorbis", {}).get("comments", {})
+            assert "MYCUSTOMKEY" in comments
+            assert comments["MYCUSTOMKEY"] == ["custom value"]
+
+    def test_id3v1_raw_metadata_includes_parsed_fields(self):
+        with temp_file_with_metadata(
+            {"title": "ID3v1 Title", "artist": "ID3v1 Artist"},
+            "id3v1",
+        ) as test_file:
+            result = get_full_metadata(test_file)
+            parsed = result.get("raw_metadata", {}).get("id3v1", {}).get("parsed_fields", {})
+            assert UnifiedMetadataKey.TITLE in parsed
+            assert parsed[UnifiedMetadataKey.TITLE] == "ID3v1 Title"
+            assert UnifiedMetadataKey.ARTISTS in parsed
+            assert parsed[UnifiedMetadataKey.ARTISTS] == "ID3v1 Artist"
+
+    def test_riff_raw_metadata_includes_known_info_tags(self):
+        with temp_file_with_metadata(
+            {"title": "RIFF Title", "artist": "RIFF Artist"},
+            "wav",
+        ) as test_file:
+            result = get_full_metadata(test_file)
+            parsed = result.get("raw_metadata", {}).get("riff", {}).get("parsed_fields", {})
+            assert "INAM" in parsed
+            assert parsed["INAM"] == "RIFF Title"
+            assert "IART" in parsed
+            assert parsed["IART"] == "RIFF Artist"
+
+    def test_riff_raw_metadata_includes_custom_fourcc(self):
+        with temp_file_with_metadata({}, "wav") as test_file:
+            ManualRIFFMetadataCreator.add_custom_info_field(test_file, "CUST", "custom value")
+            result = get_full_metadata(test_file)
+            parsed = result.get("raw_metadata", {}).get("riff", {}).get("parsed_fields", {})
+            assert "CUST" in parsed
+            assert parsed["CUST"] == "custom value"
+
+    def test_riff_raw_metadata_includes_bext_in_chunk_structure(self):
+        with temp_file_with_metadata({}, "wav") as test_file:
+            RIFFMetadataSetter.set_bext_description(test_file, "BWF Description")
+            RIFFMetadataSetter.set_bext_originator(test_file, "BWF Originator")
+            result = get_full_metadata(test_file)
+            chunk_structure = result.get("raw_metadata", {}).get("riff", {}).get("chunk_structure", {})
+            assert "bext" in chunk_structure
+            bext = chunk_structure["bext"]
+            assert bext["Description"] == "BWF Description"
+            assert bext["Originator"] == "BWF Originator"

audiometa/test/tests/unit/metadata_managers/header_info/test_riff_info_chunk_fourcc.py

@@ -0,0 +1,70 @@
+"""Unit tests for RIFF INFO chunk FourCC validation."""
+
+import pytest
+
+from audiometa.manager._rating_supporting.riff._riff_info_chunk import _is_valid_fourcc, extract_riff_metadata_directly
+
+
+@pytest.mark.unit
+class TestRiffInfoChunkFourCCValidation:
+    @pytest.mark.parametrize(
+        "fourcc_bytes",
+        [
+            b"INAM",
+            b"IART",
+            b"CUST",
+            b"    ",
+            b"0123",
+            b"AB\x20\x7e",
+        ],
+    )
+    def test_valid_fourcc_accepted(self, fourcc_bytes: bytes):
+        assert _is_valid_fourcc(fourcc_bytes) is True
+
+    @pytest.mark.parametrize(
+        "fourcc_bytes",
+        [
+            b"IN\x00M",
+            b"\x00NAM",
+            b"INAM\x00",
+            b"\x1fABC",
+            b"AB\x7f\x43",
+            b"",
+            b"I",
+            b"INA",
+            b"INAMX",
+        ],
+    )
+    def test_invalid_fourcc_rejected(self, fourcc_bytes: bytes):
+        assert _is_valid_fourcc(fourcc_bytes) is False
+
+    def test_extract_includes_valid_fourcc_excludes_invalid(self):
+        def no_skip(data: bytes) -> bytes:
+            return data
+
+        # Minimal RIFF WAVE with LIST INFO: one valid (INAM) and one invalid (IN\x00M) subchunk
+        inam_data = b"Title\x00"
+        invalid_data = b"X\x00"
+        inam_size = len(inam_data)
+        invalid_size = len(invalid_data)
+        if inam_size % 2:
+            inam_size += 1
+        if invalid_size % 2:
+            invalid_size += 1
+        list_payload = b"INFO" b"INAM" + inam_size.to_bytes(4, "little") + inam_data.ljust(
+            inam_size, b"\x00"
+        ) + b"IN\x00M" + invalid_size.to_bytes(4, "little") + invalid_data.ljust(invalid_size, b"\x00")
+        list_size = len(list_payload)
+        if list_size % 2:
+            list_payload += b"\x00"
+            list_size += 1
+        riff_body_size = 4 + 8 + list_size
+        riff = (
+            b"RIFF" + riff_body_size.to_bytes(4, "little") + b"WAVE"
+            b"LIST" + list_size.to_bytes(4, "little") + list_payload
+        )
+        result = extract_riff_metadata_directly(riff, no_skip)
+        assert "INAM" in result
+        assert result["INAM"] == ["Title"]
+        for key in result:
+            assert key == "INAM", f"Only INAM should appear, got key {key!r}"