Export collection and set compression

Author: m-mohr

fiboa_cli/conversion/duckdb.py

@@ -67,12 +67,11 @@ def convert(
         else:
             sources = "[" + ",".join([f'"{url}"' for url in urls]) + "]"
 
-        _collection = self.create_collection(cid)
-        _collection.update(self.column_additions)
-        _collection["collection"] = self.id
-        collection = json.dumps(_collection, cls=VecorelJSONEncoder).encode("utf-8")
+        collection = self.create_collection(cid)
+        collection.update(self.column_additions)
+        collection["collection"] = self.id
 
-        schemas = _collection.merge_schemas({})
+        schemas = collection.merge_schemas({})
         props = schemas.get("properties", {})
         required = schemas.get("required", [])
         pq_fields = []
@@ -91,10 +90,12 @@ def convert(
         if isinstance(output_file, Path):
             output_file = str(output_file)
 
-        pq_schema = pa.schema(pq_fields)
-        schema_bytes = pq_schema.serialize().to_pybytes()
+        # pq_schema = pa.schema(pq_fields)
+        # schema_bytes = pq_schema.serialize().to_pybytes()
         # pq_schema = pq_schema.with_metadata({"collection": collection})
 
+        collection_json = json.dumps(collection, cls=VecorelJSONEncoder).encode("utf-8")
+
         con = duckdb.connect()
         con.install_extension("spatial")
         con.load_extension("spatial")
@@ -106,14 +107,13 @@ def convert(
               ORDER BY ST_Hilbert({geom_column})
             ) TO ? (
                 FORMAT parquet,
-                compression 'brotli',
+                compression '{compression}',
                 KV_METADATA {{
                     collection: ?,
-                    "PYARROW:schema": ?
                 }}
             )
         """,
-            [output_file, collection, schema_bytes],
+            [output_file, collection_json],
         )
 
         return output_file