diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 12ab0a3c3..596811fcb 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -246,7 +246,8 @@ def __init__( - ``lz4``: LZ4 compression. - ``lz4_raw``: LZ4_RAW compression. - ``zstd(n)``: Zstandard compression with level n. - compression_level: Compression level to set. + compression_level: Compression level to set, overriding any level in + ``compression``. dictionary_enabled: Sets if dictionary encoding is enabled. If ``None``, uses the default parquet writer setting. dictionary_page_size_limit: Sets best effort maximum dictionary page size, @@ -292,12 +293,23 @@ def __init__( column_specific_options: Overrides options for specific columns. If a column is not a part of this dictionary, it will use the parameters provided here. + + Examples: + >>> ParquetWriterOptions().compression + 'zstd(3)' + >>> ParquetWriterOptions(compression_level=6).compression + 'zstd(6)' + >>> opts = ParquetWriterOptions(compression="gzip(9)", compression_level=6) + >>> opts.compression + 'gzip(6)' """ self.data_pagesize_limit = data_pagesize_limit self.write_batch_size = write_batch_size self.writer_version = writer_version self.skip_arrow_metadata = skip_arrow_metadata if compression_level is not None: + if compression is not None: + compression = compression.split("(", 1)[0] self.compression = f"{compression}({compression_level})" else: self.compression = compression diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index 1b2bbfcd9..7b822cc73 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -3069,6 +3069,58 @@ def test_write_parquet_options(df, tmp_path): assert result == expected +@pytest.mark.parametrize("use_options_method", [False, True]) +@pytest.mark.parametrize( + ("compression", "level", "codec"), + [ + (None, 6, "zstd"), + ("zstd(3)", 6, "zstd"), + ("gzip(9)", 6, "gzip"), + ("brotli(7)", 4, "brotli"), + ("gzip", 6, "gzip"), + ], +) +def test_write_parquet_options_override_compression_level( + df, tmp_path, compression, level, codec, use_options_method +): + kwargs = {} if compression is None else {"compression": compression} + options = ParquetWriterOptions(**kwargs, compression_level=level) + if use_options_method: + df.write_parquet_with_options(tmp_path, options) + else: + df.write_parquet(tmp_path, options) + + files = list(tmp_path.rglob("*.parquet")) + assert files + for file in files: + metadata = pq.ParquetFile(file).metadata + for row_group in range(metadata.num_row_groups): + for column_index in range(metadata.num_columns): + assert ( + metadata.row_group(row_group) + .column(column_index) + .compression.lower() + == codec + ) + assert pq.read_table(tmp_path).to_pydict() == df.to_pydict() + + +@pytest.mark.parametrize( + ("compression", "level"), + [("gzip(6)", 12), ("brotli(7)", 15), ("zstd(3)", 23)], +) +def test_write_parquet_options_override_invalid_level(df, tmp_path, compression, level): + options = ParquetWriterOptions(compression=compression, compression_level=level) + with pytest.raises(Exception, match=r"valid compression range .*? exceeded."): + df.write_parquet_with_options(tmp_path, options) + + +def test_write_parquet_options_level_without_codec(df, tmp_path): + options = ParquetWriterOptions(compression=None, compression_level=6) + with pytest.raises(Exception, match="Unknown or unsupported parquet compression"): + df.write_parquet_with_options(tmp_path, options) + + def test_write_parquet_options_error(df, tmp_path): options = ParquetWriterOptions(compression="gzip", compression_level=6) with pytest.raises(ValueError):