Skip to content

fix: Correct null_count in describe() #10260

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Apr 30, 2024
Merged

fix: Correct null_count in describe() #10260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f47ff6c
fix: Correct null_count in describe()
Weijun-H Apr 27, 2024
eeebda4
chore: fix fmt
Weijun-H Apr 27, 2024
98784c5
chore: Fix ci
Weijun-H Apr 27, 2024
3a22207
fix: Update comment
Weijun-H Apr 27, 2024
de67422
fix: refactor null_count calculation in describe() and add test
Weijun-H Apr 30, 2024
ec5fe18
chore
Weijun-H Apr 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions datafusion/core/src/dataframe/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,12 @@ use datafusion_common::config::{CsvOptions, FormatOptions, JsonOptions};
use datafusion_common::{
plan_err, Column, DFSchema, DataFusionError, ParamValues, SchemaError, UnnestOptions,
};
use datafusion_expr::lit;
use datafusion_expr::{
avg, count, is_null, max, median, min, stddev, utils::COUNT_STAR_EXPANSION,
avg, count, max, median, min, stddev, utils::COUNT_STAR_EXPANSION,
TableProviderFilterPushDown, UNNAMED_TABLE,
};
use datafusion_expr::{case, is_null, sum};

use async_trait::async_trait;

Expand Down Expand Up @@ -534,7 +536,13 @@ impl DataFrame {
vec![],
original_schema_fields
.clone()
.map(|f| count(is_null(col(f.name()))).alias(f.name()))
.map(|f| {
sum(case(is_null(col(f.name())))
.when(lit(true), lit(1))
.otherwise(lit(0))
.unwrap())
.alias(f.name())
})
.collect::<Vec<_>>(),
),
// mean aggregation
Expand Down
34 changes: 32 additions & 2 deletions datafusion/core/tests/dataframe/describe.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ async fn describe() -> Result<()> {
"| describe | id | bool_col | tinyint_col | smallint_col | int_col | bigint_col | float_col | double_col | date_string_col | string_col | timestamp_col | year | month |",
"+------------+-------------------+----------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+-----------------+------------+-------------------------+--------------------+-------------------+",
"| count | 7300.0 | 7300 | 7300.0 | 7300.0 | 7300.0 | 7300.0 | 7300.0 | 7300.0 | 7300 | 7300 | 7300 | 7300.0 | 7300.0 |",
"| null_count | 7300.0 | 7300 | 7300.0 | 7300.0 | 7300.0 | 7300.0 | 7300.0 | 7300.0 | 7300 | 7300 | 7300 | 7300.0 | 7300.0 |",
"| null_count | 0.0 | 0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0 | 0 | 0 | 0.0 | 0.0 |",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that certainly looks better

"| mean | 3649.5 | null | 4.5 | 4.5 | 4.5 | 45.0 | 4.949999964237213 | 45.45 | null | null | null | 2009.5 | 6.526027397260274 |",
"| std | 2107.472815166704 | null | 2.8724780750809518 | 2.8724780750809518 | 2.8724780750809518 | 28.724780750809533 | 3.1597258182544645 | 29.012028558317645 | null | null | null | 0.5000342500942125 | 3.44808750051728 |",
"| min | 0.0 | null | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 01/01/09 | 0 | 2008-12-31T23:00:00 | 2009.0 | 1.0 |",
Expand Down Expand Up @@ -69,7 +69,7 @@ async fn describe_boolean_binary() -> Result<()> {
"| describe | a | b |",
"+------------+------+------+",
"| count | 1 | 1 |",
"| null_count | 1 | 1 |",
"| null_count | 0 | 0 |",
"| mean | null | null |",
"| std | null | null |",
"| min | a | null |",
Expand All @@ -81,6 +81,36 @@ async fn describe_boolean_binary() -> Result<()> {
Ok(())
}

#[tokio::test]
async fn describe_null() -> Result<()> {
let ctx = parquet_context().await;

//add test case for only boolean boolean/binary column
let result = ctx
.sql("select 'a' as a, null as b")
.await?
.describe()
.await?
.collect()
.await?;
#[rustfmt::skip]
let expected = [
"+------------+------+------+",
"| describe | a | b |",
"+------------+------+------+",
"| count | 1 | 0 |",
"| null_count | 0 | 1 |",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❤️ -- thank you @Weijun-H

"| mean | null | null |",
"| std | null | null |",
"| min | null | null |",
"| max | null | null |",
"| median | null | null |",
"+------------+------+------+"
];
assert_batches_eq!(expected, &result);
Ok(())
}

/// Return a SessionContext with parquet file registered
async fn parquet_context() -> SessionContext {
let ctx = SessionContext::new();
Expand Down