fix: convert all number data types to float

server checks if event has any number (all ints and floats)
then update the data type of all numbers to Float64

Author: nikhilsinhaparseable

src/event/format/json.rs

@@ -184,7 +184,9 @@ fn valid_type(data_type: &DataType, value: &Value) -> bool {
         DataType::Boolean => value.is_boolean(),
         DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => value.is_i64(),
         DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => value.is_u64(),
-        DataType::Float16 | DataType::Float32 | DataType::Float64 => value.is_f64(),
+        DataType::Float16 | DataType::Float32 | DataType::Float64 => {
+            value.is_f64() || value.is_i64() || value.is_u64()
+        }
         DataType::Utf8 => value.is_string(),
         DataType::List(field) => {
             let data_type = field.data_type();

src/event/format/mod.rs

@@ -204,13 +204,41 @@ pub fn override_timestamp_fields(
     Arc::new(Schema::new(updated_fields))
 }
 
+pub fn override_num_fields_from_schema(schema: Arc<Schema>) -> Arc<Schema> {
+    Arc::new(Schema::new(
+        schema
+            .fields()
+            .iter()
+            .map(|field| {
+                if field.data_type() == &DataType::Int64
+                    || field.data_type() == &DataType::Int32
+                    || field.data_type() == &DataType::Int16
+                    || field.data_type() == &DataType::Int8
+                    || field.data_type() == &DataType::Float64
+                    || field.data_type() == &DataType::Float32
+                    || field.data_type() == &DataType::Float16
+                {
+                    Arc::new(Field::new(
+                        field.name(),
+                        DataType::Float64,
+                        field.is_nullable(),
+                    ))
+                } else {
+                    field.clone()
+                }
+            })
+            .collect::<Vec<Arc<Field>>>(),
+    ))
+}
+
 pub fn update_field_type_in_schema(
     inferred_schema: Arc<Schema>,
     existing_schema: Option<&HashMap<String, Arc<Field>>>,
     time_partition: Option<String>,
     log_records: Option<&Vec<Value>>,
 ) -> Arc<Schema> {
     let mut updated_schema = inferred_schema.clone();
+    updated_schema = override_num_fields_from_schema(updated_schema);
 
     if let Some(existing_schema) = existing_schema {
         let existing_fields = get_existing_fields(inferred_schema.clone(), Some(existing_schema));