create smaller initial files to reduce memory usage

Author: sokra

turbopack/crates/turbo-persistence/src/constants.rs

@@ -6,13 +6,13 @@ pub const MAX_MEDIUM_VALUE_SIZE: usize = 64 * 1024 * 1024;
 pub const MAX_SMALL_VALUE_SIZE: usize = 64 * 1024 - 1;
 
 /// Maximum number of entries per SST file
-pub const MAX_ENTRIES_PER_INITIAL_FILE: usize = 1024 * 1024;
+pub const MAX_ENTRIES_PER_INITIAL_FILE: usize = 256 * 1024;
 
 /// Maximum number of entries per SST file
 pub const MAX_ENTRIES_PER_COMPACTED_FILE: usize = 1024 * 1024;
 
 /// Finish file when total amount of data exceeds this
-pub const DATA_THRESHOLD_PER_INITIAL_FILE: usize = 256 * 1024 * 1024;
+pub const DATA_THRESHOLD_PER_INITIAL_FILE: usize = 64 * 1024 * 1024;
 
 /// Finish file when total amount of data exceeds this
 pub const DATA_THRESHOLD_PER_COMPACTED_FILE: usize = 256 * 1024 * 1024;

turbopack/crates/turbo-tasks-backend/src/database/turbo.rs

@@ -13,8 +13,8 @@ use crate::database::{
     write_batch::{BaseWriteBatch, ConcurrentWriteBatch, WriteBatch, WriteBuffer},
 };
 
-const COMPACT_MAX_COVERAGE: f32 = 10.0;
-const COMPACT_MAX_MERGE_SEQUENCE: usize = 16;
+const COMPACT_MAX_COVERAGE: f32 = 20.0;
+const COMPACT_MAX_MERGE_SEQUENCE: usize = 64;
 const COMPACT_MAX_MERGE_SIZE: usize = 512 * 1024 * 1024; // 512 MiB
 
 pub struct TurboKeyValueDatabase {