Skip to content

feat: Add HDFS support #5245

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 9, 2022
Merged

feat: Add HDFS support #5245

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6b0469e
feat: Add HDFS support
Xuanwo May 9, 2022
68004a3
Format toml
Xuanwo May 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 31 additions & 8 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 4 additions & 5 deletions common/base/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,14 @@ hyper = "0.14.18"
libc = { version = "0.2", optional = true }
parking_lot = "0.12.0"
poem = { version = "=1.3.16", features = ["rustls"] }
serde = { version = "1.0.136", features = ["derive"] }
serde_json = { version = "1.0.79", default-features = false, features = ["raw_value"] }
serde_yaml = { version = "0.8.23", default-features = false }
# Rollback to tagged release after 666d9e2 included in new release.
pprof = { git = "https://github.com/tikv/pprof-rs", rev = "666d9e2", features = [
pprof = { version = "0.8.0", features = [
"flamegraph",
"protobuf-codec",
"protobuf",
] }
serde = { version = "1.0.136", features = ["derive"] }
serde_json = { version = "1.0.79", default-features = false, features = ["raw_value"] }
serde_yaml = { version = "0.8.23", default-features = false }
tikv-jemalloc-ctl = { version = "0.4", optional = true }
tikv-jemalloc-sys = { version = "0.4.3" }
tokio = { version = "1.17.0", features = ["full"] }
Expand Down
2 changes: 1 addition & 1 deletion common/contexts/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@ test = false
common-base = { path = "../base" }

async-trait = "0.1.53"
opendal = { version = "0.6.0", features = ["retry"] }
opendal = { version = "0.6.1", features = ["retry"] }
time = "0.3.9"
2 changes: 1 addition & 1 deletion common/io/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ chrono = "0.4.19"
chrono-tz = "0.6.1"
futures = "0.3.21"
lexical-core = "0.8.2"
opendal = { version = "0.6.0", features = ["retry"] }
opendal = { version = "0.6.1", features = ["retry"] }
serde = { version = "1.0.136", features = ["derive"] }
time = "0.3.9"

Expand Down
2 changes: 1 addition & 1 deletion common/streams/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,4 @@ serde_json = { version = "1.0.79", default-features = false, features = ["preser
tempfile = "3.3.0"

[dev-dependencies]
opendal = { version = "0.6.0", features = ["retry"] }
opendal = { version = "0.6.1", features = ["retry"] }
3 changes: 2 additions & 1 deletion query/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ default = ["simd"]
simd = ["common-arrow/simd"]
tokio-console = ["common-tracing/console", "common-base/tracing"]
memory-profiling = ["common-base/memory-profiling", "tempfile"]
storage-hdfs = ["opendal/services-hdfs"]

[dependencies]
# Workspace dependencies
Expand Down Expand Up @@ -86,7 +87,7 @@ num = "0.4.0"
num_cpus = "1.13.1"
octocrab = "0.15.4"
once_cell = "1.10.0"
opendal = { version = "0.6.0", features = ["retry"] }
opendal = { version = "0.6.1", features = ["retry"] }
openssl = { version = "0.10", features = ["vendored"] }
paste = "1.0.7"
petgraph = "0.6.0"
Expand Down
18 changes: 18 additions & 0 deletions query/src/configs/config_storage.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,19 @@ impl fmt::Debug for AzblobStorageConfig {
}
}

#[derive(Clone, PartialEq, Serialize, Deserialize, Default, Args, Debug)]
#[serde(default)]
pub struct HdfsConfig {
#[clap(long = "storage-hdfs-name-node", default_value_t)]
pub name_node: String,
/// # TODO(xuanwo)
///
/// Clap doesn't allow us to use root directly.
#[clap(long = "storage-hdfs-root", default_value_t)]
#[serde(rename = "root")]
pub hdfs_root: String,
}

/// Storage config group.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Args)]
#[serde(default)]
Expand All @@ -185,6 +198,10 @@ pub struct StorageConfig {
// azure storage blob config.
#[clap(flatten)]
pub azblob: AzblobStorageConfig,

// hdfs storage backend config
#[clap(flatten)]
pub hdfs: HdfsConfig,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we don't add the HdfsConfig item to *.toml, the config deserialize looks will crash?

Copy link
Member

@BohuTANG BohuTANG May 9, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is another problem, we don't want unused config items explicitly:

[storage]
# fs|s3
type = "s3"

[storage.fs] -- this config

[storage.s3]
bucket = "databend"
endpoint_url = "https://s3.amazonaws.com"
access_key_id = "<your-key-id>"
secret_access_key = "<your-access-key>"

[storage.azblob] -- this config

Is it possible to configure the item only we used, like [storage.s3] here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we don't add the HdfsConfig item to *.toml, the config deserialize looks will crash?

If HdfsConfig is not added in *.toml, we will use the default value instead.

I tested this behavior locally: query is able to start without adding hdfs-related staff.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's great, I am out, thank you.
I will remove all the unused config items from the documents.

}

impl Default for StorageConfig {
Expand All @@ -196,6 +213,7 @@ impl Default for StorageConfig {
fs: FsStorageConfig::default(),
s3: S3StorageConfig::default(),
azblob: AzblobStorageConfig::default(),
hdfs: HdfsConfig::default(),
}
}
}
19 changes: 19 additions & 0 deletions query/src/sessions/session_mgr.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ use futures::future::Either;
use futures::StreamExt;
use opendal::services::azblob;
use opendal::services::fs;
#[cfg(feature = "storage-hdfs")]
use opendal::services::hdfs;
use opendal::services::memory;
use opendal::services::s3;
use opendal::Accessor;
Expand Down Expand Up @@ -371,6 +373,23 @@ impl SessionManager {

fs::Backend::build().root(&path).finish().await?
}
#[cfg(feature = "storage-hdfs")]
DalSchema::Hdfs => {
let conf = &storage_conf.hdfs;
let mut builder = hdfs::Backend::build();

// Endpoint.
{
builder.name_node(&conf.name_node);
}

// Root
{
builder.root(&conf.hdfs_root);
}

builder.finish().await?
}
DalSchema::S3 => {
let s3_conf = &storage_conf.s3;
let mut builder = s3::Backend::build();
Expand Down
8 changes: 8 additions & 0 deletions query/tests/it/configs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,10 @@ account_key = ""
container = ""
endpoint_url = ""
root = ""

[storage.hdfs]
name_node = ""
root = ""
"#;

let tom_actual = toml::to_string(&actual).unwrap();
Expand Down Expand Up @@ -266,6 +270,10 @@ account_name = ""
account_key = ""
container = ""
endpoint_url = ""
root = ""

[storage.hdfs]
name_node = ""
root = ""
"#
.as_bytes(),
Expand Down
4 changes: 4 additions & 0 deletions query/tests/it/storages/system/configs_table.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ async fn test_configs_table() -> Result<()> {
"| storage | azblob.endpoint_url | | |",
"| storage | azblob.root | | |",
"| storage | fs.data_path | _data | |",
"| storage | hdfs.name_node | | |",
"| storage | hdfs.root | | |",
"| storage | num_cpus | 0 | |",
"| storage | s3.access_key_id | | |",
"| storage | s3.bucket | | |",
Expand Down Expand Up @@ -177,6 +179,8 @@ async fn test_configs_table_redact() -> Result<()> {
"| storage | azblob.endpoint_url | | |",
"| storage | azblob.root | | |",
"| storage | fs.data_path | _data | |",
"| storage | hdfs.name_node | | |",
"| storage | hdfs.root | | |",
"| storage | num_cpus | 0 | |",
"| storage | s3.access_key_id | ******_id | |",
"| storage | s3.bucket | | |",
Expand Down