From 783ae74e863471cd10bbce4ed6bd9fa6fb759d9b Mon Sep 17 00:00:00 2001 From: Rusty Bee <145002912+rustybee42@users.noreply.github.com> Date: Thu, 21 Nov 2024 10:18:45 +0100 Subject: [PATCH 1/9] refactor/fix: overhaul BeeSerde and msg buffers * Serializer now requires a preallocated buffer in form of a &mut [u8]. This moves the responsibility to allocate the required memory to the user. Before it used a dynamically growing buffer (which would reallocate internally if needed). This change is crucial for future RDMA support. * Since we now no longer use `BytesMut`, remove the bytes crate. * The buffers handled by the store have a fixed size of 4 MiB. This is apparently the maximum size of a BeeMsg (see `WORKER_BUF(IN|OUT)_SIZE` in `Worker.h`). C++ server code also uses these fixed size. * Generalize `msg_feature_flags` to a `Header` that can be modified from within the `Serializable` implementation and can be read out from within the `Deserializable` implementation. * Collect all BeeMsg (de)serialization functions in one module and provide functions for header, body and both combined. The split is required because depending on where the data comes from / goes to different actions need to be taken. This also provides an easy interface for potential external users to handle BeeMsges. * Remove the MsgBuf struct, instead just pass a `&mut [u8]` into the dispatcher. * Add documentation * Various small code cleanups in BeeSerde and other locations --- Cargo.lock | 1 - Cargo.toml | 1 - mgmtd/src/context.rs | 2 +- mgmtd/src/db/import_v7.rs | 10 +- mgmtd/src/lib.rs | 3 +- shared/Cargo.toml | 1 - shared/src/bee_msg.rs | 168 ++++++++++++++- shared/src/bee_msg/header.rs | 66 ------ shared/src/bee_serde.rs | 371 +++++++++++++++++--------------- shared/src/conn.rs | 14 +- shared/src/conn/incoming.rs | 43 ++-- shared/src/conn/msg_buf.rs | 180 ---------------- shared/src/conn/msg_dispatch.rs | 31 +-- shared/src/conn/outgoing.rs | 90 +++++--- shared/src/conn/store.rs | 13 +- shared/src/journald_logger.rs | 3 +- 16 files changed, 489 insertions(+), 508 deletions(-) delete mode 100644 shared/src/bee_msg/header.rs delete mode 100644 shared/src/conn/msg_buf.rs diff --git a/Cargo.lock b/Cargo.lock index 8493643..f855b5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1019,7 +1019,6 @@ version = "0.0.0" dependencies = [ "anyhow", "bee_serde_derive", - "bytes", "libc", "log", "pnet_datalink", diff --git a/Cargo.toml b/Cargo.toml index b405115..5824be9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,6 @@ publish = false [workspace.dependencies] anyhow = "1" -bytes = "1" clap = { version = "4", features = ["derive"] } env_logger = "0" itertools = "0" diff --git a/mgmtd/src/context.rs b/mgmtd/src/context.rs index 0662d38..ed7321a 100644 --- a/mgmtd/src/context.rs +++ b/mgmtd/src/context.rs @@ -4,8 +4,8 @@ use crate::bee_msg::dispatch_request; use crate::license::LicenseVerifier; use crate::{ClientPulledStateNotification, StaticInfo}; use anyhow::Result; -use shared::conn::Pool; use shared::conn::msg_dispatch::*; +use shared::conn::outgoing::Pool; use shared::run_state::WeakRunStateHandle; use shared::types::{NodeId, NodeType}; use std::ops::Deref; diff --git a/mgmtd/src/db/import_v7.rs b/mgmtd/src/db/import_v7.rs index 5f92ea7..7917b57 100644 --- a/mgmtd/src/db/import_v7.rs +++ b/mgmtd/src/db/import_v7.rs @@ -80,7 +80,7 @@ fn check_format_conf(f: &Path) -> Result<()> { fn check_target_states(f: &Path) -> Result<()> { let s = std::fs::read(f)?; - let mut des = Deserializer::new(&s, 0); + let mut des = Deserializer::new(&s); let states = des.map( false, |des| TargetId::deserialize(des), @@ -182,7 +182,7 @@ struct ReadNodesResult { fn read_nodes(f: &Path) -> Result { let s = std::fs::read(f)?; - let mut des = Deserializer::new(&s, 0); + let mut des = Deserializer::new(&s); let version = des.u32()?; let root_id = des.u32()?; let root_mirrored = des.u8()?; @@ -284,7 +284,7 @@ fn storage_targets(tx: &Transaction, targets_path: &Path) -> Result<()> { fn storage_pools(tx: &Transaction, f: &Path) -> Result<()> { let s = std::fs::read(f)?; - let mut des = Deserializer::new(&s, 0); + let mut des = Deserializer::new(&s); // Serialized as size_t, which should usually be 64 bit. let count = des.i64()?; let mut used_aliases = vec![]; @@ -410,7 +410,7 @@ fn quota_default_limits(tx: &Transaction, f: &Path, pool_id: PoolId) -> Result<( Err(err) => return Err(err.into()), }; - let mut des = Deserializer::new(&s, 0); + let mut des = Deserializer::new(&s); let user_inode_limit = des.u64()?; let user_space_limit = des.u64()?; let group_inode_limit = des.u64()?; @@ -492,7 +492,7 @@ fn quota_limits( Err(err) => return Err(err.into()), }; - let mut des = Deserializer::new(&s, 0); + let mut des = Deserializer::new(&s); let limits = des.seq(false, |des| QuotaEntry::deserialize(des))?; des.finish()?; diff --git a/mgmtd/src/lib.rs b/mgmtd/src/lib.rs index cb3eba8..e4aaf19 100644 --- a/mgmtd/src/lib.rs +++ b/mgmtd/src/lib.rs @@ -19,7 +19,8 @@ use bee_msg::notify_nodes; use db::node_nic::ReplaceNic; use license::LicenseVerifier; use shared::bee_msg::target::RefreshTargetStates; -use shared::conn::{Pool, incoming}; +use shared::conn::incoming; +use shared::conn::outgoing::Pool; use shared::nic::Nic; use shared::run_state::{self, RunStateControl}; use shared::types::{AuthSecret, MGMTD_UID, NicType, NodeId, NodeType}; diff --git a/shared/Cargo.toml b/shared/Cargo.toml index d79f1e0..7cc5984 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -11,7 +11,6 @@ publish.workspace = true bee_serde_derive = { path = "../bee_serde_derive" } anyhow = { workspace = true } -bytes = { workspace = true } libc = { workspace = true } log = { workspace = true } pnet_datalink = "0" diff --git a/shared/src/bee_msg.rs b/shared/src/bee_msg.rs index e995a94..b986949 100644 --- a/shared/src/bee_msg.rs +++ b/shared/src/bee_msg.rs @@ -2,12 +2,11 @@ use crate::bee_serde::*; use crate::types::*; -use anyhow::Result; +use anyhow::{Context, Result, anyhow}; use bee_serde_derive::BeeSerde; use std::collections::{HashMap, HashSet}; pub mod buddy_group; -pub mod header; pub mod misc; pub mod node; pub mod quota; @@ -41,3 +40,168 @@ impl OpsErr { pub const AGAIN: Self = Self(22); pub const UNKNOWN_POOL: Self = Self(30); } + +/// The BeeMsg header +#[derive(Clone, Debug, PartialEq, Eq, BeeSerde)] +pub struct Header { + /// Total length of the serialized message, including the header itself + msg_len: u32, + /// Sometimes used for additional message specific payload and/or serialization info + pub msg_feature_flags: u16, + /// Sometimes used for additional message specific payload and/or serialization info + pub msg_compat_feature_flags: u8, + /// Sometimes used for additional message specific payload and/or serialization info + pub msg_flags: u8, + /// Fixed value to identify a BeeMsg header (see MSG_PREFIX below) + msg_prefix: u64, + /// Uniquely identifies the message type as defined in the C++ codebase in NetMessageTypes.h + msg_id: MsgId, + /// Sometimes used for additional message specific payload and/or serialization info + pub msg_target_id: TargetId, + /// Sometimes used for additional message specific payload and/or serialization info + pub msg_user_id: u32, + /// Mirroring related information + pub msg_seq: u64, + /// Mirroring related information + pub msg_seq_done: u64, +} + +impl Header { + /// The serialized length of the header + pub const LEN: usize = 40; + /// Fixed value for identifying BeeMsges. In theory, this has some kind of version modifier + /// (thus the + 0), but it is unused + #[allow(clippy::identity_op)] + pub const MSG_PREFIX: u64 = (0x42474653 << 32) + 0; + + /// The total length of the serialized message + pub fn msg_len(&self) -> usize { + self.msg_len as usize + } + + /// The messages id + pub fn msg_id(&self) -> MsgId { + self.msg_id + } +} + +impl Default for Header { + fn default() -> Self { + Self { + msg_len: 0, + msg_feature_flags: 0, + msg_compat_feature_flags: 0, + msg_flags: 0, + msg_prefix: Self::MSG_PREFIX, + msg_id: 0, + msg_target_id: 0, + msg_user_id: 0, + msg_seq: 0, + msg_seq_done: 0, + } + } +} + +/// Serializes a BeeMsg body into the provided buffer. +/// +/// The data is written from the beginning of the slice, it's up to the caller to pass the correct +/// sub slice if space for the header should be reserved. +/// +/// # Return value +/// Returns the number of bytes written and the header modified by serialization function. +pub fn serialize_body(msg: &M, buf: &mut [u8]) -> Result<(usize, Header)> { + let mut ser = Serializer::new(buf); + msg.serialize(&mut ser) + .context("BeeMsg body serialization failed")?; + + Ok((ser.bytes_written(), ser.finish())) +} + +/// Serializes a BeeMsg header into the provided buffer. +/// +/// # Return value +/// Returns the number of bytes written. +pub fn serialize_header(header: &Header, buf: &mut [u8]) -> Result { + let mut ser_header = Serializer::new(buf); + header + .serialize(&mut ser_header) + .context("BeeMsg header serialization failed")?; + + Ok(ser_header.bytes_written()) +} + +/// Serializes a complete BeeMsg (header + body) into the provided buffer. +/// +/// # Return value +/// Returns the number of bytes written. +pub fn serialize(msg: &M, buf: &mut [u8]) -> Result { + let (written, mut header) = serialize_body(msg, &mut buf[Header::LEN..])?; + + header.msg_len = (written + Header::LEN) as u32; + header.msg_id = M::ID; + + let _ = serialize_header(&header, &mut buf[0..Header::LEN])?; + + Ok(header.msg_len()) +} + +/// Deserializes a BeeMsg header from the provided buffer. +/// +/// # Return value +/// Returns the deserialized header. +pub fn deserialize_header(buf: &[u8]) -> Result
{ + const CTX: &str = "BeeMsg header deserialization failed"; + + let header_buf = buf + .get(..Header::LEN) + .ok_or_else(|| { + anyhow!( + "Header buffer must be at least {} bytes big, got {}", + Header::LEN, + buf.len() + ) + }) + .context(CTX)?; + + let mut des = Deserializer::new(header_buf); + let header = Header::deserialize(&mut des).context(CTX)?; + des.finish().context(CTX)?; + + if header.msg_prefix != Header::MSG_PREFIX { + return Err(anyhow!( + "Invalid BeeMsg prefix: Must be {}, got {}", + Header::MSG_PREFIX, + header.msg_prefix + )) + .context(CTX); + } + + Ok(header) +} + +/// Deserializes a BeeMsg body from the provided buffer. +/// +/// The data is read from the beginning of the slice, it's up to the caller to pass the correct +/// sub slice if space for the header should be excluded from the source. +/// +/// # Return value +/// Returns the deserialized message. +pub fn deserialize_body(header: &Header, buf: &[u8]) -> Result { + const CTX: &str = "BeeMsg body deserialization failed"; + + let mut des = Deserializer::with_header(&buf[0..(header.msg_len() - Header::LEN)], header); + let des_msg = M::deserialize(&mut des).context(CTX)?; + des.finish().context(CTX)?; + + Ok(des_msg) +} + +/// Deserializes a complete BeeMsg (header + body) from the provided buffer. +/// +/// # Return value +/// Returns the deserialized message. +pub fn deserialize(buf: &[u8]) -> Result { + let header = deserialize_header(&buf[0..Header::LEN])?; + let msg = deserialize_body(&header, &buf[Header::LEN..])?; + Ok(msg) +} diff --git a/shared/src/bee_msg/header.rs b/shared/src/bee_msg/header.rs deleted file mode 100644 index f704c95..0000000 --- a/shared/src/bee_msg/header.rs +++ /dev/null @@ -1,66 +0,0 @@ -/// Defines the BeeGFS message header -use super::*; -use crate::bee_serde::Deserializer; -use anyhow::bail; - -/// The BeeGFS message header -#[derive(Clone, Debug, Default, PartialEq, Eq, BeeSerde)] -pub struct Header { - /// Total length of the message, including the header. - /// - /// This determines the amount of bytes read and written from and to sockets. - pub msg_len: u32, - /// Sometimes used for additional message specific payload and/or serialization info - pub msg_feature_flags: u16, - pub msg_compat_feature_flags: u8, - pub msg_flags: u8, - /// Fixed value - pub msg_prefix: u64, - /// Uniquely identifies the message type as defined in the C++ codebase in NetMessageTypes.h - pub msg_id: MsgId, - pub msg_target_id: TargetId, - pub msg_user_id: u32, - pub msg_seq: u64, - pub msg_seq_done: u64, -} - -impl Header { - pub const LEN: usize = 40; - pub const DATA_VERSION: u64 = 0; - pub const MSG_PREFIX: u64 = (0x42474653 << 32) + Self::DATA_VERSION; - - /// Creates a new BeeGFS message header - /// - /// `msg_feature_flags` has to be set depending on the message. - pub fn new(body_len: usize, msg_id: MsgId, msg_feature_flags: u16) -> Self { - Self { - msg_len: (body_len + Self::LEN) as u32, - msg_feature_flags, - msg_compat_feature_flags: 0, - msg_flags: 0, - msg_prefix: Self::MSG_PREFIX, - msg_id, - msg_target_id: 0, - msg_user_id: u32::MAX, - msg_seq: 0, - msg_seq_done: 0, - } - } - - /// Deserializes the given buffer into a header - pub fn from_buf(buf: &[u8]) -> Result { - if buf.len() != Self::LEN { - bail!("Header buffer has an unexpected size of {}", buf.len()); - } - - let mut des = Deserializer::new(buf, 0); - let des_header = Header::deserialize(&mut des)?; - des.finish()?; - Ok(des_header) - } - - /// The expected total message length this header belongs to - pub fn msg_len(&self) -> usize { - self.msg_len as usize - } -} diff --git a/shared/src/bee_serde.rs b/shared/src/bee_serde.rs index 2c9f50d..14e24c5 100644 --- a/shared/src/bee_serde.rs +++ b/shared/src/bee_serde.rs @@ -1,105 +1,100 @@ -//! BeeGFS compatible network message (de-)serialization +//! BeeSerde, the BeeGFS network message (and some on disk data) (de-)serialization +use crate::bee_msg::Header; use anyhow::{Result, bail}; -use bytes::{Buf, BufMut, BytesMut}; +use std::borrow::Cow; use std::collections::HashMap; use std::hash::Hash; use std::marker::PhantomData; use std::mem::size_of; +// SERIALIZATION + +/// Makes a type BeeSerde serializable pub trait Serializable { fn serialize(&self, ser: &mut Serializer<'_>) -> Result<()>; } -pub trait Deserializable { - fn deserialize(des: &mut Deserializer<'_>) -> Result - where - Self: Sized; -} - -/// Provides conversion functionality to and from BeeSerde serializable types. -/// -/// Mainly meant for enums that need to be converted in to a raw integer type, which also might -/// differ between messages. The generic parameter allows implementing it for multiple types. -pub trait BeeSerdeConversion: Sized { - fn into_bee_serde(self) -> S; - fn try_from_bee_serde(value: S) -> Result; -} - -/// Interface for serialization helpers to be used with the `bee_serde` derive macro -/// -/// Serialization helpers are meant to control the `bee_serde` macro in case a value in the -/// message struct shall be serialized as a different type or in case it doesn't have its own -/// [BeeSerde] implementation. Also necessary for maps and sequences since the serializer can't -/// know on its own whether to include collection size or not (it's totally message dependent). -/// -/// # Example -/// -/// ```ignore -/// #[derive(Debug, BeeSerde)] -/// pub struct ExampleMsg { -/// // Serializer doesn't know by itself whether or not C/C++ BeeGFS serializer expects sequence -/// // size included or not - in this case it is not -/// #[bee_serde(as = Seq)] -/// int_sequence: Vec, -/// } -/// ``` -pub trait BeeSerdeHelper { - fn serialize_as(data: &In, ser: &mut Serializer<'_>) -> Result<()>; - fn deserialize_as(des: &mut Deserializer<'_>) -> Result; -} - -/// Serializes one BeeGFS message into a provided buffer +/// Serializes one `impl Serializable` into a target buffer +#[derive(Debug)] pub struct Serializer<'a> { /// The target buffer - target_buf: &'a mut BytesMut, - /// BeeGFS message feature flags obtained, used for conditional serialization by certain - /// messages. To be set by the serialization function. - pub msg_feature_flags: u16, - /// The number of bytes written to the buffer - bytes_written: usize, + target_buf: &'a mut [u8], + /// The position of the write cursor in the buffer. This equals to the number of bytes written. + write_pos: usize, + /// BeeMsg header, some fields are used for conditional serialization by certain + /// messages. To be set by the serialization function (except msg_len and msg_id). + // + // Note that in an ideal world, this would be generic and opaque as core bee_serde doesn't need + // to know about the type of this serialization metadata. But since it would require + // carrying the type everywhere (would make the code more complicated overall) we don't do + // it and accept a little coupling of core bee_serde to the BeeMsg header. It's almost only + // used for BeeMsg anyway (the header itself and v7 data import are the exceptions). + pub header: Header, } macro_rules! fn_serialize_primitive { - ($P:ident, $put_f:ident) => { + ($P:ident) => { pub fn $P(&mut self, v: $P) -> Result<()> { - self.target_buf.$put_f(v); - self.bytes_written += size_of::<$P>(); - Ok(()) + self.bytes(&v.to_le_bytes()) } }; } impl<'a> Serializer<'a> { - /// Creates a new Serializer object - /// - /// `msg_feature_flags` can be accessed from the (de-)serialization definition and is used for - /// conditional serialization on some messages. - /// `msg_feature_flags` is supposed to be obtained from the message definition, and is used - /// for conditional serialization by certain messages. - pub fn new(target_buf: &'a mut BytesMut) -> Self { + /// Creates a new Serializer object, writing into the given buffer. The buffer must be big + /// enough to take all the data. + pub fn with_header(buf: &'a mut [u8], header: Header) -> Self { Self { - target_buf, - msg_feature_flags: 0, - bytes_written: 0, + target_buf: buf, + write_pos: 0, + header, } } - fn_serialize_primitive!(u8, put_u8); - fn_serialize_primitive!(i8, put_i8); - fn_serialize_primitive!(u16, put_u16_le); - fn_serialize_primitive!(i16, put_i16_le); - fn_serialize_primitive!(u32, put_u32_le); - fn_serialize_primitive!(i32, put_i32_le); - fn_serialize_primitive!(u64, put_u64_le); - fn_serialize_primitive!(i64, put_i64_le); - fn_serialize_primitive!(u128, put_u128_le); - fn_serialize_primitive!(i128, put_i128_le); - - /// Serialize the given slice as bytes as expected by BeeGFS + /// Creates a new Serializer object with default header used as metadata. Meant for data that + /// does not do conditional serialization based on these fields (e.g. non BeeMsg or the + /// header itself). + pub fn new(buf: &'a mut [u8]) -> Self { + Self::with_header(buf, Header::default()) + } + + /// Finishes the serialization by consuming the serializer and returning the header + /// that might be set by certain BeeMsgs. + pub fn finish(self) -> Header { + self.header + } + + fn_serialize_primitive!(u8); + fn_serialize_primitive!(i8); + fn_serialize_primitive!(u16); + fn_serialize_primitive!(i16); + fn_serialize_primitive!(u32); + fn_serialize_primitive!(i32); + fn_serialize_primitive!(u64); + fn_serialize_primitive!(i64); + fn_serialize_primitive!(u128); + fn_serialize_primitive!(i128); + + /// Serialize the given slice as bytes. This is also the base operation for the other ops. pub fn bytes(&mut self, v: &[u8]) -> Result<()> { - self.target_buf.put(v); - self.bytes_written += v.len(); + match self + .target_buf + .get_mut(self.write_pos..(self.write_pos + v.len())) + { + Some(ref mut sub) => { + sub.clone_from_slice(v); + self.write_pos += v.len(); + } + None => { + bail!( + "Tried to write {} bytes but target buffer only has {} left", + v.len(), + self.target_buf.len() - self.write_pos + ); + } + } + Ok(()) } @@ -142,7 +137,7 @@ impl<'a> Serializer<'a> { include_total_size: bool, f: impl Fn(&mut Self, T) -> Result<()>, ) -> Result<()> { - let before = self.bytes_written; + let before = self.write_pos; // For the total size and length of the sequence we insert placeholders to be replaced // later when the values are known @@ -151,14 +146,14 @@ impl<'a> Serializer<'a> { // `BytesMut` and not the generic `BufMut` - the latter doesn't allow random access to // already written data let size_pos = if include_total_size { - let size_pos = self.bytes_written; + let size_pos = self.write_pos; self.u32(0xFFFFFFFFu32)?; size_pos } else { 0 }; - let count_pos = self.bytes_written; + let count_pos = self.write_pos; self.u32(0xFFFFFFFFu32)?; let mut count = 0u32; @@ -172,15 +167,13 @@ impl<'a> Serializer<'a> { // the placeholders in the beginning of the sequence with the actual values if include_total_size { - let written = (self.bytes_written - before) as u32; - for (p, b) in written.to_le_bytes().iter().enumerate() { - self.target_buf[size_pos + p] = *b; - } + let written = (self.write_pos - before) as u32; + self.target_buf[size_pos..(size_pos + size_of::())] + .clone_from_slice(&written.to_le_bytes()); } - for (p, b) in count.to_le_bytes().iter().enumerate() { - self.target_buf[count_pos + p] = *b; - } + self.target_buf[count_pos..(count_pos + size_of::())] + .clone_from_slice(&count.to_le_bytes()); Ok(()) } @@ -218,39 +211,56 @@ impl<'a> Serializer<'a> { Ok(()) } - /// The amount of bytes written to the buffer (so far) + /// The amount of bytes written to the buffer pub fn bytes_written(&self) -> usize { - self.bytes_written + self.write_pos } } -/// Deserializes one BeeGFS message from the given buffer +// DESERIALIZATION + +/// Makes a type BeeSerde deserializable +pub trait Deserializable { + fn deserialize(des: &mut Deserializer<'_>) -> Result + where + Self: Sized; +} + +/// Deserializes one `impl Deserializable` object from a source buffer pub struct Deserializer<'a> { /// The source buffer source_buf: &'a [u8], - /// BeeGFS message feature flags obtained from the message definition, used for - /// conditional deserialization by certain messages. - pub msg_feature_flags: u16, + /// BeeMsg header, used for conditional deserialization by certain messages. Can be + /// accessed from the deserialization definition. + pub header: Cow<'a, Header>, } macro_rules! fn_deserialize_primitive { - ($P:ident, $get_f:ident) => { + ($P:ident) => { pub fn $P(&mut self) -> Result<$P> { - self.check_remaining(size_of::<$P>())?; - Ok(self.source_buf.$get_f()) + let b = self.take(size_of::<$P>())?; + Ok($P::from_le_bytes(b.try_into()?)) } }; } impl<'a> Deserializer<'a> { - /// Creates a new Deserializer object - /// - /// `msg_feature_flags` is supposed to be obtained from the message definition, and is used - /// for conditional serialization by certain messages. - pub fn new(source_buf: &'a [u8], msg_feature_flags: u16) -> Self { + /// Creates a new Deserializer object with the given header used as metadata. Meant for BeeMsg - + /// they sometimes do conditional deserialization based on these fields. + pub fn with_header(buf: &'a [u8], header: &'a Header) -> Self { + Self { + source_buf: buf, + header: Cow::Borrowed(header), + } + } + + /// Creates a new Deserializer object with default header used as metadata. Meant for data that + /// does not do conditional deserialization based on these fields (e.g. non BeeMsg or the + /// header itself). + pub fn new(buf: &'a [u8]) -> Self { Self { - source_buf, - msg_feature_flags, + source_buf: buf, + header: Cow::Owned(Header::default()), } } @@ -265,25 +275,20 @@ impl<'a> Deserializer<'a> { Ok(()) } - fn_deserialize_primitive!(u8, get_u8); - fn_deserialize_primitive!(i8, get_i8); - fn_deserialize_primitive!(u16, get_u16_le); - fn_deserialize_primitive!(i16, get_i16_le); - fn_deserialize_primitive!(u32, get_u32_le); - fn_deserialize_primitive!(i32, get_i32_le); - fn_deserialize_primitive!(u64, get_u64_le); - fn_deserialize_primitive!(i64, get_i64_le); - fn_deserialize_primitive!(u128, get_u128_le); - fn_deserialize_primitive!(i128, get_i128_le); + fn_deserialize_primitive!(u8); + fn_deserialize_primitive!(i8); + fn_deserialize_primitive!(u16); + fn_deserialize_primitive!(i16); + fn_deserialize_primitive!(u32); + fn_deserialize_primitive!(i32); + fn_deserialize_primitive!(u64); + fn_deserialize_primitive!(i64); + fn_deserialize_primitive!(u128); + fn_deserialize_primitive!(i128); /// Deserialize a block of bytes as expected by BeeGFS pub fn bytes(&mut self, len: usize) -> Result> { - let mut v = vec![0; len]; - - self.check_remaining(len)?; - self.source_buf.copy_to_slice(&mut v); - - Ok(v) + Ok(self.take(len)?.to_owned()) } /// Deserialize a BeeGFS serialized c string @@ -293,11 +298,7 @@ impl<'a> Deserializer<'a> { /// don't. pub fn cstr(&mut self, align_to: usize) -> Result> { let len = self.u32()? as usize; - - let mut v = vec![0; len]; - - self.check_remaining(len)?; - self.source_buf.copy_to_slice(&mut v); + let v = self.take(len)?.to_owned(); let terminator: u8 = self.u8()?; if terminator != 0 { @@ -387,28 +388,61 @@ impl<'a> Deserializer<'a> { /// /// The opposite of fill_zeroes() in serialization. pub fn skip(&mut self, n: usize) -> Result<()> { - self.check_remaining(n)?; - self.source_buf.advance(n); - + self.take(n)?; Ok(()) } - /// Ensures that the source buffer has at least `n` bytes left - /// - /// Meant to check that there are enough bytes left before calling `Bytes` functions that would - /// panic otherwise (which we wan't to avoid) - fn check_remaining(&self, n: usize) -> Result<()> { - if self.source_buf.remaining() < n { - bail!( - "Unexpected end of source buffer. Needed at least {}, got {}", - n, - self.source_buf.remaining() - ); + /// Takes the next n bytes from the source buffer, checking that there are enough left. + fn take(&mut self, n: usize) -> Result<&[u8]> { + match self.source_buf.split_at_checked(n) { + Some((taken, rest)) => { + self.source_buf = rest; + Ok(taken) + } + None => { + bail!( + "Unexpected end of source buffer. Needed at least {n}, got {}", + self.source_buf.len() + ); + } } - Ok(()) } } +// HELPER / CONVENIENCE FUNCTIONS + +/// Provides conversion functionality to and from BeeSerde serializable types. +/// +/// Mainly meant for enums that need to be converted in to a raw integer type, which also might +/// differ between messages. The generic parameter allows implementing it for multiple types. +pub trait BeeSerdeConversion: Sized { + fn into_bee_serde(self) -> S; + fn try_from_bee_serde(value: S) -> Result; +} + +/// Interface for serialization helpers to be used with the `bee_serde` derive macro +/// +/// Serialization helpers are meant to control the `bee_serde` macro in case a value in the +/// message struct shall be serialized as a different type or in case it doesn't have its own +/// [BeeSerde] implementation. Also necessary for maps and sequences since the serializer can't +/// know on its own whether to include collection size or not (it's totally message dependent). +/// +/// # Example +/// +/// ```ignore +/// #[derive(Debug, BeeSerde)] +/// pub struct ExampleMsg { +/// // Serializer doesn't know by itself whether or not C/C++ BeeGFS serializer expects sequence +/// // size included or not - in this case it is not +/// #[bee_serde(as = Seq)] +/// int_sequence: Vec, +/// } +/// ``` +pub trait BeeSerdeHelper { + fn serialize_as(data: &In, ser: &mut Serializer<'_>) -> Result<()>; + fn deserialize_as(des: &mut Deserializer<'_>) -> Result; +} + /// Serialize an arbitrary type as Integer /// /// Note: Can potentially be used for non-integers, but is not practical due to the [Copy] @@ -530,7 +564,7 @@ mod test { #[test] fn primitives() { - let mut buf = BytesMut::new(); + let mut buf = vec![0; 1 + 1 + 2 + 2 + 4 + 4 + 8 + 8]; let mut ser = Serializer::new(&mut buf); ser.u8(123).unwrap(); @@ -542,10 +576,7 @@ mod test { ser.u64(0xAABBCCDDEEFF1122u64).unwrap(); ser.i64(-0x1ABBCCDDEEFF1122i64).unwrap(); - // 1 + 2 + 2 + 4 + 4 + 8 - assert_eq!(1 + 1 + 2 + 2 + 4 + 4 + 8 + 8, ser.bytes_written); - - let mut des = Deserializer::new(&buf, 0); + let mut des = Deserializer::new(&buf); assert_eq!(123, des.u8().unwrap()); assert_eq!(-123, des.i8().unwrap()); assert_eq!(22222, des.u16().unwrap()); @@ -561,17 +592,14 @@ mod test { #[test] fn bytes() { - let bytes: Vec = vec![0, 1, 2, 3, 4, 5]; - - let mut buf = BytesMut::new(); + let bytes = vec![0, 1, 2, 3, 4, 5]; + let mut buf = vec![0; 12]; let mut ser = Serializer::new(&mut buf); ser.bytes(&bytes).unwrap(); ser.bytes(&bytes).unwrap(); - assert_eq!(12, ser.bytes_written); - - let mut des = Deserializer::new(&buf, 0); + let mut des = Deserializer::new(&buf); assert_eq!(bytes, des.bytes(6).unwrap()); assert_eq!(bytes, des.bytes(6).unwrap()); @@ -581,23 +609,17 @@ mod test { #[test] fn cstr() { let str: Vec = "text".into(); - - let mut buf = BytesMut::new(); + // alignment applies to string length + null byte terminator + // Last one with align_to = 5 is intended and correct: Wrote 9 bytes, 9 % align_to = 1, + // align_to - 1 = 4 + let mut buf = vec![0; (4 + 4 + 1) + (4 + 4 + 1) + (4 + 4 + 1 + 4)]; let mut ser = Serializer::new(&mut buf); ser.cstr(&str, 0).unwrap(); ser.cstr(&str, 4).unwrap(); ser.cstr(&str, 5).unwrap(); - assert_eq!( - // alignment applies to string length + null byte terminator - // Last one with align_to = 5 is intended and correct: Wrote 9 bytes, 9 % align_to = 1, - // align_to - 1 = 4 - (4 + 4 + 1) + (4 + 4 + 1) + (4 + 4 + 1 + 4), - ser.bytes_written - ); - - let mut des = Deserializer::new(&buf, 0); + let mut des = Deserializer::new(&buf); assert_eq!(str, des.cstr(0).unwrap()); assert_eq!(str, des.cstr(4).unwrap()); assert_eq!(str, des.cstr(5).unwrap()); @@ -683,22 +705,19 @@ mod test { c2: HashMap::from([(18, vec!["aaa".into(), "bbbbb".into()])]), }; - let mut buf = BytesMut::new(); - - let mut ser = Serializer::new(&mut buf); - - s.serialize(&mut ser).unwrap(); - - assert_eq!( + let mut buf = vec![ + 0; 1 + 8 + (8 + 3 * 8) + (4 + 2 + 8) + (8 + (4 + 2 + 4)) - + (8 + (2 + (4 + (4 + 3 + 1) + (4 + 5 + 1)))), - ser.bytes_written - ); + + (8 + (2 + (4 + (4 + 3 + 1) + (4 + 5 + 1)))) + ]; - let mut des = Deserializer::new(&buf, 0); + let mut ser = Serializer::new(&mut buf); + s.serialize(&mut ser).unwrap(); + + let mut des = Deserializer::new(&buf); let s2 = S::deserialize(&mut des).unwrap(); @@ -708,23 +727,19 @@ mod test { #[test] fn wrong_buffer_len() { - let bytes: Vec = vec![0, 1, 2, 3, 4, 5]; + let mut buf = vec![0, 1, 2, 3, 4, 5]; - let mut buf = BytesMut::new(); let mut ser = Serializer::new(&mut buf); - ser.bytes(&bytes).unwrap(); + // Write too much + ser.u64(123).unwrap_err(); - let mut des = Deserializer::new(&buf, 0); + let mut des = Deserializer::new(&buf); des.bytes(5).unwrap(); - // Some buffer left des.finish().unwrap_err(); - // Consume too much des.bytes(2).unwrap_err(); - des.bytes(1).unwrap(); - // Complete buffer consumed des.finish().unwrap(); } diff --git a/shared/src/conn.rs b/shared/src/conn.rs index 840f7fe..559da65 100644 --- a/shared/src/conn.rs +++ b/shared/src/conn.rs @@ -2,11 +2,17 @@ mod async_queue; pub mod incoming; -mod msg_buf; pub mod msg_dispatch; -mod outgoing; +pub mod outgoing; mod store; mod stream; -pub use self::msg_buf::MsgBuf; -pub use outgoing::*; +/// Fixed length of the stream / TCP message buffers. +/// Must match the `WORKER_BUF(IN|OUT)_SIZE` value in `Worker.h` in the C++ +/// codebase. +const TCP_BUF_LEN: usize = 4 * 1024 * 1024; + +/// Fixed length of the datagram / UDP message buffers. +/// Must match the `DGRAMMR_(RECV|SEND)BUF_SIZE` value in `DatagramListener.*` in the C/C++ +/// codebase. Must be smaller than TCP_BUF_LEN; +const UDP_BUF_LEN: usize = 65536; diff --git a/shared/src/conn/incoming.rs b/shared/src/conn/incoming.rs index 5e1fd55..33cf52e 100644 --- a/shared/src/conn/incoming.rs +++ b/shared/src/conn/incoming.rs @@ -1,12 +1,12 @@ //! Handle incoming TCP and UDP connections and BeeMsgs. -use super::msg_buf::MsgBuf; use super::msg_dispatch::{DispatchRequest, SocketRequest, StreamRequest}; use super::stream::Stream; -use crate::bee_msg::Msg; +use super::*; use crate::bee_msg::misc::AuthenticateChannel; +use crate::bee_msg::{Header, Msg, deserialize_header}; use crate::run_state::RunStateHandle; -use anyhow::{Result, bail}; +use anyhow::{Context, Result, bail}; use std::io::{self, ErrorKind}; use std::net::SocketAddr; use std::sync::Arc; @@ -86,7 +86,7 @@ async fn stream_loop( log::debug!("Accepted incoming stream from {:?}", stream.addr()); // Use one owned buffer for reading into and writing from. - let mut buf = MsgBuf::default(); + let mut buf = vec![0; TCP_BUF_LEN]; loop { // Wait for available data or shutdown signal @@ -134,28 +134,41 @@ async fn stream_loop( /// handler and sending back a response using the [`StreamRequest`] handle. async fn read_stream( stream: &mut Stream, - buf: &mut MsgBuf, + buf: &mut [u8], dispatch: &impl DispatchRequest, stream_authentication_required: bool, ) -> Result<()> { - buf.read_from_stream(stream).await?; + // Read header + stream.read_exact(&mut buf[0..Header::LEN]).await?; + + let header = deserialize_header(&buf[0..Header::LEN])?; // check authentication if stream_authentication_required && !stream.authenticated - && buf.msg_id() != AuthenticateChannel::ID + && header.msg_id() != AuthenticateChannel::ID { bail!( "Stream is not authenticated and received message with id {}", - buf.msg_id() + header.msg_id() ); } + // Read body + stream + .read_exact(&mut buf[Header::LEN..header.msg_len()]) + .await?; + // Forward to the dispatcher. The dispatcher is responsible for deserializing, dispatching to // msg handlers and sending a response using the [`StreamRequest`] handle. dispatch - .dispatch_request(StreamRequest { stream, buf }) - .await?; + .dispatch_request(StreamRequest { + stream, + buf, + header: &header, + }) + .await + .context("Stream msg dispatch failed")?; Ok(()) } @@ -210,8 +223,11 @@ async fn recv_datagram(sock: Arc, msg_handler: impl DispatchRequest) // message spawns a new task (below) and we don't know how long the processing takes, we cannot // reuse Buffers like the TCP reader does. // A separate buffer pool could potentially be used to avoid allocating new buffers every time. - let mut buf = MsgBuf::default(); - let peer_addr = buf.recv_from_socket(&sock).await?; + let mut buf = vec![0; UDP_BUF_LEN]; + + let (_, peer_addr) = sock.recv_from(&mut buf).await?; + + let header = deserialize_header(&buf[0..Header::LEN])?; // Request shall be handled in a separate task, so the next datagram can be processed // immediately @@ -219,7 +235,8 @@ async fn recv_datagram(sock: Arc, msg_handler: impl DispatchRequest) let req = SocketRequest { sock, peer_addr, - msg_buf: &mut buf, + buf: &mut buf, + header: &header, }; // Forward to the dispatcher diff --git a/shared/src/conn/msg_buf.rs b/shared/src/conn/msg_buf.rs deleted file mode 100644 index bab3bff..0000000 --- a/shared/src/conn/msg_buf.rs +++ /dev/null @@ -1,180 +0,0 @@ -//! Reusable buffer for serialized BeeGFS messages -//! -//! This buffer provides the memory and the functionality to (de-)serialize BeeGFS messages from / -//! into and read / write / send / receive from / to streams and UDP sockets. -//! -//! They are meant to be used in two steps: -//! Serialize a message first, then write or send it to the wire. -//! OR -//! Read or receive data from the wire, then deserialize it into a message. -//! -//! # Example: Reading from stream -//! 1. `.read_from_stream()` to read in the data from stream into the buffer -//! 2. `.deserialize_msg()` to deserialize the message from the buffer -//! -//! # Important -//! If receiving data failed part way or didn't happen at all before calling `deserialize_msg`, the -//! buffer is in an invalid state. Deserializing will then most likely fail, or worse, succeed and -//! provide old or garbage data. The same applies for the opposite direction. It's up to the user to -//! make sure the buffer is used the appropriate way. -use super::stream::Stream; -use crate::bee_msg::header::Header; -use crate::bee_msg::{Msg, MsgId}; -use crate::bee_serde::{Deserializable, Deserializer, Serializable, Serializer}; -use anyhow::{Context, Result, bail}; -use bytes::BytesMut; -use std::net::SocketAddr; -use std::sync::Arc; -use tokio::net::UdpSocket; - -/// Fixed length of the datagrams to send and receive via UDP. -/// -/// Must match DGRAMMGR_*BUF_SIZE in `AbstractDatagramListener.h` (common) and `DatagramListener.h` -/// (client_module). -const DATAGRAM_LEN: usize = 65536; - -/// Reusable buffer for serialized BeeGFS messages -/// -/// See module level documentation for more information. -#[derive(Debug, Default)] -pub struct MsgBuf { - buf: BytesMut, - header: Box
, -} - -impl MsgBuf { - /// Serializes a BeeGFS message into the buffer - pub fn serialize_msg(&mut self, msg: &M) -> Result<()> { - self.buf.truncate(0); - - if self.buf.capacity() < Header::LEN { - self.buf.reserve(Header::LEN); - } - - // We need to serialize the body first since we need its total length for the header. - // Therefore, the body part (which comes AFTER the header) is split off to be passed as a - // separate BytesMut to the serializer. - let mut body = self.buf.split_off(Header::LEN); - - // Catching serialization errors to ensure buffer is unsplit afterwards in all cases - let res = (|| { - // Serialize body - let mut ser_body = Serializer::new(&mut body); - msg.serialize(&mut ser_body) - .context("BeeMsg body serialization failed")?; - - // Create and serialize header - let header = Header::new(ser_body.bytes_written(), M::ID, ser_body.msg_feature_flags); - let mut ser_header = Serializer::new(&mut self.buf); - header - .serialize(&mut ser_header) - .context("BeeMsg header serialization failed")?; - - *self.header = header; - - Ok(()) as Result<_> - })(); - - // Put header and body back together - self.buf.unsplit(body); - - res - } - - /// Deserializes the BeeGFS message present in the buffer - /// - /// # Panic - /// The function will panic if the buffer has not been filled with data before (e.g. by - /// reading from stream or receiving from a socket) - pub fn deserialize_msg(&self) -> Result { - const ERR_CTX: &str = "BeeMsg body deserialization failed"; - - let mut des = Deserializer::new(&self.buf[Header::LEN..], self.header.msg_feature_flags); - let des_msg = M::deserialize(&mut des).context(ERR_CTX)?; - des.finish().context(ERR_CTX)?; - Ok(des_msg) - } - - /// Reads a BeeGFS message from a stream into the buffer - pub(super) async fn read_from_stream(&mut self, stream: &mut Stream) -> Result<()> { - if self.buf.len() < Header::LEN { - self.buf.resize(Header::LEN, 0); - } - - stream.read_exact(&mut self.buf[0..Header::LEN]).await?; - let header = Header::from_buf(&self.buf[0..Header::LEN]) - .context("BeeMsg header deserialization failed")?; - let msg_len = header.msg_len(); - - if self.buf.len() != msg_len { - self.buf.resize(msg_len, 0); - } - - stream - .read_exact(&mut self.buf[Header::LEN..msg_len]) - .await?; - - *self.header = header; - - Ok(()) - } - - /// Writes the BeeGFS message from the buffer to a stream - /// - /// # Panic - /// The function will panic if the buffer has not been filled with data before (e.g. by - /// serializing a message) - pub(super) async fn write_to_stream(&self, stream: &mut Stream) -> Result<()> { - stream - .write_all(&self.buf[0..self.header.msg_len()]) - .await?; - Ok(()) - } - - /// Receives a BeeGFS message from a UDP socket into the buffer - pub(super) async fn recv_from_socket(&mut self, sock: &Arc) -> Result { - if self.buf.len() != DATAGRAM_LEN { - self.buf.resize(DATAGRAM_LEN, 0); - } - - match sock.recv_from(&mut self.buf).await { - Ok(n) => { - let header = Header::from_buf(&self.buf[0..Header::LEN])?; - self.buf.truncate(header.msg_len()); - *self.header = header; - Ok(n.1) - } - Err(err) => Err(err.into()), - } - } - - /// Sends the BeeGFS message in the buffer to a UDP socket - /// - /// # Panic - /// The function will panic if the buffer has not been filled with data before (e.g. by - /// serializing a message) - pub(super) async fn send_to_socket( - &self, - sock: &UdpSocket, - peer_addr: &SocketAddr, - ) -> Result<()> { - if self.buf.len() > DATAGRAM_LEN { - bail!( - "Datagram to be sent to {peer_addr:?} exceeds maximum length of {DATAGRAM_LEN} \ - bytes" - ); - } - - sock.send_to(&self.buf, peer_addr).await?; - Ok(()) - } - - /// The [MsgID] of the serialized BeeGFS message in the buffer - /// - /// # Panic - /// The function will panic if the buffer has not been filled with data before (e.g. by - /// reading from stream or receiving from a socket) - pub fn msg_id(&self) -> MsgId { - self.header.msg_id - } -} diff --git a/shared/src/conn/msg_dispatch.rs b/shared/src/conn/msg_dispatch.rs index 66b6e01..154f811 100644 --- a/shared/src/conn/msg_dispatch.rs +++ b/shared/src/conn/msg_dispatch.rs @@ -1,8 +1,7 @@ //! Facilities for dispatching TCP and UDP messages to their message handlers -use super::msg_buf::MsgBuf; use super::stream::Stream; -use crate::bee_msg::{Msg, MsgId}; +use crate::bee_msg::{Header, Msg, MsgId, deserialize_body, serialize}; use crate::bee_serde::{Deserializable, Serializable}; use anyhow::Result; use std::fmt::Debug; @@ -34,13 +33,14 @@ pub trait Request: Send + Sync { #[derive(Debug)] pub struct StreamRequest<'a> { pub(super) stream: &'a mut Stream, - pub(super) buf: &'a mut MsgBuf, + pub(super) buf: &'a mut [u8], + pub header: &'a Header, } impl Request for StreamRequest<'_> { async fn respond(self, msg: &M) -> Result<()> { - self.buf.serialize_msg(msg)?; - self.buf.write_to_stream(self.stream).await + let msg_len = serialize(msg, self.buf)?; + self.stream.write_all(&self.buf[0..msg_len]).await } fn authenticate_connection(&mut self) { @@ -58,11 +58,11 @@ impl Request for StreamRequest<'_> { } fn deserialize_msg(&self) -> Result { - self.buf.deserialize_msg() + deserialize_body(self.header, &self.buf[Header::LEN..]) } fn msg_id(&self) -> MsgId { - self.buf.msg_id() + self.header.msg_id() } } @@ -71,16 +71,17 @@ impl Request for StreamRequest<'_> { pub struct SocketRequest<'a> { pub(crate) sock: Arc, pub(crate) peer_addr: SocketAddr, - pub(crate) msg_buf: &'a mut MsgBuf, + pub(crate) buf: &'a mut [u8], + pub header: &'a Header, } impl Request for SocketRequest<'_> { async fn respond(self, msg: &M) -> Result<()> { - self.msg_buf.serialize_msg(msg)?; - - self.msg_buf - .send_to_socket(&self.sock, &self.peer_addr) - .await + let msg_len = serialize(msg, self.buf)?; + self.sock + .send_to(&self.buf[0..msg_len], &self.peer_addr) + .await?; + Ok(()) } fn authenticate_connection(&mut self) { @@ -92,10 +93,10 @@ impl Request for SocketRequest<'_> { } fn deserialize_msg(&self) -> Result { - self.msg_buf.deserialize_msg() + deserialize_body(self.header, &self.buf[Header::LEN..]) } fn msg_id(&self) -> MsgId { - self.msg_buf.msg_id() + self.header.msg_id() } } diff --git a/shared/src/conn/outgoing.rs b/shared/src/conn/outgoing.rs index 7da48e1..ca26b72 100644 --- a/shared/src/conn/outgoing.rs +++ b/shared/src/conn/outgoing.rs @@ -1,9 +1,9 @@ //! Outgoing communication functionality -use super::msg_buf::MsgBuf; use super::store::Store; -use crate::bee_msg::Msg; use crate::bee_msg::misc::AuthenticateChannel; +use crate::bee_msg::{Header, Msg, deserialize_body, deserialize_header, serialize}; use crate::bee_serde::{Deserializable, Serializable}; +use crate::conn::TCP_BUF_LEN; use crate::conn::store::StoredStream; use crate::conn::stream::Stream; use crate::types::{AuthSecret, Uid}; @@ -53,27 +53,27 @@ impl Pool { ) -> Result { log::trace!("REQUEST to {node_uid:?}: {msg:?}"); - let mut buf = self.store.pop_buf().unwrap_or_default(); + let mut buf = self.store.pop_buf_or_create(); - buf.serialize_msg(msg)?; - self.comm_stream(node_uid, &mut buf, true).await?; - let resp = buf.deserialize_msg()?; + let msg_len = serialize(msg, &mut buf)?; + let resp_header = self.comm_stream(node_uid, &mut buf, msg_len, true).await?; + let resp_msg = deserialize_body(&resp_header, &buf[Header::LEN..])?; self.store.push_buf(buf); - log::trace!("RESPONSE RECEIVED from {node_uid:?}: {resp:?}"); + log::trace!("RESPONSE RECEIVED from {node_uid:?}: {resp_msg:?}"); - Ok(resp) + Ok(resp_msg) } /// Sends a [Msg] to a node and does **not** receive a response. pub async fn send(&self, node_uid: Uid, msg: &M) -> Result<()> { log::trace!("SEND to {node_uid:?}: {msg:?}"); - let mut buf = self.store.pop_buf().unwrap_or_default(); + let mut buf = self.store.pop_buf_or_create(); - buf.serialize_msg(msg)?; - self.comm_stream(node_uid, &mut buf, false).await?; + let msg_len = serialize(msg, &mut buf)?; + self.comm_stream(node_uid, &mut buf, msg_len, false).await?; self.store.push_buf(buf); @@ -95,16 +95,19 @@ impl Pool { async fn comm_stream( &self, node_uid: Uid, - buf: &mut MsgBuf, + buf: &mut [u8], + send_len: usize, expect_response: bool, - ) -> Result<()> { + ) -> Result
{ + debug_assert_eq!(buf.len(), TCP_BUF_LEN); + // 1. Pop open streams until communication succeeds or none are left while let Some(stream) = self.store.try_pop_stream(node_uid) { match self - .write_and_read_stream(buf, stream, expect_response) + .write_and_read_stream(buf, stream, send_len, expect_response) .await { - Ok(_) => return Ok(()), + Ok(header) => return Ok(header), Err(err) => { // If the stream doesn't work anymore, just discard it and try the next one log::debug!("Communication using existing stream to {node_uid:?} failed: {err}") @@ -136,22 +139,27 @@ impl Pool { if let Some(auth_secret) = self.auth_secret { // The provided buffer contains the actual message to be sent later - // obtain an additional one for the auth message - let mut auth_buf = self.store.pop_buf().unwrap_or_default(); - auth_buf.serialize_msg(&AuthenticateChannel { auth_secret })?; - auth_buf - .write_to_stream(stream.as_mut()) + let mut auth_buf = self.store.pop_buf_or_create(); + let msg_len = + serialize(&AuthenticateChannel { auth_secret }, &mut auth_buf)?; + + stream + .as_mut() + .write_all(&auth_buf[0..msg_len]) .await .with_context(err_context)?; + self.store.push_buf(auth_buf); } // Communication using the newly opened stream should usually not fail. If // it does, abort. It might be better to just try the next address though. - self.write_and_read_stream(buf, stream, expect_response) + let resp_header = self + .write_and_read_stream(buf, stream, send_len, expect_response) .await .with_context(err_context)?; - return Ok(()); + return Ok(resp_header); } // If connecting failed, try the next address Err(err) => log::debug!("Connecting to {node_uid:?} via {addr} failed: {err}"), @@ -165,31 +173,44 @@ impl Pool { // 3. Wait for an already open stream becoming available let stream = self.store.pop_stream(node_uid).await?; - self.write_and_read_stream(buf, stream, expect_response) + let resp_header = self + .write_and_read_stream(buf, stream, send_len, expect_response) .await .with_context(|| { format!("Communication using existing stream to {node_uid:?} failed") })?; - Ok(()) + Ok(resp_header) } /// Writes data to the given stream, optionally receives a response and pushes the stream to /// the store async fn write_and_read_stream( &self, - buf: &mut MsgBuf, + buf: &mut [u8], mut stream: StoredStream, + send_len: usize, expect_response: bool, - ) -> Result<()> { - buf.write_to_stream(stream.as_mut()).await?; - - if expect_response { - buf.read_from_stream(stream.as_mut()).await?; - } + ) -> Result
{ + stream.as_mut().write_all(&buf[0..send_len]).await?; + + let header = if expect_response { + // Read header + stream.as_mut().read_exact(&mut buf[0..Header::LEN]).await?; + let header = deserialize_header(&buf[0..Header::LEN])?; + + // Read body + stream + .as_mut() + .read_exact(&mut buf[Header::LEN..header.msg_len()]) + .await?; + header + } else { + Header::default() + }; self.store.push_stream(stream); - Ok(()) + Ok(header) } /// Broadcasts a BeeMsg datagram to all given nodes using all their known addresses @@ -202,8 +223,9 @@ impl Pool { peers: impl IntoIterator, msg: &M, ) -> Result<()> { - let mut buf = self.store.pop_buf().unwrap_or_default(); - buf.serialize_msg(msg)?; + let mut buf = self.store.pop_buf_or_create(); + + let msg_len = serialize(msg, &mut buf)?; for node_uid in peers { let addrs = self.store.get_node_addrs(node_uid).unwrap_or_default(); @@ -221,7 +243,7 @@ impl Pool { continue; } - if let Err(err) = buf.send_to_socket(&self.udp_socket, addr).await { + if let Err(err) = self.udp_socket.send_to(&buf[0..msg_len], addr).await { log::debug!( "Sending datagram to node with uid {node_uid} using {addr} failed: {err}" ); diff --git a/shared/src/conn/store.rs b/shared/src/conn/store.rs index 55e1f4c..fe0231d 100644 --- a/shared/src/conn/store.rs +++ b/shared/src/conn/store.rs @@ -3,8 +3,8 @@ //! //! Also provides a permit system to limit outgoing connections to a defined maximum. +use super::TCP_BUF_LEN; use super::async_queue::AsyncQueue; -use crate::conn::MsgBuf; use crate::conn::stream::Stream; use crate::types::Uid; use anyhow::{Result, anyhow}; @@ -23,7 +23,7 @@ const TIMEOUT: Duration = Duration::from_secs(2); pub struct Store { #[allow(clippy::type_complexity)] streams: Mutex>, Arc)>>, - bufs: Mutex>, + bufs: Mutex>>, addrs: RwLock>>, connection_limit: usize, } @@ -112,12 +112,17 @@ impl Store { } /// Pop a message buffer from the store - pub fn pop_buf(&self) -> Option { + pub fn pop_buf(&self) -> Option> { self.bufs.lock().unwrap().pop_front() } + /// Pop a message buffer from the store or create a new one suitable for stream / TCP messages + pub fn pop_buf_or_create(&self) -> Vec { + self.pop_buf().unwrap_or_else(|| vec![0; TCP_BUF_LEN]) + } + /// Push back a message buffer to the store - pub fn push_buf(&self, buf: MsgBuf) { + pub fn push_buf(&self, buf: Vec) { self.bufs.lock().unwrap().push_back(buf); } diff --git a/shared/src/journald_logger.rs b/shared/src/journald_logger.rs index f1b50b5..1da217c 100644 --- a/shared/src/journald_logger.rs +++ b/shared/src/journald_logger.rs @@ -1,6 +1,5 @@ //! Journald logger implementation for the `log` interface -use bytes::BufMut; use log::{Level, LevelFilter, Log, Metadata, Record}; use std::os::unix::net::UnixDatagram; @@ -37,7 +36,7 @@ impl Log for JournaldLogger { .into_bytes(); buf.reserve(msg.len() + 8 + 1); - buf.put_u64_le(msg.len() as u64); + buf.extend((msg.len() as u64).to_le_bytes()); buf.extend(msg); buf.extend(b"\n"); From d25ba6af29e33139a54137d5d5edc1006c6a8ab8 Mon Sep 17 00:00:00 2001 From: Rusty Bee <145002912+rustybee42@users.noreply.github.com> Date: Fri, 18 Jul 2025 09:05:56 +0200 Subject: [PATCH 2/9] fix: don't break udp socket when receiving an invalid message, improve logging Found by test_empty_udp_packets_server integration test. The test needs to be changed as well to adapt to the corrected log output (it was not really correct before) --- shared/src/conn/incoming.rs | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/shared/src/conn/incoming.rs b/shared/src/conn/incoming.rs index 33cf52e..fd970cc 100644 --- a/shared/src/conn/incoming.rs +++ b/shared/src/conn/incoming.rs @@ -199,8 +199,7 @@ pub fn recv_udp( // Do the actual work res = recv_datagram(sock.clone(), dispatch.clone()) => { if let Err(err) = res { - log::error!("Error in UDP socket {sock:?}: {err:#}"); - break; + log::error!("Error on receiving datagram using UDP socket {:?}: {err:#}", sock.local_addr()); } } @@ -227,20 +226,26 @@ async fn recv_datagram(sock: Arc, msg_handler: impl DispatchRequest) let (_, peer_addr) = sock.recv_from(&mut buf).await?; - let header = deserialize_header(&buf[0..Header::LEN])?; - // Request shall be handled in a separate task, so the next datagram can be processed // immediately tokio::spawn(async move { - let req = SocketRequest { - sock, - peer_addr, - buf: &mut buf, - header: &header, - }; + if let Err(err) = async { + let header = deserialize_header(&buf[0..Header::LEN])?; + + let req = SocketRequest { + sock, + peer_addr, + buf: &mut buf, + header: &header, + }; - // Forward to the dispatcher - if let Err(err) = msg_handler.dispatch_request(req).await { + // Forward to the dispatcher + msg_handler.dispatch_request(req).await?; + + Ok::<(), anyhow::Error>(()) + } + .await + { log::error!("Error while handling datagram from {peer_addr:?}: {err:#}"); } }); From 4dd0151dd815f3d681d601a33d682a84ed1488e3 Mon Sep 17 00:00:00 2001 From: Rusty Bee <145002912+rustybee42@users.noreply.github.com> Date: Tue, 1 Apr 2025 14:28:21 +0200 Subject: [PATCH 3/9] refactor: move gRPC handlers to their own files --- mgmtd/src/grpc.rs | 74 ++- mgmtd/src/grpc/assign_pool.rs | 95 ++++ mgmtd/src/grpc/buddy_group.rs | 513 ------------------ mgmtd/src/grpc/common.rs | 13 + mgmtd/src/grpc/create_buddy_group.rs | 66 +++ mgmtd/src/grpc/create_pool.rs | 50 ++ mgmtd/src/grpc/delete_buddy_group.rs | 81 +++ mgmtd/src/grpc/delete_node.rs | 98 ++++ mgmtd/src/grpc/delete_pool.rs | 65 +++ mgmtd/src/grpc/delete_target.rs | 63 +++ mgmtd/src/grpc/get_buddy_groups.rs | 76 +++ mgmtd/src/grpc/{license.rs => get_license.rs} | 2 +- mgmtd/src/grpc/{node.rs => get_nodes.rs} | 102 +--- mgmtd/src/grpc/get_pools.rs | 134 +++++ mgmtd/src/grpc/get_quota_limits.rs | 134 +++++ mgmtd/src/grpc/get_quota_usage.rs | 168 ++++++ mgmtd/src/grpc/{target.rs => get_targets.rs} | 129 +---- mgmtd/src/grpc/mirror_root_inode.rs | 95 ++++ mgmtd/src/grpc/pool.rs | 336 ------------ mgmtd/src/grpc/quota.rs | 463 ---------------- mgmtd/src/grpc/{misc.rs => set_alias.rs} | 2 +- mgmtd/src/grpc/set_default_quota_limits.rs | 78 +++ mgmtd/src/grpc/set_quota_limits.rs | 79 +++ mgmtd/src/grpc/set_target_state.rs | 66 +++ mgmtd/src/grpc/start_resync.rs | 203 +++++++ shared/src/grpc.rs | 16 +- 26 files changed, 1626 insertions(+), 1575 deletions(-) create mode 100644 mgmtd/src/grpc/assign_pool.rs delete mode 100644 mgmtd/src/grpc/buddy_group.rs create mode 100644 mgmtd/src/grpc/common.rs create mode 100644 mgmtd/src/grpc/create_buddy_group.rs create mode 100644 mgmtd/src/grpc/create_pool.rs create mode 100644 mgmtd/src/grpc/delete_buddy_group.rs create mode 100644 mgmtd/src/grpc/delete_node.rs create mode 100644 mgmtd/src/grpc/delete_pool.rs create mode 100644 mgmtd/src/grpc/delete_target.rs create mode 100644 mgmtd/src/grpc/get_buddy_groups.rs rename mgmtd/src/grpc/{license.rs => get_license.rs} (93%) rename mgmtd/src/grpc/{node.rs => get_nodes.rs} (65%) create mode 100644 mgmtd/src/grpc/get_pools.rs create mode 100644 mgmtd/src/grpc/get_quota_limits.rs create mode 100644 mgmtd/src/grpc/get_quota_usage.rs rename mgmtd/src/grpc/{target.rs => get_targets.rs} (60%) create mode 100644 mgmtd/src/grpc/mirror_root_inode.rs delete mode 100644 mgmtd/src/grpc/pool.rs delete mode 100644 mgmtd/src/grpc/quota.rs rename mgmtd/src/grpc/{misc.rs => set_alias.rs} (98%) create mode 100644 mgmtd/src/grpc/set_default_quota_limits.rs create mode 100644 mgmtd/src/grpc/set_quota_limits.rs create mode 100644 mgmtd/src/grpc/set_target_state.rs create mode 100644 mgmtd/src/grpc/start_resync.rs diff --git a/mgmtd/src/grpc.rs b/mgmtd/src/grpc.rs index 01827fe..1a0d4ba 100644 --- a/mgmtd/src/grpc.rs +++ b/mgmtd/src/grpc.rs @@ -7,7 +7,7 @@ use crate::license::LicensedFeature; use crate::types::{ResolveEntityId, SqliteEnumExt}; use anyhow::{Context as AContext, Result, anyhow, bail}; use protobuf::{beegfs as pb, management as pm}; -use rusqlite::{OptionalExtension, Transaction, TransactionBehavior, params}; +use rusqlite::{OptionalExtension, Row, Transaction, TransactionBehavior, named_params, params}; use shared::grpc::*; use shared::impl_grpc_handler; use shared::run_state::RunStateHandle; @@ -21,13 +21,28 @@ use std::pin::Pin; use tonic::transport::{Identity, Server, ServerTlsConfig}; use tonic::{Code, Request, Response, Status}; -mod buddy_group; -mod license; -mod misc; -mod node; -mod pool; -mod quota; -mod target; +mod common; + +mod assign_pool; +mod create_buddy_group; +mod create_pool; +mod delete_buddy_group; +mod delete_node; +mod delete_pool; +mod delete_target; +mod get_buddy_groups; +mod get_license; +mod get_nodes; +mod get_pools; +mod get_quota_limits; +mod get_quota_usage; +mod get_targets; +mod mirror_root_inode; +mod set_alias; +mod set_default_quota_limits; +mod set_quota_limits; +mod set_target_state; +mod start_resync; /// Management gRPC service implementation struct #[derive(Debug)] @@ -44,8 +59,9 @@ impl pm::management_server::Management for ManagementService { // Example: Implement pm::management_server::Management::set_alias using the impl_grpc_handler // macro impl_grpc_handler! { - // => , - set_alias => misc::set_alias, + // the function to implement (as defined by the trait) as well as the handler to call (must + // be named the same and in a submodule named the same), + set_alias, // => , pm::SetAliasRequest => pm::SetAliasResponse, @@ -54,102 +70,102 @@ impl pm::management_server::Management for ManagementService { } impl_grpc_handler! { - get_nodes => node::get, + get_nodes, pm::GetNodesRequest => pm::GetNodesResponse, "Get nodes" } impl_grpc_handler! { - delete_node => node::delete, + delete_node, pm::DeleteNodeRequest => pm::DeleteNodeResponse, "Delete node" } impl_grpc_handler! { - get_targets => target::get, + get_targets, pm::GetTargetsRequest => pm::GetTargetsResponse, "Get targets" } impl_grpc_handler! { - delete_target => target::delete, + delete_target, pm::DeleteTargetRequest => pm::DeleteTargetResponse, "Delete target" } impl_grpc_handler! { - set_target_state => target::set_state, + set_target_state, pm::SetTargetStateRequest => pm::SetTargetStateResponse, "Set target state" } impl_grpc_handler! { - get_pools => pool::get, + get_pools, pm::GetPoolsRequest => pm::GetPoolsResponse, "Get pools" } impl_grpc_handler! { - create_pool => pool::create, + create_pool, pm::CreatePoolRequest => pm::CreatePoolResponse, "Create pool" } impl_grpc_handler! { - assign_pool => pool::assign, + assign_pool, pm::AssignPoolRequest => pm::AssignPoolResponse, "Assign pool" } impl_grpc_handler! { - delete_pool => pool::delete, + delete_pool, pm::DeletePoolRequest => pm::DeletePoolResponse, "Delete pool" } impl_grpc_handler! { - get_buddy_groups => buddy_group::get, + get_buddy_groups, pm::GetBuddyGroupsRequest => pm::GetBuddyGroupsResponse, "Get buddy groups" } impl_grpc_handler! { - create_buddy_group => buddy_group::create, + create_buddy_group, pm::CreateBuddyGroupRequest => pm::CreateBuddyGroupResponse, "Create buddy group" } impl_grpc_handler! { - delete_buddy_group => buddy_group::delete, + delete_buddy_group, pm::DeleteBuddyGroupRequest => pm::DeleteBuddyGroupResponse, "Delete buddy group" } impl_grpc_handler! { - mirror_root_inode => buddy_group::mirror_root_inode, + mirror_root_inode, pm::MirrorRootInodeRequest => pm::MirrorRootInodeResponse, "Mirror root inode" } impl_grpc_handler! { - start_resync => buddy_group::start_resync, + start_resync, pm::StartResyncRequest => pm::StartResyncResponse, "Start resync" } impl_grpc_handler! { - set_default_quota_limits => quota::set_default_quota_limits, + set_default_quota_limits, pm::SetDefaultQuotaLimitsRequest => pm::SetDefaultQuotaLimitsResponse, "Set default quota limits" } impl_grpc_handler! { - set_quota_limits => quota::set_quota_limits, + set_quota_limits, pm::SetQuotaLimitsRequest => pm::SetQuotaLimitsResponse, "Set quota limits" } impl_grpc_handler! { - get_quota_limits => quota::get_quota_limits, + get_quota_limits, pm::GetQuotaLimitsRequest => STREAM(GetQuotaLimitsStream, pm::GetQuotaLimitsResponse), "Get quota limits" } impl_grpc_handler! { - get_quota_usage => quota::get_quota_usage, + get_quota_usage, pm::GetQuotaUsageRequest => STREAM(GetQuotaUsageStream, pm::GetQuotaUsageResponse), "Get quota usage" } impl_grpc_handler! { - get_license => license::get, + get_license, pm::GetLicenseRequest => pm::GetLicenseResponse, "Get license" } diff --git a/mgmtd/src/grpc/assign_pool.rs b/mgmtd/src/grpc/assign_pool.rs new file mode 100644 index 0000000..c04de9f --- /dev/null +++ b/mgmtd/src/grpc/assign_pool.rs @@ -0,0 +1,95 @@ +use super::*; +use shared::bee_msg::storage_pool::RefreshStoragePools; + +/// Assigns a pool to a list of targets and buddy groups. +pub(crate) async fn assign_pool( + ctx: Context, + req: pm::AssignPoolRequest, +) -> Result { + needs_license(&ctx, LicensedFeature::Storagepool)?; + fail_on_pre_shutdown(&ctx)?; + + let pool: EntityId = required_field(req.pool)?.try_into()?; + + let pool = ctx + .db + .write_tx(move |tx| { + let pool = pool.resolve(tx, EntityType::Pool)?; + do_assign(tx, pool.num_id().try_into()?, req.targets, req.buddy_groups)?; + Ok(pool) + }) + .await?; + + log::info!("Pool assigned: {pool}"); + + notify_nodes( + &ctx, + &[NodeType::Meta, NodeType::Storage], + &RefreshStoragePools { ack_id: "".into() }, + ) + .await; + + Ok(pm::AssignPoolResponse { + pool: Some(pool.into()), + }) +} + +/// Do the actual assign work +pub(super) fn do_assign( + tx: &Transaction, + pool_id: PoolId, + targets: Vec, + groups: Vec, +) -> Result<()> { + // Target being part of a buddy group can not be assigned individually + let mut check_group_membership = tx.prepare_cached(sql!( + "SELECT COUNT(*) FROM storage_buddy_groups AS g + INNER JOIN targets AS p_t ON p_t.target_id = g.p_target_id AND p_t.node_type = g.node_type + INNER JOIN targets AS s_t ON s_t.target_id = g.s_target_id AND s_t.node_type = g.node_type + WHERE p_t.target_uid = ?1 OR s_t.target_uid = ?1" + ))?; + + let mut assign_target = tx.prepare_cached(sql!( + "UPDATE targets SET pool_id = ?1 WHERE target_uid = ?2" + ))?; + + // Do the checks and assign for each target in the given list. This is expensive, but shouldn't + // matter as this command should only be run occasionally and not with very large lists of + // targets. + for t in targets { + let eid = EntityId::try_from(t)?; + let target = eid.resolve(tx, EntityType::Target)?; + if check_group_membership.query_row([target.uid], |row| row.get::<_, i64>(0))? > 0 { + bail!("Target {eid} can't be assigned directly as it's part of a buddy group"); + } + + assign_target.execute(params![pool_id, target.uid])?; + } + + let mut assign_group = tx.prepare_cached(sql!( + "UPDATE buddy_groups SET pool_id = ?1 WHERE group_uid = ?2" + ))?; + + // Targets being part of buddy groups are auto-assigned to the new pool + let mut assign_grouped_targets = tx.prepare_cached(sql!( + "UPDATE targets SET pool_id = ?1 + FROM ( + SELECT p_t.target_uid AS p_uid, s_t.target_uid AS s_uid FROM buddy_groups AS g + INNER JOIN targets AS p_t ON p_t.target_id = g.p_target_id AND p_t.node_type = g.node_type + INNER JOIN targets AS s_t ON s_t.target_id = g.s_target_id AND s_t.node_type = g.node_type + WHERE group_uid = ?2 + ) + WHERE target_uid IN (p_uid, s_uid)" + ))?; + + // Assign each group and their targets to the new pool + for g in groups { + let eid = EntityId::try_from(g)?; + let group = eid.resolve(tx, EntityType::BuddyGroup)?; + + assign_group.execute(params![pool_id, group.uid])?; + assign_grouped_targets.execute(params![pool_id, group.uid])?; + } + + Ok(()) +} diff --git a/mgmtd/src/grpc/buddy_group.rs b/mgmtd/src/grpc/buddy_group.rs deleted file mode 100644 index 63f2753..0000000 --- a/mgmtd/src/grpc/buddy_group.rs +++ /dev/null @@ -1,513 +0,0 @@ -use super::*; -use db::misc::MetaRoot; -use protobuf::{beegfs as pb, management as pm}; -use shared::bee_msg::OpsErr; -use shared::bee_msg::buddy_group::{ - BuddyResyncJobState, GetMetaResyncStats, GetMetaResyncStatsResp, GetStorageResyncStats, - GetStorageResyncStatsResp, RemoveBuddyGroup, RemoveBuddyGroupResp, SetLastBuddyCommOverride, - SetMetadataMirroring, SetMetadataMirroringResp, SetMirrorBuddyGroup, -}; -use shared::bee_msg::target::{RefreshTargetStates, SetTargetConsistencyStatesResp}; -use tokio::time::{Duration, Instant, sleep}; - -/// Delivers the list of buddy groups -pub(crate) async fn get( - ctx: Context, - _req: pm::GetBuddyGroupsRequest, -) -> Result { - let buddy_groups = ctx - .db - .read_tx(|tx| { - Ok(tx.query_map_collect( - sql!( - "SELECT group_uid, group_id, bg.alias, bg.node_type, - p_target_uid, p_t.target_id, p_t.alias, - s_target_uid, s_t.target_id, s_t.alias, - p.pool_uid, bg.pool_id, p.alias, - p_t.consistency, s_t.consistency - FROM buddy_groups_ext AS bg - INNER JOIN targets_ext AS p_t ON p_t.target_uid = p_target_uid - INNER JOIN targets_ext AS s_t ON s_t.target_uid = s_target_uid - LEFT JOIN pools_ext AS p USING(node_type, pool_id)" - ), - [], - |row| { - let node_type = NodeType::from_row(row, 3)?.into_proto_i32(); - let p_con_state = TargetConsistencyState::from_row(row, 13)?.into_proto_i32(); - let s_con_state = TargetConsistencyState::from_row(row, 14)?.into_proto_i32(); - - Ok(pm::get_buddy_groups_response::BuddyGroup { - id: Some(pb::EntityIdSet { - uid: row.get(0)?, - legacy_id: Some(pb::LegacyId { - num_id: row.get(1)?, - node_type, - }), - alias: row.get(2)?, - }), - node_type, - primary_target: Some(pb::EntityIdSet { - uid: row.get(4)?, - legacy_id: Some(pb::LegacyId { - num_id: row.get(5)?, - node_type, - }), - alias: row.get(6)?, - }), - secondary_target: Some(pb::EntityIdSet { - uid: row.get(7)?, - legacy_id: Some(pb::LegacyId { - num_id: row.get(8)?, - node_type, - }), - alias: row.get(9)?, - }), - storage_pool: if let Some(uid) = row.get::<_, Option>(10)? { - Some(pb::EntityIdSet { - uid: Some(uid), - legacy_id: Some(pb::LegacyId { - num_id: row.get(11)?, - node_type, - }), - alias: row.get(12)?, - }) - } else { - None - }, - primary_consistency_state: p_con_state, - secondary_consistency_state: s_con_state, - }) - }, - )?) - }) - .await?; - - Ok(pm::GetBuddyGroupsResponse { buddy_groups }) -} - -/// Creates a new buddy group -pub(crate) async fn create( - ctx: Context, - req: pm::CreateBuddyGroupRequest, -) -> Result { - needs_license(&ctx, LicensedFeature::Mirroring)?; - fail_on_pre_shutdown(&ctx)?; - - let node_type: NodeTypeServer = req.node_type().try_into()?; - let alias: Alias = required_field(req.alias)?.try_into()?; - let num_id: BuddyGroupId = req.num_id.unwrap_or_default().try_into()?; - let p_target: EntityId = required_field(req.primary_target)?.try_into()?; - let s_target: EntityId = required_field(req.secondary_target)?.try_into()?; - - let (group, p_target, s_target) = ctx - .db - .write_tx(move |tx| { - let p_target = p_target.resolve(tx, EntityType::Target)?; - let s_target = s_target.resolve(tx, EntityType::Target)?; - - let (group_uid, group_id) = db::buddy_group::insert( - tx, - num_id, - Some(alias.clone()), - node_type, - p_target.num_id().try_into()?, - s_target.num_id().try_into()?, - )?; - Ok(( - EntityIdSet { - uid: group_uid, - alias, - legacy_id: LegacyId { - node_type: node_type.into(), - num_id: group_id.into(), - }, - }, - p_target, - s_target, - )) - }) - .await?; - - log::info!("Buddy group created: {group}"); - - notify_nodes( - &ctx, - &[NodeType::Meta, NodeType::Storage, NodeType::Client], - &SetMirrorBuddyGroup { - ack_id: "".into(), - node_type: node_type.into(), - primary_target_id: p_target.num_id().try_into()?, - secondary_target_id: s_target.num_id().try_into()?, - group_id: group.num_id().try_into()?, - allow_update: 0, - }, - ) - .await; - - Ok(pm::CreateBuddyGroupResponse { - group: Some(group.into()), - }) -} - -/// Deletes a buddy group. This function is racy as it is a two step process, talking to other -/// nodes in between. Since it is rarely used, that's ok though. -pub(crate) async fn delete( - ctx: Context, - req: pm::DeleteBuddyGroupRequest, -) -> Result { - needs_license(&ctx, LicensedFeature::Mirroring)?; - fail_on_pre_shutdown(&ctx)?; - - let group: EntityId = required_field(req.group)?.try_into()?; - let execute: bool = required_field(req.execute)?; - - // 1. Check deletion is allowed - let (group, p_node_uid, s_node_uid) = ctx - .db - .conn(move |conn| { - let tx = conn.transaction()?; - - let group = group.resolve(&tx, EntityType::BuddyGroup)?; - - if group.node_type() != NodeType::Storage { - bail!("Only storage buddy groups can be deleted"); - } - - let (p_node_uid, s_node_uid) = - db::buddy_group::prepare_storage_deletion(&tx, group.num_id().try_into()?)?; - - if execute { - tx.commit()?; - } - Ok((group, p_node_uid, s_node_uid)) - }) - .await?; - - // 2. Forward request to the groups nodes - let group_id: BuddyGroupId = group.num_id().try_into()?; - let remove_bee_msg = RemoveBuddyGroup { - node_type: NodeType::Storage, - group_id, - check_only: if execute { 0 } else { 1 }, - force: 0, - }; - - let p_res: RemoveBuddyGroupResp = ctx.conn.request(p_node_uid, &remove_bee_msg).await?; - let s_res: RemoveBuddyGroupResp = ctx.conn.request(s_node_uid, &remove_bee_msg).await?; - - if p_res.result != OpsErr::SUCCESS || s_res.result != OpsErr::SUCCESS { - bail!( - "Removing storage buddy group on primary and/or secondary storage node failed. \ -Primary result: {:?}, Secondary result: {:?}", - p_res.result, - s_res.result - ); - } - - // 3. If the deletion request succeeded, remove the group from the database - ctx.db - .conn(move |conn| { - let tx = conn.transaction()?; - - db::buddy_group::delete_storage(&tx, group_id)?; - - if execute { - tx.commit()?; - } - Ok(()) - }) - .await?; - - if execute { - log::info!("Buddy group deleted: {group}"); - } - - Ok(pm::DeleteBuddyGroupResponse { - group: Some(group.into()), - }) -} - -/// Enable metadata mirroring for the root directory -pub(crate) async fn mirror_root_inode( - ctx: Context, - _req: pm::MirrorRootInodeRequest, -) -> Result { - needs_license(&ctx, LicensedFeature::Mirroring)?; - fail_on_pre_shutdown(&ctx)?; - - let offline_timeout = ctx.info.user_config.node_offline_timeout.as_secs(); - let meta_root = ctx - .db - .read_tx(move |tx| { - let node_uid = match db::misc::get_meta_root(tx)? { - MetaRoot::Normal(_, node_uid) => node_uid, - MetaRoot::Mirrored(_) => bail!("Root inode is already mirrored"), - MetaRoot::Unknown => bail!("Root inode unknown"), - }; - - let count = tx.query_row( - sql!( - "SELECT COUNT(*) FROM root_inode AS ri - INNER JOIN buddy_groups AS mg - ON mg.p_target_id = ri.target_id AND mg.node_type = ?1" - ), - [NodeType::Meta.sql_variant()], - |row| row.get::<_, i64>(0), - )?; - - if count < 1 { - bail!("The meta target holding the root inode is not part of a buddy group."); - } - - // Check that no clients are connected to prevent data corruption. Note that there is - // still a small chance for a client being mounted again before the action is taken on - // the root meta server below. In the end, it's the administrators responsibility to not - // let any client mount during that process. - let clients = tx.query_row(sql!("SELECT COUNT(*) FROM client_nodes"), [], |row| { - row.get::<_, i64>(0) - })?; - - if clients > 0 { - bail!( - "This operation requires that all clients are disconnected/unmounted. \ -{clients} clients are still mounted." - ); - } - - let mut server_stmt = tx.prepare(sql!( - "SELECT COUNT(*) FROM nodes - WHERE node_type = ?1 AND UNIXEPOCH('now') - UNIXEPOCH(last_contact) < ?2 - AND node_uid != ?3" - ))?; - - let metas = server_stmt.query_row( - params![NodeType::Meta.sql_variant(), offline_timeout, node_uid], - |row| row.get::<_, i64>(0), - )?; - let storages = server_stmt.query_row( - params![NodeType::Storage.sql_variant(), offline_timeout, node_uid], - |row| row.get::<_, i64>(0), - )?; - - if metas > 0 || storages > 0 { - bail!( - "This operation requires that all nodes except the root meta node are shut \ -down. {metas} meta nodes (excluding the root meta node) and {storages} storage nodes have \ -communicated during the last {offline_timeout}s." - ); - } - - Ok(node_uid) - }) - .await?; - - let resp: SetMetadataMirroringResp = ctx - .conn - .request(meta_root, &SetMetadataMirroring {}) - .await?; - - match resp.result { - OpsErr::SUCCESS => ctx.db.write_tx(db::misc::enable_metadata_mirroring).await?, - _ => bail!( - "The root meta server failed to mirror the root inode: {:?}", - resp.result - ), - } - - log::info!("Root inode has been mirrored"); - Ok(pm::MirrorRootInodeResponse {}) -} - -/// Starts a resync of a storage or metadata target from its buddy target -pub(crate) async fn start_resync( - ctx: Context, - req: pm::StartResyncRequest, -) -> Result { - needs_license(&ctx, LicensedFeature::Mirroring)?; - fail_on_pre_shutdown(&ctx)?; - - let buddy_group: EntityId = required_field(req.buddy_group)?.try_into()?; - let timestamp: i64 = required_field(req.timestamp)?; - let restart: bool = required_field(req.restart)?; - - // For resync source is always primary target and destination is secondary target - let (src_target_id, dest_target_id, src_node_uid, node_type, group) = ctx - .db - .read_tx(move |tx| { - let group = buddy_group.resolve(tx, EntityType::BuddyGroup)?; - let node_type: NodeTypeServer = group.node_type().try_into()?; - - let (src_target_id, dest_target_id, src_node_uid): (TargetId, TargetId, Uid) = tx - .query_row_cached( - sql!( - "SELECT g.p_target_id, g.s_target_id, src_t.node_uid - FROM buddy_groups AS g - INNER JOIN targets_ext AS src_t - ON src_t.target_id = g.p_target_id AND src_t.node_type = g.node_type - WHERE group_uid = ?1" - ), - [group.uid], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - )?; - - Ok(( - src_target_id, - dest_target_id, - src_node_uid, - node_type, - group, - )) - }) - .await?; - - // We handle meta and storage servers separately as storage servers allow restarts and metas - // dont. - match node_type { - NodeTypeServer::Meta => { - if timestamp > -1 { - bail!( - "Metadata targets can only do full resync, timestamp and timespan is \ -not supported." - ); - } - if restart { - bail!("Resync cannot be restarted or aborted for metadata servers."); - } - - let resp: GetMetaResyncStatsResp = ctx - .conn - .request( - src_node_uid, - &GetMetaResyncStats { - target_id: src_target_id, - }, - ) - .await?; - - if resp.state == BuddyResyncJobState::Running { - bail!("Resync already running on buddy group {group}"); - } - } - NodeTypeServer::Storage => { - if !restart { - let resp: GetStorageResyncStatsResp = ctx - .conn - .request( - src_node_uid, - &GetStorageResyncStats { - target_id: src_target_id, - }, - ) - .await?; - - if resp.state == BuddyResyncJobState::Running { - bail!("Resync already running on buddy group {group}"); - } - - if timestamp > -1 { - override_last_buddy_comm(&ctx, src_node_uid, src_target_id, &group, timestamp) - .await?; - } - } else { - if timestamp < 0 { - bail!("Resync for storage targets can only be restarted with timestamp."); - } - - override_last_buddy_comm(&ctx, src_node_uid, src_target_id, &group, timestamp) - .await?; - - log::info!("Waiting for the already running resync operations to abort."); - - let timeout = tokio::time::Duration::from_secs(180); - let start = Instant::now(); - - // This sleep and poll loop is bad style, but the simplest way to do it for - // now. A better solution would be to intercept the message from the server that - // tells us the resync is finished, but that is a bit more complex and, with the - // current system, still unreliable. - loop { - let resp: GetStorageResyncStatsResp = ctx - .conn - .request( - src_node_uid, - &GetStorageResyncStats { - target_id: src_target_id, - }, - ) - .await?; - - if resp.state != BuddyResyncJobState::Running { - break; - } - - if start.elapsed() >= timeout { - bail!("Timeout. Unable to abort resync on buddy group {group}"); - } - - sleep(Duration::from_secs(2)).await; - } - } - } - } - - // set destination target state as needs-resync in mgmtd database - ctx.db - .write_tx(move |tx| { - db::target::update_consistency_states( - tx, - [(dest_target_id, TargetConsistencyState::NeedsResync)], - node_type, - )?; - Ok(()) - }) - .await?; - - // This also triggers the source node to fetch the new needs resync state and start the resync - // using the internode syncer loop. In case of overriding last buddy communication on storage - // nodes, this means that there is a max 3s window where the communication timestamp can be - // overwritten again before resync starts, effectively ignoring it. There is nothing we can do - // about that without changing the storage server. - // - // Note that sending a SetTargetConsistencyStateMsg does have no effect on making this quicker, - // so we omit it. - notify_nodes( - &ctx, - &[NodeType::Meta, NodeType::Storage, NodeType::Client], - &RefreshTargetStates { ack_id: "".into() }, - ) - .await; - - return Ok(pm::StartResyncResponse {}); - - /// Override last buddy communication timestamp on source storage node - /// Note that this might be overwritten again on the storage server between - async fn override_last_buddy_comm( - ctx: &Context, - src_node_uid: Uid, - src_target_id: TargetId, - group: &EntityIdSet, - timestamp: i64, - ) -> Result<()> { - let resp: SetTargetConsistencyStatesResp = ctx - .conn - .request( - src_node_uid, - &SetLastBuddyCommOverride { - target_id: src_target_id, - timestamp, - abort_resync: 0, - }, - ) - .await?; - - if resp.result != OpsErr::SUCCESS { - bail!( - "Could not override last buddy communication timestamp on primary of buddy group {group}. \ -Failed with resp {:?}", - resp.result - ); - } - - Ok(()) - } -} diff --git a/mgmtd/src/grpc/common.rs b/mgmtd/src/grpc/common.rs new file mode 100644 index 0000000..38ff5be --- /dev/null +++ b/mgmtd/src/grpc/common.rs @@ -0,0 +1,13 @@ +pub(super) const QUOTA_NOT_ENABLED_STR: &str = "Quota support is not enabled"; + +// Fetching pages of 1M from quota_usage takes around 2100ms on my slow developer laptop (using a +// release build). In comparison, a page size of 100k takes around 750ms which is far worse. This +// feels like a good middle point to not let the requester wait too long and not waste too many db +// thread cycles with overhead. +pub(super) const QUOTA_STREAM_PAGE_LIMIT: usize = 1_000_000; + +// Need to hit a compromise between memory footprint and speed. Bigger is better if multiple +// pages need to be fetched but doesn't matter too much if not. Each entry is roughly 50 - +// 60 bytes, so 100k (= 5-6MB) feels fine. And it is still big enough to give a significant +// boost to throughput for big numbers. +pub(super) const QUOTA_STREAM_BUF_SIZE: usize = 100_000; diff --git a/mgmtd/src/grpc/create_buddy_group.rs b/mgmtd/src/grpc/create_buddy_group.rs new file mode 100644 index 0000000..e26772a --- /dev/null +++ b/mgmtd/src/grpc/create_buddy_group.rs @@ -0,0 +1,66 @@ +use super::*; +use shared::bee_msg::buddy_group::SetMirrorBuddyGroup; + +/// Creates a new buddy group +pub(crate) async fn create_buddy_group( + ctx: Context, + req: pm::CreateBuddyGroupRequest, +) -> Result { + needs_license(&ctx, LicensedFeature::Mirroring)?; + fail_on_pre_shutdown(&ctx)?; + + let node_type: NodeTypeServer = req.node_type().try_into()?; + let alias: Alias = required_field(req.alias)?.try_into()?; + let num_id: BuddyGroupId = req.num_id.unwrap_or_default().try_into()?; + let p_target: EntityId = required_field(req.primary_target)?.try_into()?; + let s_target: EntityId = required_field(req.secondary_target)?.try_into()?; + + let (group, p_target, s_target) = ctx + .db + .write_tx(move |tx| { + let p_target = p_target.resolve(tx, EntityType::Target)?; + let s_target = s_target.resolve(tx, EntityType::Target)?; + + let (group_uid, group_id) = db::buddy_group::insert( + tx, + num_id, + Some(alias.clone()), + node_type, + p_target.num_id().try_into()?, + s_target.num_id().try_into()?, + )?; + Ok(( + EntityIdSet { + uid: group_uid, + alias, + legacy_id: LegacyId { + node_type: node_type.into(), + num_id: group_id.into(), + }, + }, + p_target, + s_target, + )) + }) + .await?; + + log::info!("Buddy group created: {group}"); + + notify_nodes( + &ctx, + &[NodeType::Meta, NodeType::Storage, NodeType::Client], + &SetMirrorBuddyGroup { + ack_id: "".into(), + node_type: node_type.into(), + primary_target_id: p_target.num_id().try_into()?, + secondary_target_id: s_target.num_id().try_into()?, + group_id: group.num_id().try_into()?, + allow_update: 0, + }, + ) + .await; + + Ok(pm::CreateBuddyGroupResponse { + group: Some(group.into()), + }) +} diff --git a/mgmtd/src/grpc/create_pool.rs b/mgmtd/src/grpc/create_pool.rs new file mode 100644 index 0000000..d8267f9 --- /dev/null +++ b/mgmtd/src/grpc/create_pool.rs @@ -0,0 +1,50 @@ +use super::*; +use assign_pool::do_assign; +use shared::bee_msg::storage_pool::RefreshStoragePools; + +/// Creates a new pool, optionally assigning targets and groups +pub(crate) async fn create_pool( + ctx: Context, + req: pm::CreatePoolRequest, +) -> Result { + needs_license(&ctx, LicensedFeature::Storagepool)?; + fail_on_pre_shutdown(&ctx)?; + + if req.node_type() != pb::NodeType::Storage { + bail!("node type must be storage"); + } + + let alias: Alias = required_field(req.alias)?.try_into()?; + let num_id: PoolId = req.num_id.unwrap_or_default().try_into()?; + + let (pool_uid, alias, pool_id) = ctx + .db + .write_tx(move |tx| { + let (pool_uid, pool_id) = db::storage_pool::insert(tx, num_id, &alias)?; + do_assign(tx, pool_id, req.targets, req.buddy_groups)?; + Ok((pool_uid, alias, pool_id)) + }) + .await?; + + let pool = EntityIdSet { + uid: pool_uid, + alias, + legacy_id: LegacyId { + node_type: NodeType::Storage, + num_id: pool_id.into(), + }, + }; + + log::info!("Pool created: {pool}"); + + notify_nodes( + &ctx, + &[NodeType::Meta, NodeType::Storage], + &RefreshStoragePools { ack_id: "".into() }, + ) + .await; + + Ok(pm::CreatePoolResponse { + pool: Some(pool.into()), + }) +} diff --git a/mgmtd/src/grpc/delete_buddy_group.rs b/mgmtd/src/grpc/delete_buddy_group.rs new file mode 100644 index 0000000..2a67daf --- /dev/null +++ b/mgmtd/src/grpc/delete_buddy_group.rs @@ -0,0 +1,81 @@ +use super::*; +use shared::bee_msg::OpsErr; +use shared::bee_msg::buddy_group::{RemoveBuddyGroup, RemoveBuddyGroupResp}; + +/// Deletes a buddy group. This function is racy as it is a two step process, talking to other +/// nodes in between. Since it is rarely used, that's ok though. +pub(crate) async fn delete_buddy_group( + ctx: Context, + req: pm::DeleteBuddyGroupRequest, +) -> Result { + needs_license(&ctx, LicensedFeature::Mirroring)?; + fail_on_pre_shutdown(&ctx)?; + + let group: EntityId = required_field(req.group)?.try_into()?; + let execute: bool = required_field(req.execute)?; + + // 1. Check deletion is allowed + let (group, p_node_uid, s_node_uid) = ctx + .db + .conn(move |conn| { + let tx = conn.transaction()?; + + let group = group.resolve(&tx, EntityType::BuddyGroup)?; + + if group.node_type() != NodeType::Storage { + bail!("Only storage buddy groups can be deleted"); + } + + let (p_node_uid, s_node_uid) = + db::buddy_group::prepare_storage_deletion(&tx, group.num_id().try_into()?)?; + + if execute { + tx.commit()?; + } + Ok((group, p_node_uid, s_node_uid)) + }) + .await?; + + // 2. Forward request to the groups nodes + let group_id: BuddyGroupId = group.num_id().try_into()?; + let remove_bee_msg = RemoveBuddyGroup { + node_type: NodeType::Storage, + group_id, + check_only: if execute { 0 } else { 1 }, + force: 0, + }; + + let p_res: RemoveBuddyGroupResp = ctx.conn.request(p_node_uid, &remove_bee_msg).await?; + let s_res: RemoveBuddyGroupResp = ctx.conn.request(s_node_uid, &remove_bee_msg).await?; + + if p_res.result != OpsErr::SUCCESS || s_res.result != OpsErr::SUCCESS { + bail!( + "Removing storage buddy group on primary and/or secondary storage node failed. \ +Primary result: {:?}, Secondary result: {:?}", + p_res.result, + s_res.result + ); + } + + // 3. If the deletion request succeeded, remove the group from the database + ctx.db + .conn(move |conn| { + let tx = conn.transaction()?; + + db::buddy_group::delete_storage(&tx, group_id)?; + + if execute { + tx.commit()?; + } + Ok(()) + }) + .await?; + + if execute { + log::info!("Buddy group deleted: {group}"); + } + + Ok(pm::DeleteBuddyGroupResponse { + group: Some(group.into()), + }) +} diff --git a/mgmtd/src/grpc/delete_node.rs b/mgmtd/src/grpc/delete_node.rs new file mode 100644 index 0000000..f3fb407 --- /dev/null +++ b/mgmtd/src/grpc/delete_node.rs @@ -0,0 +1,98 @@ +use super::*; +use shared::bee_msg::node::RemoveNode; + +/// Deletes a node. If it is a meta node, deletes its target first. +pub(crate) async fn delete_node( + ctx: Context, + req: pm::DeleteNodeRequest, +) -> Result { + fail_on_pre_shutdown(&ctx)?; + + let node: EntityId = required_field(req.node)?.try_into()?; + let execute: bool = required_field(req.execute)?; + + let node = ctx + .db + .conn(move |conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + + let node = node.resolve(&tx, EntityType::Node)?; + + if node.uid == MGMTD_UID { + bail!("Management node can not be deleted"); + } + + // Meta nodes have an auto-assigned target which needs to be deleted first. + if node.node_type() == NodeType::Meta { + let assigned_groups: usize = tx.query_row_cached( + sql!( + "SELECT COUNT(*) FROM meta_buddy_groups + WHERE p_target_id = ?1 OR s_target_id = ?1" + ), + [node.num_id()], + |row| row.get(0), + )?; + + if assigned_groups > 0 { + bail!("The target belonging to meta node {node} is part of a buddy group"); + } + + let target_has_root_inode: usize = tx.query_row( + sql!("SELECT COUNT(*) FROM root_inode WHERE target_id = ?1"), + [node.num_id()], + |row| row.get(0), + )?; + + if target_has_root_inode > 0 { + bail!("The target belonging to meta node {node} has the root inode"); + } + + // There should be exactly one meta target per meta node + tx.execute( + sql!("DELETE FROM targets WHERE node_id = ?1 AND node_type = ?2"), + params![node.num_id(), NodeType::Meta.sql_variant()], + )?; + } else { + let assigned_targets: usize = tx.query_row_cached( + sql!("SELECT COUNT(*) FROM targets_ext WHERE node_uid = ?1"), + [node.uid], + |row| row.get(0), + )?; + + if assigned_targets > 0 { + bail!("Node {node} still has targets assigned"); + } + } + + db::node::delete(&tx, node.uid)?; + + if execute { + tx.commit()?; + } + Ok(node) + }) + .await?; + + if execute { + log::info!("Node deleted: {node}"); + + notify_nodes( + &ctx, + match node.node_type() { + NodeType::Meta => &[NodeType::Meta, NodeType::Client], + NodeType::Storage => &[NodeType::Meta, NodeType::Storage, NodeType::Client], + _ => &[], + }, + &RemoveNode { + node_type: node.node_type(), + node_id: node.num_id(), + ack_id: "".into(), + }, + ) + .await; + } + + Ok(pm::DeleteNodeResponse { + node: Some(node.into()), + }) +} diff --git a/mgmtd/src/grpc/delete_pool.rs b/mgmtd/src/grpc/delete_pool.rs new file mode 100644 index 0000000..7810074 --- /dev/null +++ b/mgmtd/src/grpc/delete_pool.rs @@ -0,0 +1,65 @@ +use super::*; +use shared::bee_msg::storage_pool::RefreshStoragePools; + +/// Deletes a pool. The pool must be empty. +pub(crate) async fn delete_pool( + ctx: Context, + req: pm::DeletePoolRequest, +) -> Result { + needs_license(&ctx, LicensedFeature::Storagepool)?; + fail_on_pre_shutdown(&ctx)?; + + let pool: EntityId = required_field(req.pool)?.try_into()?; + let execute: bool = required_field(req.execute)?; + + let pool = ctx + .db + .conn(move |conn| { + let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + + let pool = pool.resolve(&tx, EntityType::Pool)?; + + let assigned_targets: usize = tx.query_row( + sql!("SELECT COUNT(*) FROM storage_targets WHERE pool_id = ?1"), + [pool.num_id()], + |row| row.get(0), + )?; + + let assigned_buddy_groups: usize = tx.query_row( + sql!("SELECT COUNT(*) FROM storage_buddy_groups WHERE pool_id = ?1"), + [pool.num_id()], + |row| row.get(0), + )?; + + if assigned_targets > 0 || assigned_buddy_groups > 0 { + bail!( + "{assigned_targets} targets and {assigned_buddy_groups} buddy groups \ +are still assigned to this pool" + ) + } + + let affected = tx.execute(sql!("DELETE FROM pools WHERE pool_uid = ?1"), [pool.uid])?; + check_affected_rows(affected, [1])?; + + if execute { + tx.commit()?; + } + Ok(pool) + }) + .await?; + + if execute { + log::info!("Pool deleted: {pool}"); + + notify_nodes( + &ctx, + &[NodeType::Meta, NodeType::Storage], + &RefreshStoragePools { ack_id: "".into() }, + ) + .await; + } + + Ok(pm::DeletePoolResponse { + pool: Some(pool.into()), + }) +} diff --git a/mgmtd/src/grpc/delete_target.rs b/mgmtd/src/grpc/delete_target.rs new file mode 100644 index 0000000..ed0bc5c --- /dev/null +++ b/mgmtd/src/grpc/delete_target.rs @@ -0,0 +1,63 @@ +use super::*; +use shared::bee_msg::misc::RefreshCapacityPools; + +/// Deletes a target +pub(crate) async fn delete_target( + ctx: Context, + req: pm::DeleteTargetRequest, +) -> Result { + fail_on_pre_shutdown(&ctx)?; + + let target: EntityId = required_field(req.target)?.try_into()?; + let execute: bool = required_field(req.execute)?; + + let target = ctx + .db + .conn(move |conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + + let target = target.resolve(&tx, EntityType::Target)?; + + if target.node_type() != NodeType::Storage { + bail!("Only storage targets can be deleted directly"); + } + + let assigned_groups: usize = tx.query_row_cached( + sql!( + "SELECT COUNT(*) FROM buddy_groups_ext + WHERE p_target_uid = ?1 OR s_target_uid = ?1" + ), + [target.uid], + |row| row.get(0), + )?; + + if assigned_groups > 0 { + bail!("Target {target} is part of a buddy group"); + } + + db::target::delete_storage(&tx, target.num_id().try_into()?)?; + + if execute { + tx.commit()?; + } + Ok(target) + }) + .await?; + + if execute { + log::info!("Target deleted: {target}"); + + notify_nodes( + &ctx, + &[NodeType::Meta], + &RefreshCapacityPools { ack_id: "".into() }, + ) + .await; + } + + let target = Some(target.into()); + + log::warn!("{target:?}"); + + Ok(pm::DeleteTargetResponse { target }) +} diff --git a/mgmtd/src/grpc/get_buddy_groups.rs b/mgmtd/src/grpc/get_buddy_groups.rs new file mode 100644 index 0000000..65667b2 --- /dev/null +++ b/mgmtd/src/grpc/get_buddy_groups.rs @@ -0,0 +1,76 @@ +use super::*; + +/// Delivers the list of buddy groups +pub(crate) async fn get_buddy_groups( + ctx: Context, + _req: pm::GetBuddyGroupsRequest, +) -> Result { + let buddy_groups = ctx + .db + .read_tx(|tx| { + Ok(tx.query_map_collect( + sql!( + "SELECT group_uid, group_id, bg.alias, bg.node_type, + p_target_uid, p_t.target_id, p_t.alias, + s_target_uid, s_t.target_id, s_t.alias, + p.pool_uid, bg.pool_id, p.alias, + p_t.consistency, s_t.consistency + FROM buddy_groups_ext AS bg + INNER JOIN targets_ext AS p_t ON p_t.target_uid = p_target_uid + INNER JOIN targets_ext AS s_t ON s_t.target_uid = s_target_uid + LEFT JOIN pools_ext AS p USING(node_type, pool_id)" + ), + [], + |row| { + let node_type = NodeType::from_row(row, 3)?.into_proto_i32(); + let p_con_state = TargetConsistencyState::from_row(row, 13)?.into_proto_i32(); + let s_con_state = TargetConsistencyState::from_row(row, 14)?.into_proto_i32(); + + Ok(pm::get_buddy_groups_response::BuddyGroup { + id: Some(pb::EntityIdSet { + uid: row.get(0)?, + legacy_id: Some(pb::LegacyId { + num_id: row.get(1)?, + node_type, + }), + alias: row.get(2)?, + }), + node_type, + primary_target: Some(pb::EntityIdSet { + uid: row.get(4)?, + legacy_id: Some(pb::LegacyId { + num_id: row.get(5)?, + node_type, + }), + alias: row.get(6)?, + }), + secondary_target: Some(pb::EntityIdSet { + uid: row.get(7)?, + legacy_id: Some(pb::LegacyId { + num_id: row.get(8)?, + node_type, + }), + alias: row.get(9)?, + }), + storage_pool: if let Some(uid) = row.get::<_, Option>(10)? { + Some(pb::EntityIdSet { + uid: Some(uid), + legacy_id: Some(pb::LegacyId { + num_id: row.get(11)?, + node_type, + }), + alias: row.get(12)?, + }) + } else { + None + }, + primary_consistency_state: p_con_state, + secondary_consistency_state: s_con_state, + }) + }, + )?) + }) + .await?; + + Ok(pm::GetBuddyGroupsResponse { buddy_groups }) +} diff --git a/mgmtd/src/grpc/license.rs b/mgmtd/src/grpc/get_license.rs similarity index 93% rename from mgmtd/src/grpc/license.rs rename to mgmtd/src/grpc/get_license.rs index 906e115..2343b9d 100644 --- a/mgmtd/src/grpc/license.rs +++ b/mgmtd/src/grpc/get_license.rs @@ -1,7 +1,7 @@ use super::*; use protobuf::management::{self as pm, GetLicenseResponse}; -pub(crate) async fn get( +pub(crate) async fn get_license( ctx: Context, req: pm::GetLicenseRequest, ) -> Result { diff --git a/mgmtd/src/grpc/node.rs b/mgmtd/src/grpc/get_nodes.rs similarity index 65% rename from mgmtd/src/grpc/node.rs rename to mgmtd/src/grpc/get_nodes.rs index 2b97adf..5877cf1 100644 --- a/mgmtd/src/grpc/node.rs +++ b/mgmtd/src/grpc/get_nodes.rs @@ -1,10 +1,12 @@ use super::*; -use shared::bee_msg::node::RemoveNode; use std::net::{IpAddr, Ipv6Addr}; use std::str::FromStr; /// Delivers a list of nodes -pub(crate) async fn get(ctx: Context, req: pm::GetNodesRequest) -> Result { +pub(crate) async fn get_nodes( + ctx: Context, + req: pm::GetNodesRequest, +) -> Result { let (mut nodes, nics, meta_root_node, meta_root_buddy_group, fs_uuid) = ctx .db .read_tx(move |tx| { @@ -158,99 +160,3 @@ pub(crate) async fn get(ctx: Context, req: pm::GetNodesRequest) -> Result Result { - fail_on_pre_shutdown(&ctx)?; - - let node: EntityId = required_field(req.node)?.try_into()?; - let execute: bool = required_field(req.execute)?; - - let node = ctx - .db - .conn(move |conn| { - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - - let node = node.resolve(&tx, EntityType::Node)?; - - if node.uid == MGMTD_UID { - bail!("Management node can not be deleted"); - } - - // Meta nodes have an auto-assigned target which needs to be deleted first. - if node.node_type() == NodeType::Meta { - let assigned_groups: usize = tx.query_row_cached( - sql!( - "SELECT COUNT(*) FROM meta_buddy_groups - WHERE p_target_id = ?1 OR s_target_id = ?1" - ), - [node.num_id()], - |row| row.get(0), - )?; - - if assigned_groups > 0 { - bail!("The target belonging to meta node {node} is part of a buddy group"); - } - - let target_has_root_inode: usize = tx.query_row( - sql!("SELECT COUNT(*) FROM root_inode WHERE target_id = ?1"), - [node.num_id()], - |row| row.get(0), - )?; - - if target_has_root_inode > 0 { - bail!("The target belonging to meta node {node} has the root inode"); - } - - // There should be exactly one meta target per meta node - tx.execute( - sql!("DELETE FROM targets WHERE node_id = ?1 AND node_type = ?2"), - params![node.num_id(), NodeType::Meta.sql_variant()], - )?; - } else { - let assigned_targets: usize = tx.query_row_cached( - sql!("SELECT COUNT(*) FROM targets_ext WHERE node_uid = ?1"), - [node.uid], - |row| row.get(0), - )?; - - if assigned_targets > 0 { - bail!("Node {node} still has targets assigned"); - } - } - - db::node::delete(&tx, node.uid)?; - - if execute { - tx.commit()?; - } - Ok(node) - }) - .await?; - - if execute { - log::info!("Node deleted: {node}"); - - notify_nodes( - &ctx, - match node.node_type() { - NodeType::Meta => &[NodeType::Meta, NodeType::Client], - NodeType::Storage => &[NodeType::Meta, NodeType::Storage, NodeType::Client], - _ => &[], - }, - &RemoveNode { - node_type: node.node_type(), - node_id: node.num_id(), - ack_id: "".into(), - }, - ) - .await; - } - - Ok(pm::DeleteNodeResponse { - node: Some(node.into()), - }) -} diff --git a/mgmtd/src/grpc/get_pools.rs b/mgmtd/src/grpc/get_pools.rs new file mode 100644 index 0000000..52f8df3 --- /dev/null +++ b/mgmtd/src/grpc/get_pools.rs @@ -0,0 +1,134 @@ +use super::*; + +/// Delivers the list of pools +pub(crate) async fn get_pools( + ctx: Context, + req: pm::GetPoolsRequest, +) -> Result { + let (mut pools, targets, buddy_groups) = ctx + .db + .read_tx(move |tx| { + let make_sp = |row: &Row| -> rusqlite::Result { + Ok(pm::get_pools_response::StoragePool { + id: Some(pb::EntityIdSet { + uid: row.get(0)?, + legacy_id: Some(pb::LegacyId { + num_id: row.get(1)?, + node_type: pb::NodeType::Storage.into(), + }), + alias: row.get(2)?, + }), + ..Default::default() + }) + }; + + let pools: Vec<_> = if req.with_quota_limits { + tx.query_map_collect( + sql!( + "SELECT p.pool_uid, p.pool_id, alias, + qus.value, qui.value, qgs.value, qgi.value + FROM storage_pools AS p + INNER JOIN entities ON uid = pool_uid + LEFT JOIN quota_default_limits AS qus ON qus.pool_id = p.pool_id + AND qus.id_type = :user AND qus.quota_type = :space + LEFT JOIN quota_default_limits AS qui ON qui.pool_id = p.pool_id + AND qui.id_type = :user AND qui.quota_type = :inode + LEFT JOIN quota_default_limits AS qgs ON qgs.pool_id = p.pool_id + AND qgs.id_type = :group AND qgs.quota_type = :space + LEFT JOIN quota_default_limits AS qgi ON qgi.pool_id = p.pool_id + AND qgi.id_type = :group AND qgi.quota_type = :inode" + ), + named_params![ + ":user": QuotaIdType::User.sql_variant(), + ":group": QuotaIdType::Group.sql_variant(), + ":space": QuotaType::Space.sql_variant(), + ":inode": QuotaType::Inode.sql_variant() + ], + |row| { + let mut sp = make_sp(row)?; + sp.user_space_limit = row.get::<_, Option>(3)?.or(Some(-1)); + sp.user_inode_limit = row.get::<_, Option>(4)?.or(Some(-1)); + sp.group_space_limit = row.get::<_, Option>(5)?.or(Some(-1)); + sp.group_inode_limit = row.get::<_, Option>(6)?.or(Some(-1)); + Ok(sp) + }, + )? + } else { + tx.query_map_collect( + sql!( + "SELECT pool_uid, pool_id, alias + FROM storage_pools + INNER JOIN entities ON uid = pool_uid" + ), + [], + make_sp, + )? + }; + + let targets: Vec<(Uid, _)> = tx.query_map_collect( + sql!( + "SELECT target_uid, target_id, alias, pool_uid + FROM storage_targets + INNER JOIN entities ON uid = target_uid + INNER JOIN pools USING(node_type, pool_id)" + ), + [], + |row| { + Ok(( + row.get(3)?, + pb::EntityIdSet { + uid: row.get(0)?, + legacy_id: Some(pb::LegacyId { + num_id: row.get(1)?, + node_type: pb::NodeType::Storage.into(), + }), + alias: row.get(2)?, + }, + )) + }, + )?; + + let buddy_groups: Vec<(Uid, _)> = tx.query_map_collect( + sql!( + "SELECT group_uid, group_id, alias, pool_uid + FROM storage_buddy_groups + INNER JOIN entities ON uid = group_uid + INNER JOIN pools USING(pool_id)" + ), + [], + |row| { + Ok(( + row.get(3)?, + pb::EntityIdSet { + uid: row.get(0)?, + legacy_id: Some(pb::LegacyId { + num_id: row.get(1)?, + node_type: pb::NodeType::Storage.into(), + }), + alias: row.get(2)?, + }, + )) + }, + )?; + + Ok((pools, targets, buddy_groups)) + }) + .await?; + + // Merge pool, target and buddy group lists together + for p in &mut pools { + for t in &targets { + if p.id.as_ref().is_some_and(|e| e.uid == Some(t.0)) { + p.targets.push(t.1.clone()); + } + } + + for t in &buddy_groups { + if p.id.as_ref().is_some_and(|e| e.uid == Some(t.0)) { + p.buddy_groups.push(t.1.clone()); + } + } + } + + Ok(pm::GetPoolsResponse { pools }) +} diff --git a/mgmtd/src/grpc/get_quota_limits.rs b/mgmtd/src/grpc/get_quota_limits.rs new file mode 100644 index 0000000..9219301 --- /dev/null +++ b/mgmtd/src/grpc/get_quota_limits.rs @@ -0,0 +1,134 @@ +use super::common::{QUOTA_NOT_ENABLED_STR, QUOTA_STREAM_BUF_SIZE, QUOTA_STREAM_PAGE_LIMIT}; +use super::*; +use itertools::Itertools; +use std::fmt::Write; + +pub(crate) async fn get_quota_limits( + ctx: Context, + req: pm::GetQuotaLimitsRequest, +) -> Result> { + needs_license(&ctx, LicensedFeature::Quota)?; + + if !ctx.info.user_config.quota_enable { + bail!(QUOTA_NOT_ENABLED_STR); + } + + let pool_id = if let Some(pool) = req.pool { + let pool: EntityId = pool.try_into()?; + let pool_id = ctx + .db + .read_tx(move |tx| pool.resolve(tx, EntityType::Pool)) + .await? + .num_id(); + Some(pool_id) + } else { + None + }; + + let mut r#where = "FALSE ".to_string(); + + let mut filter = + |min: Option, max: Option, list: &[u32], typ: QuotaIdType| -> Result<()> { + if min.is_some() || max.is_some() || !list.is_empty() { + write!(r#where, "OR (l.id_type = {} ", typ.sql_variant())?; + + if min.is_some() || max.is_some() { + write!( + r#where, + "AND l.quota_id BETWEEN {} AND {} ", + min.unwrap_or(0), + max.unwrap_or(u32::MAX) + )?; + } + if !list.is_empty() { + write!(r#where, "AND l.quota_id IN ({}) ", list.iter().join(","))?; + } + if let Some(pool_id) = pool_id { + write!(r#where, "AND l.pool_id = {pool_id} ")?; + } + + write!(r#where, ") ")?; + } + + Ok(()) + }; + + filter( + req.user_id_min, + req.user_id_max, + &req.user_id_list, + QuotaIdType::User, + )?; + + filter( + req.group_id_min, + req.group_id_max, + &req.group_id_list, + QuotaIdType::Group, + )?; + + let sql = format!( + "SELECT l.quota_id, l.id_type, l.pool_id, sp.alias, sp.pool_uid, + MAX(CASE WHEN l.quota_type = {space} THEN l.value END) AS space_limit, + MAX(CASE WHEN l.quota_type = {inode} THEN l.value END) AS inode_limit + FROM quota_limits AS l + INNER JOIN pools_ext AS sp USING(node_type, pool_id) + WHERE {where} + GROUP BY l.quota_id, l.id_type, l.pool_id + LIMIT ?1, ?2", + space = QuotaType::Space.sql_variant(), + inode = QuotaType::Inode.sql_variant() + ); + + let stream = resp_stream(QUOTA_STREAM_BUF_SIZE, async move |stream| { + let mut offset = 0; + + loop { + let sql = sql.clone(); + let entries: Vec<_> = ctx + .db + .read_tx(move |tx| { + tx.query_map_collect(&sql, [offset, QUOTA_STREAM_PAGE_LIMIT], |row| { + Ok(pm::QuotaInfo { + pool: Some(pb::EntityIdSet { + uid: row.get(4)?, + legacy_id: Some(pb::LegacyId { + num_id: row.get(2)?, + node_type: pb::NodeType::Storage.into(), + }), + alias: row.get(3)?, + }), + id_type: QuotaIdType::from_row(row, 1)?.into_proto_i32(), + quota_id: Some(row.get(0)?), + space_limit: row.get(5)?, + inode_limit: row.get(6)?, + space_used: None, + inode_used: None, + }) + }) + .map_err(Into::into) + }) + .await?; + + let len = entries.len(); + + // Send the entries to the client + for entry in entries { + stream + .send(pm::GetQuotaLimitsResponse { + limits: Some(entry), + }) + .await?; + } + + // This was the last page? Then we are done + if len < QUOTA_STREAM_PAGE_LIMIT { + return Ok(()); + } + + offset += QUOTA_STREAM_PAGE_LIMIT; + } + }); + + Ok(stream) +} diff --git a/mgmtd/src/grpc/get_quota_usage.rs b/mgmtd/src/grpc/get_quota_usage.rs new file mode 100644 index 0000000..cdb05f3 --- /dev/null +++ b/mgmtd/src/grpc/get_quota_usage.rs @@ -0,0 +1,168 @@ +use super::common::{QUOTA_NOT_ENABLED_STR, QUOTA_STREAM_BUF_SIZE, QUOTA_STREAM_PAGE_LIMIT}; +use super::*; +use itertools::Itertools; +use std::fmt::Write; + +pub(crate) async fn get_quota_usage( + ctx: Context, + req: pm::GetQuotaUsageRequest, +) -> Result> { + needs_license(&ctx, LicensedFeature::Quota)?; + + if !ctx.info.user_config.quota_enable { + bail!(QUOTA_NOT_ENABLED_STR); + } + + let mut r#where = "FALSE ".to_string(); + + let mut filter = + |min: Option, max: Option, list: &[u32], typ: QuotaIdType| -> Result<()> { + if min.is_some() || max.is_some() || !list.is_empty() { + write!(r#where, "OR (u.id_type = {} ", typ.sql_variant())?; + + if min.is_some() || max.is_some() { + write!( + r#where, + "AND u.quota_id BETWEEN {} AND {} ", + min.unwrap_or(0), + max.unwrap_or(u32::MAX) + )?; + } + if !list.is_empty() { + write!(r#where, "AND u.quota_id IN ({}) ", list.iter().join(","))?; + } + + write!(r#where, ") ")?; + } + + Ok(()) + }; + + filter( + req.user_id_min, + req.user_id_max, + &req.user_id_list, + QuotaIdType::User, + )?; + + filter( + req.group_id_min, + req.group_id_max, + &req.group_id_list, + QuotaIdType::Group, + )?; + + let mut having = "TRUE ".to_string(); + + if let Some(pool) = req.pool { + let pool: EntityId = pool.try_into()?; + let pool_uid = ctx + .db + .read_tx(move |tx| pool.resolve(tx, EntityType::Pool)) + .await? + .uid; + + write!(having, "AND sp.pool_uid = {pool_uid} ")?; + } + if let Some(exceeded) = req.exceeded { + let base = "(space_used > space_limit AND space_limit > -1 + OR inode_used > inode_limit AND inode_limit > -1)"; + if exceeded { + write!(having, "AND {base} ")?; + } else { + write!(having, "AND NOT {base} ")?; + } + } + + let sql = format!( + "SELECT u.quota_id, u.id_type, sp.pool_id, sp.alias, sp.pool_uid, + MAX(CASE WHEN u.quota_type = {space} THEN + COALESCE(l.value, d.value, -1) + END) AS space_limit, + MAX(CASE WHEN u.quota_type = {inode} THEN + COALESCE(l.value, d.value, -1) + END) AS inode_limit, + SUM(CASE WHEN u.quota_type = {space} THEN u.value END) AS space_used, + SUM(CASE WHEN u.quota_type = {inode} THEN u.value END) AS inode_used + FROM quota_usage AS u + INNER JOIN targets AS st USING(node_type, target_id) + INNER JOIN pools_ext AS sp USING(node_type, pool_id) + LEFT JOIN quota_default_limits AS d USING(id_type, quota_type, pool_id) + LEFT JOIN quota_limits AS l USING(quota_id, id_type, quota_type, pool_id) + WHERE {where} + GROUP BY u.quota_id, u.id_type, st.pool_id + HAVING {having} + LIMIT ?1, ?2", + space = QuotaType::Space.sql_variant(), + inode = QuotaType::Inode.sql_variant() + ); + + let stream = resp_stream(QUOTA_STREAM_BUF_SIZE, async move |stream| { + let mut offset = 0; + + loop { + let sql = sql.clone(); + let entries: Vec<_> = ctx + .db + .read_tx(move |tx| { + tx.query_map_collect(&sql, [offset, QUOTA_STREAM_PAGE_LIMIT], |row| { + Ok(pm::QuotaInfo { + pool: Some(pb::EntityIdSet { + uid: row.get(4)?, + legacy_id: Some(pb::LegacyId { + num_id: row.get(2)?, + node_type: pb::NodeType::Storage.into(), + }), + alias: row.get(3)?, + }), + id_type: QuotaIdType::from_row(row, 1)?.into_proto_i32(), + quota_id: Some(row.get(0)?), + space_limit: row.get(5)?, + inode_limit: row.get(6)?, + space_used: row.get(7)?, + inode_used: row.get(8)?, + }) + }) + .map_err(Into::into) + }) + .await?; + + let len = entries.len(); + let mut entries = entries.into_iter(); + + // If this if the first entry, include the quota refresh period. Do not send it again + // after to minimize message size. + if offset == 0 { + if let Some(entry) = entries.next() { + stream + .send(pm::GetQuotaUsageResponse { + entry: Some(entry), + refresh_period_s: Some( + ctx.info.user_config.quota_update_interval.as_secs(), + ), + }) + .await?; + } + } + + // Send all the (remaining) entries to the client + for entry in entries { + stream + .send(pm::GetQuotaUsageResponse { + entry: Some(entry), + refresh_period_s: None, + }) + .await?; + } + + // This was the last page? Then we are done + if len < QUOTA_STREAM_PAGE_LIMIT { + return Ok(()); + } + + offset += QUOTA_STREAM_PAGE_LIMIT; + } + }); + + Ok(stream) +} diff --git a/mgmtd/src/grpc/target.rs b/mgmtd/src/grpc/get_targets.rs similarity index 60% rename from mgmtd/src/grpc/target.rs rename to mgmtd/src/grpc/get_targets.rs index afce3ee..af9985e 100644 --- a/mgmtd/src/grpc/target.rs +++ b/mgmtd/src/grpc/get_targets.rs @@ -1,10 +1,5 @@ use super::*; use crate::cap_pool::{CapPoolCalculator, CapacityInfo}; -use shared::bee_msg::OpsErr; -use shared::bee_msg::misc::RefreshCapacityPools; -use shared::bee_msg::target::{ - RefreshTargetStates, SetTargetConsistencyStates, SetTargetConsistencyStatesResp, -}; use std::time::Duration; impl CapacityInfo for &pm::get_targets_response::Target { @@ -18,7 +13,7 @@ impl CapacityInfo for &pm::get_targets_response::Target { } /// Delivers the list of targets -pub(crate) async fn get( +pub(crate) async fn get_targets( ctx: Context, _req: pm::GetTargetsRequest, ) -> Result { @@ -162,125 +157,3 @@ pub(crate) async fn get( Ok(pm::GetTargetsResponse { targets }) } - -/// Deletes a target -pub(crate) async fn delete( - ctx: Context, - req: pm::DeleteTargetRequest, -) -> Result { - fail_on_pre_shutdown(&ctx)?; - - let target: EntityId = required_field(req.target)?.try_into()?; - let execute: bool = required_field(req.execute)?; - - let target = ctx - .db - .conn(move |conn| { - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - - let target = target.resolve(&tx, EntityType::Target)?; - - if target.node_type() != NodeType::Storage { - bail!("Only storage targets can be deleted directly"); - } - - let assigned_groups: usize = tx.query_row_cached( - sql!( - "SELECT COUNT(*) FROM buddy_groups_ext - WHERE p_target_uid = ?1 OR s_target_uid = ?1" - ), - [target.uid], - |row| row.get(0), - )?; - - if assigned_groups > 0 { - bail!("Target {target} is part of a buddy group"); - } - - db::target::delete_storage(&tx, target.num_id().try_into()?)?; - - if execute { - tx.commit()?; - } - Ok(target) - }) - .await?; - - if execute { - log::info!("Target deleted: {target}"); - - notify_nodes( - &ctx, - &[NodeType::Meta], - &RefreshCapacityPools { ack_id: "".into() }, - ) - .await; - } - - let target = Some(target.into()); - - log::warn!("{target:?}"); - - Ok(pm::DeleteTargetResponse { target }) -} - -/// Set consistency state for a target -pub(crate) async fn set_state( - ctx: Context, - req: pm::SetTargetStateRequest, -) -> Result { - fail_on_pre_shutdown(&ctx)?; - - let state: TargetConsistencyState = req.consistency_state().try_into()?; - let target: EntityId = required_field(req.target)?.try_into()?; - - let (target, node_uid) = ctx - .db - .write_tx(move |tx| { - let target = target.resolve(tx, EntityType::Target)?; - - let node: i64 = tx.query_row_cached( - sql!("SELECT node_uid FROM targets_ext WHERE target_uid = ?1"), - [target.uid], - |row| row.get(0), - )?; - - db::target::update_consistency_states( - tx, - [(target.num_id().try_into()?, state)], - NodeTypeServer::try_from(target.node_type())?, - )?; - - Ok((target, node)) - }) - .await?; - - let resp: SetTargetConsistencyStatesResp = ctx - .conn - .request( - node_uid, - &SetTargetConsistencyStates { - node_type: target.node_type(), - target_ids: vec![target.num_id().try_into().unwrap()], - states: vec![state], - ack_id: "".into(), - set_online: 0, - }, - ) - .await?; - if resp.result != OpsErr::SUCCESS { - bail!( - "Management successfully set the target state, but the target {target} failed to process it: {:?}", - resp.result - ); - } - - notify_nodes( - &ctx, - &[NodeType::Meta, NodeType::Storage, NodeType::Client], - &RefreshTargetStates { ack_id: "".into() }, - ) - .await; - - Ok(pm::SetTargetStateResponse {}) -} diff --git a/mgmtd/src/grpc/mirror_root_inode.rs b/mgmtd/src/grpc/mirror_root_inode.rs new file mode 100644 index 0000000..cc02558 --- /dev/null +++ b/mgmtd/src/grpc/mirror_root_inode.rs @@ -0,0 +1,95 @@ +use super::*; +use db::misc::MetaRoot; +use shared::bee_msg::OpsErr; +use shared::bee_msg::buddy_group::{SetMetadataMirroring, SetMetadataMirroringResp}; + +/// Enable metadata mirroring for the root directory +pub(crate) async fn mirror_root_inode( + ctx: Context, + _req: pm::MirrorRootInodeRequest, +) -> Result { + needs_license(&ctx, LicensedFeature::Mirroring)?; + fail_on_pre_shutdown(&ctx)?; + + let offline_timeout = ctx.info.user_config.node_offline_timeout.as_secs(); + let meta_root = ctx + .db + .read_tx(move |tx| { + let node_uid = match db::misc::get_meta_root(tx)? { + MetaRoot::Normal(_, node_uid) => node_uid, + MetaRoot::Mirrored(_) => bail!("Root inode is already mirrored"), + MetaRoot::Unknown => bail!("Root inode unknown"), + }; + + let count = tx.query_row( + sql!( + "SELECT COUNT(*) FROM root_inode AS ri + INNER JOIN buddy_groups AS mg + ON mg.p_target_id = ri.target_id AND mg.node_type = ?1" + ), + [NodeType::Meta.sql_variant()], + |row| row.get::<_, i64>(0), + )?; + + if count < 1 { + bail!("The meta target holding the root inode is not part of a buddy group."); + } + + // Check that no clients are connected to prevent data corruption. Note that there is + // still a small chance for a client being mounted again before the action is taken on + // the root meta server below. In the end, it's the administrators responsibility to not + // let any client mount during that process. + let clients = tx.query_row(sql!("SELECT COUNT(*) FROM client_nodes"), [], |row| { + row.get::<_, i64>(0) + })?; + + if clients > 0 { + bail!( + "This operation requires that all clients are disconnected/unmounted. \ +{clients} clients are still mounted." + ); + } + + let mut server_stmt = tx.prepare(sql!( + "SELECT COUNT(*) FROM nodes + WHERE node_type = ?1 AND UNIXEPOCH('now') - UNIXEPOCH(last_contact) < ?2 + AND node_uid != ?3" + ))?; + + let metas = server_stmt.query_row( + params![NodeType::Meta.sql_variant(), offline_timeout, node_uid], + |row| row.get::<_, i64>(0), + )?; + let storages = server_stmt.query_row( + params![NodeType::Storage.sql_variant(), offline_timeout, node_uid], + |row| row.get::<_, i64>(0), + )?; + + if metas > 0 || storages > 0 { + bail!( + "This operation requires that all nodes except the root meta node are shut \ +down. {metas} meta nodes (excluding the root meta node) and {storages} storage nodes have \ +communicated during the last {offline_timeout}s." + ); + } + + Ok(node_uid) + }) + .await?; + + let resp: SetMetadataMirroringResp = ctx + .conn + .request(meta_root, &SetMetadataMirroring {}) + .await?; + + match resp.result { + OpsErr::SUCCESS => ctx.db.write_tx(db::misc::enable_metadata_mirroring).await?, + _ => bail!( + "The root meta server failed to mirror the root inode: {:?}", + resp.result + ), + } + + log::info!("Root inode has been mirrored"); + Ok(pm::MirrorRootInodeResponse {}) +} diff --git a/mgmtd/src/grpc/pool.rs b/mgmtd/src/grpc/pool.rs deleted file mode 100644 index 106b689..0000000 --- a/mgmtd/src/grpc/pool.rs +++ /dev/null @@ -1,336 +0,0 @@ -use super::*; -use rusqlite::{Row, named_params}; -use shared::bee_msg::storage_pool::RefreshStoragePools; - -/// Delivers the list of pools -pub(crate) async fn get(ctx: Context, req: pm::GetPoolsRequest) -> Result { - let (mut pools, targets, buddy_groups) = ctx - .db - .read_tx(move |tx| { - let make_sp = |row: &Row| -> rusqlite::Result { - Ok(pm::get_pools_response::StoragePool { - id: Some(pb::EntityIdSet { - uid: row.get(0)?, - legacy_id: Some(pb::LegacyId { - num_id: row.get(1)?, - node_type: pb::NodeType::Storage.into(), - }), - alias: row.get(2)?, - }), - ..Default::default() - }) - }; - - let pools: Vec<_> = if req.with_quota_limits { - tx.query_map_collect( - sql!( - "SELECT p.pool_uid, p.pool_id, alias, - qus.value, qui.value, qgs.value, qgi.value - FROM storage_pools AS p - INNER JOIN entities ON uid = pool_uid - LEFT JOIN quota_default_limits AS qus ON qus.pool_id = p.pool_id - AND qus.id_type = :user AND qus.quota_type = :space - LEFT JOIN quota_default_limits AS qui ON qui.pool_id = p.pool_id - AND qui.id_type = :user AND qui.quota_type = :inode - LEFT JOIN quota_default_limits AS qgs ON qgs.pool_id = p.pool_id - AND qgs.id_type = :group AND qgs.quota_type = :space - LEFT JOIN quota_default_limits AS qgi ON qgi.pool_id = p.pool_id - AND qgi.id_type = :group AND qgi.quota_type = :inode" - ), - named_params![ - ":user": QuotaIdType::User.sql_variant(), - ":group": QuotaIdType::Group.sql_variant(), - ":space": QuotaType::Space.sql_variant(), - ":inode": QuotaType::Inode.sql_variant() - ], - |row| { - let mut sp = make_sp(row)?; - sp.user_space_limit = row.get::<_, Option>(3)?.or(Some(-1)); - sp.user_inode_limit = row.get::<_, Option>(4)?.or(Some(-1)); - sp.group_space_limit = row.get::<_, Option>(5)?.or(Some(-1)); - sp.group_inode_limit = row.get::<_, Option>(6)?.or(Some(-1)); - Ok(sp) - }, - )? - } else { - tx.query_map_collect( - sql!( - "SELECT pool_uid, pool_id, alias - FROM storage_pools - INNER JOIN entities ON uid = pool_uid" - ), - [], - make_sp, - )? - }; - - let targets: Vec<(Uid, _)> = tx.query_map_collect( - sql!( - "SELECT target_uid, target_id, alias, pool_uid - FROM storage_targets - INNER JOIN entities ON uid = target_uid - INNER JOIN pools USING(node_type, pool_id)" - ), - [], - |row| { - Ok(( - row.get(3)?, - pb::EntityIdSet { - uid: row.get(0)?, - legacy_id: Some(pb::LegacyId { - num_id: row.get(1)?, - node_type: pb::NodeType::Storage.into(), - }), - alias: row.get(2)?, - }, - )) - }, - )?; - - let buddy_groups: Vec<(Uid, _)> = tx.query_map_collect( - sql!( - "SELECT group_uid, group_id, alias, pool_uid - FROM storage_buddy_groups - INNER JOIN entities ON uid = group_uid - INNER JOIN pools USING(pool_id)" - ), - [], - |row| { - Ok(( - row.get(3)?, - pb::EntityIdSet { - uid: row.get(0)?, - legacy_id: Some(pb::LegacyId { - num_id: row.get(1)?, - node_type: pb::NodeType::Storage.into(), - }), - alias: row.get(2)?, - }, - )) - }, - )?; - - Ok((pools, targets, buddy_groups)) - }) - .await?; - - // Merge pool, target and buddy group lists together - for p in &mut pools { - for t in &targets { - if p.id.as_ref().is_some_and(|e| e.uid == Some(t.0)) { - p.targets.push(t.1.clone()); - } - } - - for t in &buddy_groups { - if p.id.as_ref().is_some_and(|e| e.uid == Some(t.0)) { - p.buddy_groups.push(t.1.clone()); - } - } - } - - Ok(pm::GetPoolsResponse { pools }) -} - -/// Creates a new pool, optionally assigning targets and groups -pub(crate) async fn create( - ctx: Context, - req: pm::CreatePoolRequest, -) -> Result { - needs_license(&ctx, LicensedFeature::Storagepool)?; - fail_on_pre_shutdown(&ctx)?; - - if req.node_type() != pb::NodeType::Storage { - bail!("node type must be storage"); - } - - let alias: Alias = required_field(req.alias)?.try_into()?; - let num_id: PoolId = req.num_id.unwrap_or_default().try_into()?; - - let (pool_uid, alias, pool_id) = ctx - .db - .write_tx(move |tx| { - let (pool_uid, pool_id) = db::storage_pool::insert(tx, num_id, &alias)?; - assign_pool(tx, pool_id, req.targets, req.buddy_groups)?; - Ok((pool_uid, alias, pool_id)) - }) - .await?; - - let pool = EntityIdSet { - uid: pool_uid, - alias, - legacy_id: LegacyId { - node_type: NodeType::Storage, - num_id: pool_id.into(), - }, - }; - - log::info!("Pool created: {pool}"); - - notify_nodes( - &ctx, - &[NodeType::Meta, NodeType::Storage], - &RefreshStoragePools { ack_id: "".into() }, - ) - .await; - - Ok(pm::CreatePoolResponse { - pool: Some(pool.into()), - }) -} - -/// Assigns a pool to a list of targets and buddy groups. -pub(crate) async fn assign( - ctx: Context, - req: pm::AssignPoolRequest, -) -> Result { - needs_license(&ctx, LicensedFeature::Storagepool)?; - fail_on_pre_shutdown(&ctx)?; - - let pool: EntityId = required_field(req.pool)?.try_into()?; - - let pool = ctx - .db - .write_tx(move |tx| { - let pool = pool.resolve(tx, EntityType::Pool)?; - assign_pool(tx, pool.num_id().try_into()?, req.targets, req.buddy_groups)?; - Ok(pool) - }) - .await?; - - log::info!("Pool assigned: {pool}"); - - notify_nodes( - &ctx, - &[NodeType::Meta, NodeType::Storage], - &RefreshStoragePools { ack_id: "".into() }, - ) - .await; - - Ok(pm::AssignPoolResponse { - pool: Some(pool.into()), - }) -} - -/// Do the actual assign work -fn assign_pool( - tx: &Transaction, - pool_id: PoolId, - targets: Vec, - groups: Vec, -) -> Result<()> { - // Target being part of a buddy group can not be assigned individually - let mut check_group_membership = tx.prepare_cached(sql!( - "SELECT COUNT(*) FROM storage_buddy_groups AS g - INNER JOIN targets AS p_t ON p_t.target_id = g.p_target_id AND p_t.node_type = g.node_type - INNER JOIN targets AS s_t ON s_t.target_id = g.s_target_id AND s_t.node_type = g.node_type - WHERE p_t.target_uid = ?1 OR s_t.target_uid = ?1" - ))?; - - let mut assign_target = tx.prepare_cached(sql!( - "UPDATE targets SET pool_id = ?1 WHERE target_uid = ?2" - ))?; - - // Do the checks and assign for each target in the given list. This is expensive, but shouldn't - // matter as this command should only be run occasionally and not with very large lists of - // targets. - for t in targets { - let eid = EntityId::try_from(t)?; - let target = eid.resolve(tx, EntityType::Target)?; - if check_group_membership.query_row([target.uid], |row| row.get::<_, i64>(0))? > 0 { - bail!("Target {eid} can't be assigned directly as it's part of a buddy group"); - } - - assign_target.execute(params![pool_id, target.uid])?; - } - - let mut assign_group = tx.prepare_cached(sql!( - "UPDATE buddy_groups SET pool_id = ?1 WHERE group_uid = ?2" - ))?; - - // Targets being part of buddy groups are auto-assigned to the new pool - let mut assign_grouped_targets = tx.prepare_cached(sql!( - "UPDATE targets SET pool_id = ?1 - FROM ( - SELECT p_t.target_uid AS p_uid, s_t.target_uid AS s_uid FROM buddy_groups AS g - INNER JOIN targets AS p_t ON p_t.target_id = g.p_target_id AND p_t.node_type = g.node_type - INNER JOIN targets AS s_t ON s_t.target_id = g.s_target_id AND s_t.node_type = g.node_type - WHERE group_uid = ?2 - ) - WHERE target_uid IN (p_uid, s_uid)" - ))?; - - // Assign each group and their targets to the new pool - for g in groups { - let eid = EntityId::try_from(g)?; - let group = eid.resolve(tx, EntityType::BuddyGroup)?; - - assign_group.execute(params![pool_id, group.uid])?; - assign_grouped_targets.execute(params![pool_id, group.uid])?; - } - - Ok(()) -} - -/// Deletes a pool. The pool must be empty. -pub(crate) async fn delete( - ctx: Context, - req: pm::DeletePoolRequest, -) -> Result { - needs_license(&ctx, LicensedFeature::Storagepool)?; - fail_on_pre_shutdown(&ctx)?; - - let pool: EntityId = required_field(req.pool)?.try_into()?; - let execute: bool = required_field(req.execute)?; - - let pool = ctx - .db - .conn(move |conn| { - let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; - - let pool = pool.resolve(&tx, EntityType::Pool)?; - - let assigned_targets: usize = tx.query_row( - sql!("SELECT COUNT(*) FROM storage_targets WHERE pool_id = ?1"), - [pool.num_id()], - |row| row.get(0), - )?; - - let assigned_buddy_groups: usize = tx.query_row( - sql!("SELECT COUNT(*) FROM storage_buddy_groups WHERE pool_id = ?1"), - [pool.num_id()], - |row| row.get(0), - )?; - - if assigned_targets > 0 || assigned_buddy_groups > 0 { - bail!( - "{assigned_targets} targets and {assigned_buddy_groups} buddy groups \ -are still assigned to this pool" - ) - } - - let affected = tx.execute(sql!("DELETE FROM pools WHERE pool_uid = ?1"), [pool.uid])?; - check_affected_rows(affected, [1])?; - - if execute { - tx.commit()?; - } - Ok(pool) - }) - .await?; - - if execute { - log::info!("Pool deleted: {pool}"); - - notify_nodes( - &ctx, - &[NodeType::Meta, NodeType::Storage], - &RefreshStoragePools { ack_id: "".into() }, - ) - .await; - } - - Ok(pm::DeletePoolResponse { - pool: Some(pool.into()), - }) -} diff --git a/mgmtd/src/grpc/quota.rs b/mgmtd/src/grpc/quota.rs deleted file mode 100644 index dceecb7..0000000 --- a/mgmtd/src/grpc/quota.rs +++ /dev/null @@ -1,463 +0,0 @@ -use super::*; -use itertools::Itertools; -use std::cmp::Ordering; -use std::fmt::Write; - -const QUOTA_NOT_ENABLED: &str = "Quota support is not enabled"; - -pub(crate) async fn set_default_quota_limits( - ctx: Context, - req: pm::SetDefaultQuotaLimitsRequest, -) -> Result { - needs_license(&ctx, LicensedFeature::Quota)?; - fail_on_pre_shutdown(&ctx)?; - - if !ctx.info.user_config.quota_enable { - bail!(QUOTA_NOT_ENABLED); - } - - let pool: EntityId = required_field(req.pool)?.try_into()?; - - fn update( - tx: &Transaction, - limit: i64, - pool_id: PoolId, - id_type: QuotaIdType, - quota_type: QuotaType, - ) -> Result<()> { - match limit.cmp(&-1) { - Ordering::Less => bail!("invalid {id_type} {quota_type} limit {limit}"), - Ordering::Equal => { - tx.execute_cached( - sql!( - "DELETE FROM quota_default_limits - WHERE pool_id = ?1 AND id_type = ?2 AND quota_type = ?3" - ), - params![pool_id, id_type.sql_variant(), quota_type.sql_variant()], - )?; - } - Ordering::Greater => { - tx.execute_cached( - sql!( - "REPLACE INTO quota_default_limits (pool_id, id_type, quota_type, value) - VALUES(?1, ?2, ?3, ?4)" - ), - params![ - pool_id, - id_type.sql_variant(), - quota_type.sql_variant(), - limit - ], - )?; - } - } - - Ok(()) - } - - ctx.db - .write_tx(move |tx| { - let pool = pool.resolve(tx, EntityType::Pool)?; - let pool_id: PoolId = pool.num_id().try_into()?; - - if let Some(l) = req.user_space_limit { - update(tx, l, pool_id, QuotaIdType::User, QuotaType::Space)?; - } - if let Some(l) = req.user_inode_limit { - update(tx, l, pool_id, QuotaIdType::User, QuotaType::Inode)?; - } - if let Some(l) = req.group_space_limit { - update(tx, l, pool_id, QuotaIdType::Group, QuotaType::Space)?; - } - if let Some(l) = req.group_inode_limit { - update(tx, l, pool_id, QuotaIdType::Group, QuotaType::Inode)?; - } - - Ok(()) - }) - .await?; - - Ok(pm::SetDefaultQuotaLimitsResponse {}) -} - -pub(crate) async fn set_quota_limits( - ctx: Context, - req: pm::SetQuotaLimitsRequest, -) -> Result { - needs_license(&ctx, LicensedFeature::Quota)?; - fail_on_pre_shutdown(&ctx)?; - - if !ctx.info.user_config.quota_enable { - bail!(QUOTA_NOT_ENABLED); - } - - ctx.db - .write_tx(|tx| { - let mut insert_stmt = tx.prepare_cached(sql!( - "REPLACE INTO quota_limits - (quota_id, id_type, quota_type, pool_id, value) - VALUES (?1, ?2, ?3, ?4, ?5)" - ))?; - - let mut delete_stmt = tx.prepare_cached(sql!( - "DELETE FROM quota_limits - WHERE quota_id = ?1 AND id_type = ?2 AND quota_type = ?3 AND pool_id = ?4" - ))?; - - for lim in req.limits { - let id_type: QuotaIdType = lim.id_type().try_into()?; - let quota_id = required_field(lim.quota_id)?; - - let pool: EntityId = required_field(lim.pool)?.try_into()?; - let pool_id = pool.resolve(tx, EntityType::Pool)?.num_id(); - - if let Some(l) = lim.space_limit { - if l > -1 { - insert_stmt.execute(params![ - quota_id, - id_type.sql_variant(), - QuotaType::Space.sql_variant(), - pool_id, - l - ])? - } else { - delete_stmt.execute(params![ - quota_id, - id_type.sql_variant(), - QuotaType::Space.sql_variant(), - pool_id, - ])? - }; - } - - if let Some(l) = lim.inode_limit { - if l > -1 { - insert_stmt.execute(params![ - quota_id, - id_type.sql_variant(), - QuotaType::Inode.sql_variant(), - pool_id, - l - ])? - } else { - delete_stmt.execute(params![ - quota_id, - id_type.sql_variant(), - QuotaType::Inode.sql_variant(), - pool_id, - ])? - }; - } - } - - Ok(()) - }) - .await?; - - Ok(pm::SetQuotaLimitsResponse {}) -} - -// Fetching pages of 1M from quota_usage takes around 2100ms on my slow developer laptop (using a -// release build). In comparison, a page size of 100k takes around 750ms which is far worse. This -// feels like a good middle point to not let the requester wait too long and not waste too many db -// thread cycles with overhead. -const PAGE_LIMIT: usize = 1_000_000; -// Need to hit a compromise between memory footprint and speed. Bigger is better if multiple -// pages need to be fetched but doesn't matter too much if not. Each entry is roughly 50 - -// 60 bytes, so 100k (= 5-6MB) feels fine. And it is still big enough to give a significant -// boost to throughput for big numbers. -const BUF_SIZE: usize = 100_000; - -pub(crate) async fn get_quota_limits( - ctx: Context, - req: pm::GetQuotaLimitsRequest, -) -> Result> { - needs_license(&ctx, LicensedFeature::Quota)?; - - if !ctx.info.user_config.quota_enable { - bail!(QUOTA_NOT_ENABLED); - } - - let pool_id = if let Some(pool) = req.pool { - let pool: EntityId = pool.try_into()?; - let pool_id = ctx - .db - .read_tx(move |tx| pool.resolve(tx, EntityType::Pool)) - .await? - .num_id(); - Some(pool_id) - } else { - None - }; - - let mut r#where = "FALSE ".to_string(); - - let mut filter = - |min: Option, max: Option, list: &[u32], typ: QuotaIdType| -> Result<()> { - if min.is_some() || max.is_some() || !list.is_empty() { - write!(r#where, "OR (l.id_type = {} ", typ.sql_variant())?; - - if min.is_some() || max.is_some() { - write!( - r#where, - "AND l.quota_id BETWEEN {} AND {} ", - min.unwrap_or(0), - max.unwrap_or(u32::MAX) - )?; - } - if !list.is_empty() { - write!(r#where, "AND l.quota_id IN ({}) ", list.iter().join(","))?; - } - if let Some(pool_id) = pool_id { - write!(r#where, "AND l.pool_id = {pool_id} ")?; - } - - write!(r#where, ") ")?; - } - - Ok(()) - }; - - filter( - req.user_id_min, - req.user_id_max, - &req.user_id_list, - QuotaIdType::User, - )?; - - filter( - req.group_id_min, - req.group_id_max, - &req.group_id_list, - QuotaIdType::Group, - )?; - - let sql = format!( - "SELECT l.quota_id, l.id_type, l.pool_id, sp.alias, sp.pool_uid, - MAX(CASE WHEN l.quota_type = {space} THEN l.value END) AS space_limit, - MAX(CASE WHEN l.quota_type = {inode} THEN l.value END) AS inode_limit - FROM quota_limits AS l - INNER JOIN pools_ext AS sp USING(node_type, pool_id) - WHERE {where} - GROUP BY l.quota_id, l.id_type, l.pool_id - LIMIT ?1, ?2", - space = QuotaType::Space.sql_variant(), - inode = QuotaType::Inode.sql_variant() - ); - - let stream = resp_stream(BUF_SIZE, async move |stream| { - let mut offset = 0; - - loop { - let sql = sql.clone(); - let entries: Vec<_> = ctx - .db - .read_tx(move |tx| { - tx.query_map_collect(&sql, [offset, PAGE_LIMIT], |row| { - Ok(pm::QuotaInfo { - pool: Some(pb::EntityIdSet { - uid: row.get(4)?, - legacy_id: Some(pb::LegacyId { - num_id: row.get(2)?, - node_type: pb::NodeType::Storage.into(), - }), - alias: row.get(3)?, - }), - id_type: QuotaIdType::from_row(row, 1)?.into_proto_i32(), - quota_id: Some(row.get(0)?), - space_limit: row.get(5)?, - inode_limit: row.get(6)?, - space_used: None, - inode_used: None, - }) - }) - .map_err(Into::into) - }) - .await?; - - let len = entries.len(); - - // Send the entries to the client - for entry in entries { - stream - .send(pm::GetQuotaLimitsResponse { - limits: Some(entry), - }) - .await?; - } - - // This was the last page? Then we are done - if len < PAGE_LIMIT { - return Ok(()); - } - - offset += PAGE_LIMIT; - } - }); - - Ok(stream) -} - -pub(crate) async fn get_quota_usage( - ctx: Context, - req: pm::GetQuotaUsageRequest, -) -> Result> { - needs_license(&ctx, LicensedFeature::Quota)?; - - if !ctx.info.user_config.quota_enable { - bail!(QUOTA_NOT_ENABLED); - } - - let mut r#where = "FALSE ".to_string(); - - let mut filter = - |min: Option, max: Option, list: &[u32], typ: QuotaIdType| -> Result<()> { - if min.is_some() || max.is_some() || !list.is_empty() { - write!(r#where, "OR (u.id_type = {} ", typ.sql_variant())?; - - if min.is_some() || max.is_some() { - write!( - r#where, - "AND u.quota_id BETWEEN {} AND {} ", - min.unwrap_or(0), - max.unwrap_or(u32::MAX) - )?; - } - if !list.is_empty() { - write!(r#where, "AND u.quota_id IN ({}) ", list.iter().join(","))?; - } - - write!(r#where, ") ")?; - } - - Ok(()) - }; - - filter( - req.user_id_min, - req.user_id_max, - &req.user_id_list, - QuotaIdType::User, - )?; - - filter( - req.group_id_min, - req.group_id_max, - &req.group_id_list, - QuotaIdType::Group, - )?; - - let mut having = "TRUE ".to_string(); - - if let Some(pool) = req.pool { - let pool: EntityId = pool.try_into()?; - let pool_uid = ctx - .db - .read_tx(move |tx| pool.resolve(tx, EntityType::Pool)) - .await? - .uid; - - write!(having, "AND sp.pool_uid = {pool_uid} ")?; - } - if let Some(exceeded) = req.exceeded { - let base = "(space_used > space_limit AND space_limit > -1 - OR inode_used > inode_limit AND inode_limit > -1)"; - if exceeded { - write!(having, "AND {base} ")?; - } else { - write!(having, "AND NOT {base} ")?; - } - } - - let sql = format!( - "SELECT u.quota_id, u.id_type, sp.pool_id, sp.alias, sp.pool_uid, - MAX(CASE WHEN u.quota_type = {space} THEN - COALESCE(l.value, d.value, -1) - END) AS space_limit, - MAX(CASE WHEN u.quota_type = {inode} THEN - COALESCE(l.value, d.value, -1) - END) AS inode_limit, - SUM(CASE WHEN u.quota_type = {space} THEN u.value END) AS space_used, - SUM(CASE WHEN u.quota_type = {inode} THEN u.value END) AS inode_used - FROM quota_usage AS u - INNER JOIN targets AS st USING(node_type, target_id) - INNER JOIN pools_ext AS sp USING(node_type, pool_id) - LEFT JOIN quota_default_limits AS d USING(id_type, quota_type, pool_id) - LEFT JOIN quota_limits AS l USING(quota_id, id_type, quota_type, pool_id) - WHERE {where} - GROUP BY u.quota_id, u.id_type, st.pool_id - HAVING {having} - LIMIT ?1, ?2", - space = QuotaType::Space.sql_variant(), - inode = QuotaType::Inode.sql_variant() - ); - - let stream = resp_stream(BUF_SIZE, async move |stream| { - let mut offset = 0; - - loop { - let sql = sql.clone(); - let entries: Vec<_> = ctx - .db - .read_tx(move |tx| { - tx.query_map_collect(&sql, [offset, PAGE_LIMIT], |row| { - Ok(pm::QuotaInfo { - pool: Some(pb::EntityIdSet { - uid: row.get(4)?, - legacy_id: Some(pb::LegacyId { - num_id: row.get(2)?, - node_type: pb::NodeType::Storage.into(), - }), - alias: row.get(3)?, - }), - id_type: QuotaIdType::from_row(row, 1)?.into_proto_i32(), - quota_id: Some(row.get(0)?), - space_limit: row.get(5)?, - inode_limit: row.get(6)?, - space_used: row.get(7)?, - inode_used: row.get(8)?, - }) - }) - .map_err(Into::into) - }) - .await?; - - let len = entries.len(); - let mut entries = entries.into_iter(); - - // If this if the first entry, include the quota refresh period. Do not send it again - // after to minimize message size. - if offset == 0 { - if let Some(entry) = entries.next() { - stream - .send(pm::GetQuotaUsageResponse { - entry: Some(entry), - refresh_period_s: Some( - ctx.info.user_config.quota_update_interval.as_secs(), - ), - }) - .await?; - } - } - - // Send all the (remaining) entries to the client - for entry in entries { - stream - .send(pm::GetQuotaUsageResponse { - entry: Some(entry), - refresh_period_s: None, - }) - .await?; - } - - // This was the last page? Then we are done - if len < PAGE_LIMIT { - return Ok(()); - } - - offset += PAGE_LIMIT; - } - }); - - Ok(stream) -} diff --git a/mgmtd/src/grpc/misc.rs b/mgmtd/src/grpc/set_alias.rs similarity index 98% rename from mgmtd/src/grpc/misc.rs rename to mgmtd/src/grpc/set_alias.rs index e18f126..2a07c67 100644 --- a/mgmtd/src/grpc/misc.rs +++ b/mgmtd/src/grpc/set_alias.rs @@ -1,6 +1,6 @@ use super::*; use db::node_nic::map_bee_msg_nics; -use shared::bee_msg::node::*; +use shared::bee_msg::node::Heartbeat; /// Sets the entity alias for any entity pub(crate) async fn set_alias( diff --git a/mgmtd/src/grpc/set_default_quota_limits.rs b/mgmtd/src/grpc/set_default_quota_limits.rs new file mode 100644 index 0000000..9da61d9 --- /dev/null +++ b/mgmtd/src/grpc/set_default_quota_limits.rs @@ -0,0 +1,78 @@ +use super::common::QUOTA_NOT_ENABLED_STR; +use super::*; +use std::cmp::Ordering; + +pub(crate) async fn set_default_quota_limits( + ctx: Context, + req: pm::SetDefaultQuotaLimitsRequest, +) -> Result { + needs_license(&ctx, LicensedFeature::Quota)?; + fail_on_pre_shutdown(&ctx)?; + + if !ctx.info.user_config.quota_enable { + bail!(QUOTA_NOT_ENABLED_STR); + } + + let pool: EntityId = required_field(req.pool)?.try_into()?; + + fn update( + tx: &Transaction, + limit: i64, + pool_id: PoolId, + id_type: QuotaIdType, + quota_type: QuotaType, + ) -> Result<()> { + match limit.cmp(&-1) { + Ordering::Less => bail!("invalid {id_type} {quota_type} limit {limit}"), + Ordering::Equal => { + tx.execute_cached( + sql!( + "DELETE FROM quota_default_limits + WHERE pool_id = ?1 AND id_type = ?2 AND quota_type = ?3" + ), + params![pool_id, id_type.sql_variant(), quota_type.sql_variant()], + )?; + } + Ordering::Greater => { + tx.execute_cached( + sql!( + "REPLACE INTO quota_default_limits (pool_id, id_type, quota_type, value) + VALUES(?1, ?2, ?3, ?4)" + ), + params![ + pool_id, + id_type.sql_variant(), + quota_type.sql_variant(), + limit + ], + )?; + } + } + + Ok(()) + } + + ctx.db + .write_tx(move |tx| { + let pool = pool.resolve(tx, EntityType::Pool)?; + let pool_id: PoolId = pool.num_id().try_into()?; + + if let Some(l) = req.user_space_limit { + update(tx, l, pool_id, QuotaIdType::User, QuotaType::Space)?; + } + if let Some(l) = req.user_inode_limit { + update(tx, l, pool_id, QuotaIdType::User, QuotaType::Inode)?; + } + if let Some(l) = req.group_space_limit { + update(tx, l, pool_id, QuotaIdType::Group, QuotaType::Space)?; + } + if let Some(l) = req.group_inode_limit { + update(tx, l, pool_id, QuotaIdType::Group, QuotaType::Inode)?; + } + + Ok(()) + }) + .await?; + + Ok(pm::SetDefaultQuotaLimitsResponse {}) +} diff --git a/mgmtd/src/grpc/set_quota_limits.rs b/mgmtd/src/grpc/set_quota_limits.rs new file mode 100644 index 0000000..1bb35d9 --- /dev/null +++ b/mgmtd/src/grpc/set_quota_limits.rs @@ -0,0 +1,79 @@ +use super::common::QUOTA_NOT_ENABLED_STR; +use super::*; + +pub(crate) async fn set_quota_limits( + ctx: Context, + req: pm::SetQuotaLimitsRequest, +) -> Result { + needs_license(&ctx, LicensedFeature::Quota)?; + fail_on_pre_shutdown(&ctx)?; + + if !ctx.info.user_config.quota_enable { + bail!(QUOTA_NOT_ENABLED_STR); + } + + ctx.db + .write_tx(|tx| { + let mut insert_stmt = tx.prepare_cached(sql!( + "REPLACE INTO quota_limits + (quota_id, id_type, quota_type, pool_id, value) + VALUES (?1, ?2, ?3, ?4, ?5)" + ))?; + + let mut delete_stmt = tx.prepare_cached(sql!( + "DELETE FROM quota_limits + WHERE quota_id = ?1 AND id_type = ?2 AND quota_type = ?3 AND pool_id = ?4" + ))?; + + for lim in req.limits { + let id_type: QuotaIdType = lim.id_type().try_into()?; + let quota_id = required_field(lim.quota_id)?; + + let pool: EntityId = required_field(lim.pool)?.try_into()?; + let pool_id = pool.resolve(tx, EntityType::Pool)?.num_id(); + + if let Some(l) = lim.space_limit { + if l > -1 { + insert_stmt.execute(params![ + quota_id, + id_type.sql_variant(), + QuotaType::Space.sql_variant(), + pool_id, + l + ])? + } else { + delete_stmt.execute(params![ + quota_id, + id_type.sql_variant(), + QuotaType::Space.sql_variant(), + pool_id, + ])? + }; + } + + if let Some(l) = lim.inode_limit { + if l > -1 { + insert_stmt.execute(params![ + quota_id, + id_type.sql_variant(), + QuotaType::Inode.sql_variant(), + pool_id, + l + ])? + } else { + delete_stmt.execute(params![ + quota_id, + id_type.sql_variant(), + QuotaType::Inode.sql_variant(), + pool_id, + ])? + }; + } + } + + Ok(()) + }) + .await?; + + Ok(pm::SetQuotaLimitsResponse {}) +} diff --git a/mgmtd/src/grpc/set_target_state.rs b/mgmtd/src/grpc/set_target_state.rs new file mode 100644 index 0000000..bf98e95 --- /dev/null +++ b/mgmtd/src/grpc/set_target_state.rs @@ -0,0 +1,66 @@ +use super::*; +use shared::bee_msg::OpsErr; +use shared::bee_msg::target::{ + RefreshTargetStates, SetTargetConsistencyStates, SetTargetConsistencyStatesResp, +}; + +/// Set consistency state for a target +pub(crate) async fn set_target_state( + ctx: Context, + req: pm::SetTargetStateRequest, +) -> Result { + fail_on_pre_shutdown(&ctx)?; + + let state: TargetConsistencyState = req.consistency_state().try_into()?; + let target: EntityId = required_field(req.target)?.try_into()?; + + let (target, node_uid) = ctx + .db + .write_tx(move |tx| { + let target = target.resolve(tx, EntityType::Target)?; + + let node: i64 = tx.query_row_cached( + sql!("SELECT node_uid FROM targets_ext WHERE target_uid = ?1"), + [target.uid], + |row| row.get(0), + )?; + + db::target::update_consistency_states( + tx, + [(target.num_id().try_into()?, state)], + NodeTypeServer::try_from(target.node_type())?, + )?; + + Ok((target, node)) + }) + .await?; + + let resp: SetTargetConsistencyStatesResp = ctx + .conn + .request( + node_uid, + &SetTargetConsistencyStates { + node_type: target.node_type(), + target_ids: vec![target.num_id().try_into().unwrap()], + states: vec![state], + ack_id: "".into(), + set_online: 0, + }, + ) + .await?; + if resp.result != OpsErr::SUCCESS { + bail!( + "Management successfully set the target state, but the target {target} failed to process it: {:?}", + resp.result + ); + } + + notify_nodes( + &ctx, + &[NodeType::Meta, NodeType::Storage, NodeType::Client], + &RefreshTargetStates { ack_id: "".into() }, + ) + .await; + + Ok(pm::SetTargetStateResponse {}) +} diff --git a/mgmtd/src/grpc/start_resync.rs b/mgmtd/src/grpc/start_resync.rs new file mode 100644 index 0000000..ded2407 --- /dev/null +++ b/mgmtd/src/grpc/start_resync.rs @@ -0,0 +1,203 @@ +use super::*; +use shared::bee_msg::OpsErr; +use shared::bee_msg::buddy_group::{ + BuddyResyncJobState, GetMetaResyncStats, GetMetaResyncStatsResp, GetStorageResyncStats, + GetStorageResyncStatsResp, SetLastBuddyCommOverride, +}; +use shared::bee_msg::target::{RefreshTargetStates, SetTargetConsistencyStatesResp}; +use std::time::{Duration, Instant}; +use tokio::time::sleep; + +/// Starts a resync of a storage or metadata target from its buddy target +pub(crate) async fn start_resync( + ctx: Context, + req: pm::StartResyncRequest, +) -> Result { + needs_license(&ctx, LicensedFeature::Mirroring)?; + fail_on_pre_shutdown(&ctx)?; + + let buddy_group: EntityId = required_field(req.buddy_group)?.try_into()?; + let timestamp: i64 = required_field(req.timestamp)?; + let restart: bool = required_field(req.restart)?; + + // For resync source is always primary target and destination is secondary target + let (src_target_id, dest_target_id, src_node_uid, node_type, group) = ctx + .db + .read_tx(move |tx| { + let group = buddy_group.resolve(tx, EntityType::BuddyGroup)?; + let node_type: NodeTypeServer = group.node_type().try_into()?; + + let (src_target_id, dest_target_id, src_node_uid): (TargetId, TargetId, Uid) = tx + .query_row_cached( + sql!( + "SELECT g.p_target_id, g.s_target_id, src_t.node_uid + FROM buddy_groups AS g + INNER JOIN targets_ext AS src_t + ON src_t.target_id = g.p_target_id AND src_t.node_type = g.node_type + WHERE group_uid = ?1" + ), + [group.uid], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + + Ok(( + src_target_id, + dest_target_id, + src_node_uid, + node_type, + group, + )) + }) + .await?; + + // We handle meta and storage servers separately as storage servers allow restarts and metas + // dont. + match node_type { + NodeTypeServer::Meta => { + if timestamp > -1 { + bail!( + "Metadata targets can only do full resync, timestamp and timespan is \ +not supported." + ); + } + if restart { + bail!("Resync cannot be restarted or aborted for metadata servers."); + } + + let resp: GetMetaResyncStatsResp = ctx + .conn + .request( + src_node_uid, + &GetMetaResyncStats { + target_id: src_target_id, + }, + ) + .await?; + + if resp.state == BuddyResyncJobState::Running { + bail!("Resync already running on buddy group {group}"); + } + } + NodeTypeServer::Storage => { + if !restart { + let resp: GetStorageResyncStatsResp = ctx + .conn + .request( + src_node_uid, + &GetStorageResyncStats { + target_id: src_target_id, + }, + ) + .await?; + + if resp.state == BuddyResyncJobState::Running { + bail!("Resync already running on buddy group {group}"); + } + + if timestamp > -1 { + override_last_buddy_comm(&ctx, src_node_uid, src_target_id, &group, timestamp) + .await?; + } + } else { + if timestamp < 0 { + bail!("Resync for storage targets can only be restarted with timestamp."); + } + + override_last_buddy_comm(&ctx, src_node_uid, src_target_id, &group, timestamp) + .await?; + + log::info!("Waiting for the already running resync operations to abort."); + + let timeout = tokio::time::Duration::from_secs(180); + let start = Instant::now(); + + // This sleep and poll loop is bad style, but the simplest way to do it for + // now. A better solution would be to intercept the message from the server that + // tells us the resync is finished, but that is a bit more complex and, with the + // current system, still unreliable. + loop { + let resp: GetStorageResyncStatsResp = ctx + .conn + .request( + src_node_uid, + &GetStorageResyncStats { + target_id: src_target_id, + }, + ) + .await?; + + if resp.state != BuddyResyncJobState::Running { + break; + } + + if start.elapsed() >= timeout { + bail!("Timeout. Unable to abort resync on buddy group {group}"); + } + + sleep(Duration::from_secs(2)).await; + } + } + } + } + + // set destination target state as needs-resync in mgmtd database + ctx.db + .write_tx(move |tx| { + db::target::update_consistency_states( + tx, + [(dest_target_id, TargetConsistencyState::NeedsResync)], + node_type, + )?; + Ok(()) + }) + .await?; + + // This also triggers the source node to fetch the new needs resync state and start the resync + // using the internode syncer loop. In case of overriding last buddy communication on storage + // nodes, this means that there is a max 3s window where the communication timestamp can be + // overwritten again before resync starts, effectively ignoring it. There is nothing we can do + // about that without changing the storage server. + // + // Note that sending a SetTargetConsistencyStateMsg does have no effect on making this quicker, + // so we omit it. + notify_nodes( + &ctx, + &[NodeType::Meta, NodeType::Storage, NodeType::Client], + &RefreshTargetStates { ack_id: "".into() }, + ) + .await; + + return Ok(pm::StartResyncResponse {}); + + /// Override last buddy communication timestamp on source storage node + /// Note that this might be overwritten again on the storage server between + async fn override_last_buddy_comm( + ctx: &Context, + src_node_uid: Uid, + src_target_id: TargetId, + group: &EntityIdSet, + timestamp: i64, + ) -> Result<()> { + let resp: SetTargetConsistencyStatesResp = ctx + .conn + .request( + src_node_uid, + &SetLastBuddyCommOverride { + target_id: src_target_id, + timestamp, + abort_resync: 0, + }, + ) + .await?; + + if resp.result != OpsErr::SUCCESS { + bail!( + "Could not override last buddy communication timestamp on primary of buddy group {group}. \ +Failed with resp {:?}", + resp.result + ); + } + + Ok(()) + } +} diff --git a/shared/src/grpc.rs b/shared/src/grpc.rs index 34bafb7..14fc31f 100644 --- a/shared/src/grpc.rs +++ b/shared/src/grpc.rs @@ -20,23 +20,23 @@ use tonic::{Code, Status}; #[macro_export] macro_rules! impl_grpc_handler { // Implements the function for a response stream RPC. - ($impl_fn:ident => $handle_fn:path, $req_msg:path => STREAM($resp_stream:ident, $resp_msg:path), $ctx_str:literal) => { + ($impl_fn:ident, $req_msg:path => STREAM($resp_stream:ident, $resp_msg:path), $ctx_str:literal) => { // A response stream RPC requires to define a `Stream` associated type and use this // as the response for the handler. type $resp_stream = RespStream<$resp_msg>; - impl_grpc_handler!(@INNER $impl_fn => $handle_fn, $req_msg => Self::$resp_stream, $ctx_str); + impl_grpc_handler!(@INNER $impl_fn, $req_msg => Self::$resp_stream, $ctx_str); }; // Implements the function for a unary RPC. - ($impl_fn:ident => $handle_fn:path, $req_msg:path => $resp_msg:path, $ctx_str:literal) => { - impl_grpc_handler!(@INNER $impl_fn => $handle_fn, $req_msg => $resp_msg, $ctx_str); + ($impl_fn:ident, $req_msg:path => $resp_msg:path, $ctx_str:literal) => { + impl_grpc_handler!(@INNER $impl_fn, $req_msg => $resp_msg, $ctx_str); }; // Generates the actual function. Note that we implement the `async fn` manually to avoid having // to use `#[tonic::async_trait]`. This is exactly how that macro does it in the background, but // we can't rely on that here within this macro as attribute macros are evaluated first. - (@INNER $impl_fn:ident => $handle_fn:path, $req_msg:path => $resp_msg:path, $ctx_str:literal) => { + (@INNER $impl_fn:ident, $req_msg:path => $resp_msg:path, $ctx_str:literal) => { fn $impl_fn<'a, 'async_trait>( &'a self, req: Request<$req_msg>, @@ -46,7 +46,11 @@ macro_rules! impl_grpc_handler { Self: 'async_trait, { Box::pin(async move { - let res = $handle_fn(self.ctx.clone(), req.into_inner()).await; + // The self.app is misplaced here as this is supposed to be a reusable macro. + // It assumes what is passed to the handler function and that might be different + // for different users of this. + // I don't have a quick idea how to fix this in an elegant way, so we keep it for. + let res = $impl_fn::$impl_fn(self.ctx.clone(), req.into_inner()).await; match res { Ok(res) => Ok(Response::new(res)), From bbed2d81ec544930c799cde40159865274158648 Mon Sep 17 00:00:00 2001 From: Rusty Bee <145002912+rustybee42@users.noreply.github.com> Date: Tue, 1 Apr 2025 15:14:24 +0200 Subject: [PATCH 4/9] refactor: move BeeMsg handlers to their own files --- mgmtd/src/bee_msg.rs | 33 +- mgmtd/src/bee_msg/ack.rs | 9 + mgmtd/src/bee_msg/authenticate_channel.rs | 24 + .../change_target_consistency_states.rs | 63 +++ mgmtd/src/bee_msg/common.rs | 260 +++++++++++ mgmtd/src/bee_msg/get_mirror_buddy_groups.rs | 41 ++ .../{misc.rs => get_node_capacity_pools.rs} | 62 --- mgmtd/src/bee_msg/get_nodes.rs | 59 +++ ...roup.rs => get_states_and_buddy_groups.rs} | 48 +- .../{storage_pool.rs => get_storage_pools.rs} | 0 mgmtd/src/bee_msg/get_target_mappings.rs | 28 ++ mgmtd/src/bee_msg/get_target_states.rs | 42 ++ mgmtd/src/bee_msg/heartbeat.rs | 34 ++ mgmtd/src/bee_msg/heartbeat_request.rs | 40 ++ mgmtd/src/bee_msg/map_targets.rs | 61 +++ mgmtd/src/bee_msg/map_targets_resp.rs | 10 + mgmtd/src/bee_msg/node.rs | 393 ----------------- mgmtd/src/bee_msg/peer_info.rs | 10 + mgmtd/src/bee_msg/refresh_capacity_pools.rs | 17 + mgmtd/src/bee_msg/register_node.rs | 25 ++ mgmtd/src/bee_msg/register_target.rs | 48 ++ mgmtd/src/bee_msg/remove_node.rs | 60 +++ mgmtd/src/bee_msg/remove_node_resp.rs | 10 + .../{quota.rs => request_exceeded_quota.rs} | 0 mgmtd/src/bee_msg/set_channel_direct.rs | 9 + .../bee_msg/set_mirror_buddy_groups_resp.rs | 9 + mgmtd/src/bee_msg/set_storage_target_info.rs | 48 ++ .../bee_msg/set_target_consistency_states.rs | 50 +++ mgmtd/src/bee_msg/target.rs | 412 ------------------ 29 files changed, 985 insertions(+), 920 deletions(-) create mode 100644 mgmtd/src/bee_msg/ack.rs create mode 100644 mgmtd/src/bee_msg/authenticate_channel.rs create mode 100644 mgmtd/src/bee_msg/change_target_consistency_states.rs create mode 100644 mgmtd/src/bee_msg/common.rs create mode 100644 mgmtd/src/bee_msg/get_mirror_buddy_groups.rs rename mgmtd/src/bee_msg/{misc.rs => get_node_capacity_pools.rs} (79%) create mode 100644 mgmtd/src/bee_msg/get_nodes.rs rename mgmtd/src/bee_msg/{buddy_group.rs => get_states_and_buddy_groups.rs} (60%) rename mgmtd/src/bee_msg/{storage_pool.rs => get_storage_pools.rs} (100%) create mode 100644 mgmtd/src/bee_msg/get_target_mappings.rs create mode 100644 mgmtd/src/bee_msg/get_target_states.rs create mode 100644 mgmtd/src/bee_msg/heartbeat.rs create mode 100644 mgmtd/src/bee_msg/heartbeat_request.rs create mode 100644 mgmtd/src/bee_msg/map_targets.rs create mode 100644 mgmtd/src/bee_msg/map_targets_resp.rs delete mode 100644 mgmtd/src/bee_msg/node.rs create mode 100644 mgmtd/src/bee_msg/peer_info.rs create mode 100644 mgmtd/src/bee_msg/refresh_capacity_pools.rs create mode 100644 mgmtd/src/bee_msg/register_node.rs create mode 100644 mgmtd/src/bee_msg/register_target.rs create mode 100644 mgmtd/src/bee_msg/remove_node.rs create mode 100644 mgmtd/src/bee_msg/remove_node_resp.rs rename mgmtd/src/bee_msg/{quota.rs => request_exceeded_quota.rs} (100%) create mode 100644 mgmtd/src/bee_msg/set_channel_direct.rs create mode 100644 mgmtd/src/bee_msg/set_mirror_buddy_groups_resp.rs create mode 100644 mgmtd/src/bee_msg/set_storage_target_info.rs create mode 100644 mgmtd/src/bee_msg/set_target_consistency_states.rs delete mode 100644 mgmtd/src/bee_msg/target.rs diff --git a/mgmtd/src/bee_msg.rs b/mgmtd/src/bee_msg.rs index 0d2a542..6aa9218 100644 --- a/mgmtd/src/bee_msg.rs +++ b/mgmtd/src/bee_msg.rs @@ -18,12 +18,33 @@ use sqlite_check::sql; use std::collections::HashMap; use std::fmt::Display; -mod buddy_group; -mod misc; -mod node; -mod quota; -mod storage_pool; -mod target; +mod common; + +mod ack; +mod authenticate_channel; +mod change_target_consistency_states; +mod get_mirror_buddy_groups; +mod get_node_capacity_pools; +mod get_nodes; +mod get_states_and_buddy_groups; +mod get_storage_pools; +mod get_target_mappings; +mod get_target_states; +mod heartbeat; +mod heartbeat_request; +mod map_targets; +mod map_targets_resp; +mod peer_info; +mod refresh_capacity_pools; +mod register_node; +mod register_target; +mod remove_node; +mod remove_node_resp; +mod request_exceeded_quota; +mod set_channel_direct; +mod set_mirror_buddy_groups_resp; +mod set_storage_target_info; +mod set_target_consistency_states; /// Msg request handler for requests where no response is expected. /// To handle a message, implement this and add it to the dispatch list with `=> _`. diff --git a/mgmtd/src/bee_msg/ack.rs b/mgmtd/src/bee_msg/ack.rs new file mode 100644 index 0000000..3eb9bad --- /dev/null +++ b/mgmtd/src/bee_msg/ack.rs @@ -0,0 +1,9 @@ +use super::*; +use shared::bee_msg::misc::*; + +impl HandleNoResponse for Ack { + async fn handle(self, _ctx: &Context, req: &mut impl Request) -> Result<()> { + log::debug!("Ignoring Ack from {:?}: Id: {:?}", req.addr(), self.ack_id); + Ok(()) + } +} diff --git a/mgmtd/src/bee_msg/authenticate_channel.rs b/mgmtd/src/bee_msg/authenticate_channel.rs new file mode 100644 index 0000000..9d8a5e3 --- /dev/null +++ b/mgmtd/src/bee_msg/authenticate_channel.rs @@ -0,0 +1,24 @@ +use super::*; +use shared::bee_msg::misc::*; + +impl HandleNoResponse for AuthenticateChannel { + async fn handle(self, ctx: &Context, req: &mut impl Request) -> Result<()> { + if let Some(ref secret) = ctx.info.auth_secret { + if secret == &self.auth_secret { + req.authenticate_connection(); + } else { + log::error!( + "Peer {:?} tried to authenticate stream with wrong secret", + req.addr() + ); + } + } else { + log::debug!( + "Peer {:?} tried to authenticate stream, but authentication is not required", + req.addr() + ); + } + + Ok(()) + } +} diff --git a/mgmtd/src/bee_msg/change_target_consistency_states.rs b/mgmtd/src/bee_msg/change_target_consistency_states.rs new file mode 100644 index 0000000..20433f4 --- /dev/null +++ b/mgmtd/src/bee_msg/change_target_consistency_states.rs @@ -0,0 +1,63 @@ +use super::*; +use common::update_last_contact_times; +use shared::bee_msg::target::*; + +impl HandleWithResponse for ChangeTargetConsistencyStates { + type Response = ChangeTargetConsistencyStatesResp; + + fn error_response() -> Self::Response { + ChangeTargetConsistencyStatesResp { + result: OpsErr::INTERNAL, + } + } + + async fn handle(self, ctx: &Context, __req: &mut impl Request) -> Result { + fail_on_pre_shutdown(ctx)?; + + // self.old_states is currently completely ignored. If something reports a non-GOOD state, I + // see no apparent reason to that the old state matches before setting. We have the + // authority, whatever nodes think their old state was doesn't matter. + + let changed = ctx + .db + .write_tx(move |tx| { + let node_type = self.node_type.try_into()?; + + // Check given target Ids exist + db::target::validate_ids(tx, &self.target_ids, node_type)?; + + // Old management updates contact time while handling this message (comes usually in + // every 30 seconds), so we do it as well + update_last_contact_times(tx, &self.target_ids, node_type)?; + + let affected = db::target::update_consistency_states( + tx, + self.target_ids + .into_iter() + .zip(self.new_states.iter().copied()), + node_type, + )?; + + Ok(affected > 0) + }) + .await?; + + log::debug!( + "Updated target consistency states for {:?} nodes", + self.node_type + ); + + if changed { + notify_nodes( + ctx, + &[NodeType::Meta, NodeType::Storage, NodeType::Client], + &RefreshTargetStates { ack_id: "".into() }, + ) + .await; + } + + Ok(ChangeTargetConsistencyStatesResp { + result: OpsErr::SUCCESS, + }) + } +} diff --git a/mgmtd/src/bee_msg/common.rs b/mgmtd/src/bee_msg/common.rs new file mode 100644 index 0000000..28ff36a --- /dev/null +++ b/mgmtd/src/bee_msg/common.rs @@ -0,0 +1,260 @@ +use super::*; +use crate::db::node_nic::ReplaceNic; +use db::misc::MetaRoot; +use rusqlite::Transaction; +use shared::bee_msg::node::*; +use shared::bee_msg::target::*; +use shared::types::{NodeId, TargetId}; +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +/// Processes incoming node information. Registers new nodes if config allows it +pub(super) async fn update_node(msg: RegisterNode, ctx: &Context) -> Result { + let nics = msg.nics.clone(); + let requested_node_id = msg.node_id; + let info = ctx.info; + + let licensed_machines = match ctx.license.get_num_machines() { + Ok(n) => n, + Err(err) => { + log::debug!( + "Could not obtain number of licensed machines, defaulting to unlimited: {err:#}" + ); + u32::MAX + } + }; + + let (node, meta_root, is_new) = ctx + .db + .write_tx_no_sync(move |tx| { + let node = if msg.node_id == 0 { + // No node ID given => new node + None + } else { + // If Some is returned, the node with node_id already exists + try_resolve_num_id(tx, EntityType::Node, msg.node_type, msg.node_id)? + }; + + let machine_uuid = if matches!(msg.node_type, NodeType::Meta | NodeType::Storage) + && !msg.machine_uuid.is_empty() + { + Some(std::str::from_utf8(&msg.machine_uuid)?) + } else { + None + }; + + if let Some(machine_uuid) = machine_uuid { + if db::node::count_machines(tx, machine_uuid, node.as_ref().map(|n| n.uid))? + >= licensed_machines + { + bail!("Licensed machine limit reached. Node registration denied."); + } + } + + let (node, is_new) = if let Some(node) = node { + // Existing node, update data + db::node::update(tx, node.uid, msg.port, machine_uuid)?; + + (node, false) + } else { + // New node, do additional checks and insert data + + // Check node registration is allowed. This should ignore registering client + // nodes. + if msg.node_type != NodeType::Client && info.user_config.registration_disable { + bail!("Registration of new nodes is not allowed"); + } + + let new_alias = if msg.node_type == NodeType::Client { + // In versions prior to 8.0 the string node ID generated by the client + // started with a number which is not allowed by the new alias schema. + // As part of BeeGFS 8 the nodeID generated for each client mount was + // updated to no longer start with a number, thus it is unlikely this + // would happen unless BeeGFS 8 was mounted by a BeeGFS 7 client. + + let new_alias = String::from_utf8(msg.node_alias) + .ok() + .and_then(|s| Alias::try_from(s).ok()); + + if new_alias.is_none() { + log::warn!( + "Unable to use alias requested by client (possibly the\ +client version < 8.0)" + ); + } + new_alias + } else { + None + }; + + // Insert new node entry + let node = db::node::insert(tx, msg.node_id, new_alias, msg.node_type, msg.port)?; + + // if this is a meta node, auto-add a corresponding meta target after the node. + if msg.node_type == NodeType::Meta { + // Convert the NodeID to a TargetID. Due to the difference in bitsize, meta + // node IDs are not allowed to be bigger than u16 + let Ok(target_id) = TargetId::try_from(node.num_id()) else { + bail!( + "{} is not a valid numeric meta node id\ +(must be between 1 and 65535)", + node.num_id() + ); + }; + + db::target::insert_meta(tx, target_id, None)?; + } + + (node, true) + }; + + // Update the corresponding nic lists + db::node_nic::replace( + tx, + node.uid, + msg.nics.iter().map(|e| ReplaceNic { + nic_type: e.nic_type, + addr: &e.addr, + name: String::from_utf8_lossy(&e.name), + }), + )?; + + let meta_root = match node.node_type() { + // In case this is a meta node, the requester expects info about the meta + // root + NodeType::Meta => db::misc::get_meta_root(tx)?, + _ => MetaRoot::Unknown, + }; + + Ok((node, meta_root, is_new)) + }) + .await?; + + ctx.conn.replace_node_addrs( + node.uid, + nics.clone() + .into_iter() + .map(|e| SocketAddr::new(e.addr, msg.port)) + .collect::>(), + ); + + if is_new { + log::info!("Registered new node {node} (Requested Numeric Id: {requested_node_id})",); + } else { + log::debug!("Updated node {node} node",); + } + + let node_num_id = node.num_id(); + + // notify all nodes + notify_nodes( + ctx, + match node.node_type() { + NodeType::Meta => &[NodeType::Meta, NodeType::Client], + NodeType::Storage => &[NodeType::Meta, NodeType::Storage, NodeType::Client], + NodeType::Client => &[NodeType::Meta], + _ => &[], + }, + &Heartbeat { + instance_version: 0, + nic_list_version: 0, + node_type: node.node_type(), + node_alias: String::from(node.alias).into_bytes(), + ack_id: "".into(), + node_num_id, + root_num_id: match meta_root { + MetaRoot::Unknown => 0, + MetaRoot::Normal(node_id, _) => node_id, + MetaRoot::Mirrored(group_id) => group_id.into(), + }, + is_root_mirrored: match meta_root { + MetaRoot::Unknown | MetaRoot::Normal(_, _) => 0, + MetaRoot::Mirrored(_) => 1, + }, + port: msg.port, + port_tcp_unused: msg.port, + nic_list: nics, + machine_uuid: vec![], // No need for the other nodes to know machine UUIDs + }, + ) + .await; + + Ok(node_num_id) +} + +pub(super) fn get_targets_with_states( + tx: &Transaction, + pre_shutdown: bool, + node_type: NodeTypeServer, + node_offline_timeout: Duration, +) -> Result> { + let targets = tx.query_map_collect( + sql!( + "SELECT t.target_id, t.consistency, + (UNIXEPOCH('now') - UNIXEPOCH(t.last_update)), gp.p_target_id, gs.s_target_id + FROM targets AS t + INNER JOIN nodes AS n USING(node_type, node_id) + LEFT JOIN buddy_groups AS gp ON gp.p_target_id = t.target_id AND gp.node_type = t.node_type + LEFT JOIN buddy_groups AS gs ON gs.s_target_id = t.target_id AND gs.node_type = t.node_type + WHERE t.node_type = ?1" + ), + [node_type.sql_variant()], + |row| { + let is_primary = row.get::<_, Option>(3)?.is_some(); + let is_secondary = row.get::<_, Option>(4)?.is_some(); + + Ok(( + row.get(0)?, + TargetConsistencyState::from_row(row, 1)?, + if !pre_shutdown || is_secondary { + let age = Duration::from_secs(row.get(2)?); + + // We never want to report a primary node of a buddy group as offline since this + // is considered invalid. Instead we just report ProbablyOffline and wait for the switchover. + if !is_primary && age > node_offline_timeout { + TargetReachabilityState::Offline + } else if age > node_offline_timeout / 2 { + TargetReachabilityState::ProbablyOffline + } else { + TargetReachabilityState::Online + } + } else { + TargetReachabilityState::ProbablyOffline + }, + )) + }, + )?; + + Ok(targets) +} + +/// Updates the `last_contact` time for all the nodes belonging to the passed targets and the +/// targets `last_update` times themselves. +pub(super) fn update_last_contact_times( + tx: &Transaction, + target_ids: &[TargetId], + node_type: NodeTypeServer, +) -> Result<()> { + let target_ids_param = sqlite::rarray_param(target_ids.iter().copied()); + + tx.execute_cached( + sql!( + "UPDATE nodes AS n SET last_contact = DATETIME('now') + WHERE n.node_uid IN ( + SELECT DISTINCT node_uid FROM targets_ext + WHERE target_id IN rarray(?1) AND node_type = ?2)" + ), + rusqlite::params![&target_ids_param, node_type.sql_variant()], + )?; + + tx.execute_cached( + sql!( + "UPDATE targets SET last_update = DATETIME('now') + WHERE target_id IN rarray(?1) AND node_type = ?2" + ), + rusqlite::params![&target_ids_param, node_type.sql_variant()], + )?; + + Ok(()) +} diff --git a/mgmtd/src/bee_msg/get_mirror_buddy_groups.rs b/mgmtd/src/bee_msg/get_mirror_buddy_groups.rs new file mode 100644 index 0000000..1995826 --- /dev/null +++ b/mgmtd/src/bee_msg/get_mirror_buddy_groups.rs @@ -0,0 +1,41 @@ +use super::*; +use shared::bee_msg::buddy_group::*; + +impl HandleWithResponse for GetMirrorBuddyGroups { + type Response = GetMirrorBuddyGroupsResp; + + async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + let groups: Vec<(BuddyGroupId, TargetId, TargetId)> = ctx + .db + .read_tx(move |tx| { + tx.query_map_collect( + sql!( + "SELECT group_id, p_target_id, s_target_id FROM buddy_groups_ext + WHERE node_type = ?1" + ), + [self.node_type.sql_variant()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .map_err(Into::into) + }) + .await?; + + let mut buddy_groups = Vec::with_capacity(groups.len()); + let mut primary_targets = Vec::with_capacity(groups.len()); + let mut secondary_targets = Vec::with_capacity(groups.len()); + + for g in groups { + buddy_groups.push(g.0); + primary_targets.push(g.1); + secondary_targets.push(g.2); + } + + let resp = GetMirrorBuddyGroupsResp { + buddy_groups, + primary_targets, + secondary_targets, + }; + + Ok(resp) + } +} diff --git a/mgmtd/src/bee_msg/misc.rs b/mgmtd/src/bee_msg/get_node_capacity_pools.rs similarity index 79% rename from mgmtd/src/bee_msg/misc.rs rename to mgmtd/src/bee_msg/get_node_capacity_pools.rs index 9594819..50e2918 100644 --- a/mgmtd/src/bee_msg/misc.rs +++ b/mgmtd/src/bee_msg/get_node_capacity_pools.rs @@ -2,68 +2,6 @@ use super::*; use crate::cap_pool::{CapPoolCalculator, CapacityInfo}; use rusqlite::Transaction; use shared::bee_msg::misc::*; -use shared::types::PoolId; -use sqlite::TransactionExt; -use sqlite_check::sql; - -impl HandleNoResponse for Ack { - async fn handle(self, _ctx: &Context, req: &mut impl Request) -> Result<()> { - log::debug!("Ignoring Ack from {:?}: Id: {:?}", req.addr(), self.ack_id); - Ok(()) - } -} - -impl HandleNoResponse for AuthenticateChannel { - async fn handle(self, ctx: &Context, req: &mut impl Request) -> Result<()> { - if let Some(ref secret) = ctx.info.auth_secret { - if secret == &self.auth_secret { - req.authenticate_connection(); - } else { - log::error!( - "Peer {:?} tried to authenticate stream with wrong secret", - req.addr() - ); - } - } else { - log::debug!( - "Peer {:?} tried to authenticate stream, but authentication is not required", - req.addr() - ); - } - - Ok(()) - } -} - -impl HandleNoResponse for PeerInfo { - async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result<()> { - // This is supposed to give some information about a connection, but it looks - // like this isnt used at all - Ok(()) - } -} - -impl HandleNoResponse for SetChannelDirect { - async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result<()> { - // do nothing - Ok(()) - } -} - -impl HandleWithResponse for RefreshCapacityPools { - type Response = Ack; - - async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result { - // This message is superfluous and therefore ignored. It is meant to tell the - // mgmtd to trigger a capacity pool pull immediately after a node starts. - // meta and storage send a SetTargetInfo before this msg though, - // so we handle triggering pulls there. - - Ok(Ack { - ack_id: self.ack_id, - }) - } -} #[derive(Debug)] struct TargetOrBuddyGroup { diff --git a/mgmtd/src/bee_msg/get_nodes.rs b/mgmtd/src/bee_msg/get_nodes.rs new file mode 100644 index 0000000..165e755 --- /dev/null +++ b/mgmtd/src/bee_msg/get_nodes.rs @@ -0,0 +1,59 @@ +use super::*; +use db::misc::MetaRoot; +use db::node_nic::map_bee_msg_nics; +use shared::bee_msg::node::*; + +impl HandleWithResponse for GetNodes { + type Response = GetNodesResp; + + async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + let res = ctx + .db + .read_tx(move |tx| { + let node_type = self.node_type; + let res = ( + db::node::get_with_type(tx, node_type)?, + db::node_nic::get_with_type(tx, node_type)?, + match self.node_type { + shared::types::NodeType::Meta => db::misc::get_meta_root(tx)?, + _ => MetaRoot::Unknown, + }, + ); + + Ok(res) + }) + .await?; + + let mut nodes: Vec = res + .0 + .into_iter() + .map(|n| Node { + alias: n.alias.into_bytes(), + num_id: n.id, + nic_list: map_bee_msg_nics(res.1.iter().filter(|e| e.node_uid == n.uid).cloned()) + .collect(), + port: n.port, + _unused_tcp_port: n.port, + node_type: n.node_type, + }) + .collect(); + + nodes.sort_by(|a, b| a.num_id.cmp(&b.num_id)); + + let resp = GetNodesResp { + nodes, + root_num_id: match res.2 { + MetaRoot::Unknown => 0, + MetaRoot::Normal(node_id, _) => node_id, + MetaRoot::Mirrored(group_id) => group_id.into(), + }, + is_root_mirrored: match res.2 { + MetaRoot::Unknown => 0, + MetaRoot::Normal(_, _) => 0, + MetaRoot::Mirrored(_) => 1, + }, + }; + + Ok(resp) + } +} diff --git a/mgmtd/src/bee_msg/buddy_group.rs b/mgmtd/src/bee_msg/get_states_and_buddy_groups.rs similarity index 60% rename from mgmtd/src/bee_msg/buddy_group.rs rename to mgmtd/src/bee_msg/get_states_and_buddy_groups.rs index cbb8d53..54459a3 100644 --- a/mgmtd/src/bee_msg/buddy_group.rs +++ b/mgmtd/src/bee_msg/get_states_and_buddy_groups.rs @@ -1,52 +1,6 @@ use super::*; +use common::get_targets_with_states; use shared::bee_msg::buddy_group::*; -use target::get_targets_with_states; - -impl HandleWithResponse for GetMirrorBuddyGroups { - type Response = GetMirrorBuddyGroupsResp; - - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - let groups: Vec<(BuddyGroupId, TargetId, TargetId)> = ctx - .db - .read_tx(move |tx| { - tx.query_map_collect( - sql!( - "SELECT group_id, p_target_id, s_target_id FROM buddy_groups_ext - WHERE node_type = ?1" - ), - [self.node_type.sql_variant()], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .map_err(Into::into) - }) - .await?; - - let mut buddy_groups = Vec::with_capacity(groups.len()); - let mut primary_targets = Vec::with_capacity(groups.len()); - let mut secondary_targets = Vec::with_capacity(groups.len()); - - for g in groups { - buddy_groups.push(g.0); - primary_targets.push(g.1); - secondary_targets.push(g.2); - } - - let resp = GetMirrorBuddyGroupsResp { - buddy_groups, - primary_targets, - secondary_targets, - }; - - Ok(resp) - } -} - -impl HandleNoResponse for SetMirrorBuddyGroupResp { - async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result<()> { - // response from server nodes to SetMirrorBuddyGroup notification - Ok(()) - } -} impl HandleWithResponse for GetStatesAndBuddyGroups { type Response = GetStatesAndBuddyGroupsResp; diff --git a/mgmtd/src/bee_msg/storage_pool.rs b/mgmtd/src/bee_msg/get_storage_pools.rs similarity index 100% rename from mgmtd/src/bee_msg/storage_pool.rs rename to mgmtd/src/bee_msg/get_storage_pools.rs diff --git a/mgmtd/src/bee_msg/get_target_mappings.rs b/mgmtd/src/bee_msg/get_target_mappings.rs new file mode 100644 index 0000000..ba8743d --- /dev/null +++ b/mgmtd/src/bee_msg/get_target_mappings.rs @@ -0,0 +1,28 @@ +use super::*; +use shared::bee_msg::target::*; + +impl HandleWithResponse for GetTargetMappings { + type Response = GetTargetMappingsResp; + + async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + let mapping: HashMap = ctx + .db + .read_tx(move |tx| { + tx.query_map_collect( + sql!( + "SELECT target_id, node_id + FROM storage_targets + WHERE node_id IS NOT NULL" + ), + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(Into::into) + }) + .await?; + + let resp = GetTargetMappingsResp { mapping }; + + Ok(resp) + } +} diff --git a/mgmtd/src/bee_msg/get_target_states.rs b/mgmtd/src/bee_msg/get_target_states.rs new file mode 100644 index 0000000..a9349c3 --- /dev/null +++ b/mgmtd/src/bee_msg/get_target_states.rs @@ -0,0 +1,42 @@ +use super::*; +use common::get_targets_with_states; +use shared::bee_msg::target::*; + +impl HandleWithResponse for GetTargetStates { + type Response = GetTargetStatesResp; + + async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + let pre_shutdown = ctx.run_state.pre_shutdown(); + let node_offline_timeout = ctx.info.user_config.node_offline_timeout; + + let targets = ctx + .db + .read_tx(move |tx| { + get_targets_with_states( + tx, + pre_shutdown, + self.node_type.try_into()?, + node_offline_timeout, + ) + }) + .await?; + + let mut target_ids = Vec::with_capacity(targets.len()); + let mut reachability_states = Vec::with_capacity(targets.len()); + let mut consistency_states = Vec::with_capacity(targets.len()); + + for e in targets { + target_ids.push(e.0); + consistency_states.push(e.1); + reachability_states.push(e.2); + } + + let resp = GetTargetStatesResp { + targets: target_ids, + consistency_states, + reachability_states, + }; + + Ok(resp) + } +} diff --git a/mgmtd/src/bee_msg/heartbeat.rs b/mgmtd/src/bee_msg/heartbeat.rs new file mode 100644 index 0000000..4417a28 --- /dev/null +++ b/mgmtd/src/bee_msg/heartbeat.rs @@ -0,0 +1,34 @@ +use super::*; +use common::update_node; +use shared::bee_msg::misc::Ack; +use shared::bee_msg::node::*; + +impl HandleWithResponse for Heartbeat { + type Response = Ack; + + async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(ctx)?; + + update_node( + RegisterNode { + instance_version: self.instance_version, + nic_list_version: self.nic_list_version, + node_alias: self.node_alias, + nics: self.nic_list, + node_type: self.node_type, + node_id: self.node_num_id, + root_num_id: self.root_num_id, + is_root_mirrored: self.is_root_mirrored, + port: self.port, + port_tcp_unused: self.port_tcp_unused, + machine_uuid: self.machine_uuid, + }, + ctx, + ) + .await?; + + Ok(Ack { + ack_id: self.ack_id, + }) + } +} diff --git a/mgmtd/src/bee_msg/heartbeat_request.rs b/mgmtd/src/bee_msg/heartbeat_request.rs new file mode 100644 index 0000000..095156b --- /dev/null +++ b/mgmtd/src/bee_msg/heartbeat_request.rs @@ -0,0 +1,40 @@ +use super::*; +use db::node_nic::map_bee_msg_nics; +use shared::bee_msg::node::*; + +impl HandleWithResponse for HeartbeatRequest { + type Response = Heartbeat; + + async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + let (alias, nics) = ctx + .db + .read_tx(|tx| { + Ok(( + db::entity::get_alias(tx, MGMTD_UID)? + .ok_or_else(|| TypedError::value_not_found("management uid", MGMTD_UID))?, + db::node_nic::get_with_node(tx, MGMTD_UID)?, + )) + }) + .await + .unwrap_or_default(); + + let (alias, nics) = (alias, map_bee_msg_nics(nics).collect()); + + let resp = Heartbeat { + instance_version: 0, + nic_list_version: 0, + node_type: shared::types::NodeType::Management, + node_alias: alias.into_bytes(), + ack_id: "".into(), + node_num_id: MGMTD_ID, + root_num_id: 0, + is_root_mirrored: 0, + port: ctx.info.user_config.beemsg_port, + port_tcp_unused: ctx.info.user_config.beemsg_port, + nic_list: nics, + machine_uuid: vec![], // No need for the other nodes to know machine UUIDs + }; + + Ok(resp) + } +} diff --git a/mgmtd/src/bee_msg/map_targets.rs b/mgmtd/src/bee_msg/map_targets.rs new file mode 100644 index 0000000..a421ac4 --- /dev/null +++ b/mgmtd/src/bee_msg/map_targets.rs @@ -0,0 +1,61 @@ +use super::*; +use shared::bee_msg::target::*; + +impl HandleWithResponse for MapTargets { + type Response = MapTargetsResp; + + async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(ctx)?; + + let target_ids = self.target_ids.keys().copied().collect::>(); + + ctx.db + .write_tx(move |tx| { + // Check node Id exists + let node = LegacyId { + node_type: NodeType::Storage, + num_id: self.node_id, + } + .resolve(tx, EntityType::Node)?; + // Check all target Ids exist + db::target::validate_ids(tx, &target_ids, NodeTypeServer::Storage)?; + // Due to the check above, this must always match all the given ids + db::target::update_storage_node_mappings(tx, &target_ids, node.num_id())?; + Ok(()) + }) + .await?; + + // At this point, all mappings must have been successful + + log::info!( + "Mapped storage targets with Ids {:?} to node {}", + self.target_ids.keys(), + self.node_id + ); + + notify_nodes( + ctx, + &[NodeType::Meta, NodeType::Storage, NodeType::Client], + &MapTargets { + target_ids: self.target_ids.clone(), + node_id: self.node_id, + ack_id: "".into(), + }, + ) + .await; + + // Storage server expects a separate status code for each target map requested. We, however, + // do a all-or-nothing approach. If e.g. one target id doesn't exist (which is an + // exceptional error and should usually not happen anyway), we fail the whole + // operation. Therefore we can just send a list of successes. + let resp = MapTargetsResp { + results: self + .target_ids + .into_iter() + .map(|e| (e.0, OpsErr::SUCCESS)) + .collect(), + }; + + Ok(resp) + } +} diff --git a/mgmtd/src/bee_msg/map_targets_resp.rs b/mgmtd/src/bee_msg/map_targets_resp.rs new file mode 100644 index 0000000..1b52481 --- /dev/null +++ b/mgmtd/src/bee_msg/map_targets_resp.rs @@ -0,0 +1,10 @@ +use super::*; +use shared::bee_msg::target::*; + +impl HandleNoResponse for MapTargetsResp { + async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result<()> { + // This is sent from the nodes as a result of the MapTargets notification after + // map_targets was called. We just ignore it. + Ok(()) + } +} diff --git a/mgmtd/src/bee_msg/node.rs b/mgmtd/src/bee_msg/node.rs deleted file mode 100644 index 338ccfb..0000000 --- a/mgmtd/src/bee_msg/node.rs +++ /dev/null @@ -1,393 +0,0 @@ -use super::*; -use crate::db::node_nic::ReplaceNic; -use db::misc::MetaRoot; -use db::node_nic::map_bee_msg_nics; -use shared::bee_msg::misc::Ack; -use shared::bee_msg::node::*; -use shared::types::{MGMTD_ID, MGMTD_UID, NodeId, TargetId}; -use std::net::SocketAddr; -use std::sync::Arc; - -impl HandleWithResponse for GetNodes { - type Response = GetNodesResp; - - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - let res = ctx - .db - .read_tx(move |tx| { - let node_type = self.node_type; - let res = ( - db::node::get_with_type(tx, node_type)?, - db::node_nic::get_with_type(tx, node_type)?, - match self.node_type { - shared::types::NodeType::Meta => db::misc::get_meta_root(tx)?, - _ => MetaRoot::Unknown, - }, - ); - - Ok(res) - }) - .await?; - - let mut nodes: Vec = res - .0 - .into_iter() - .map(|n| Node { - alias: n.alias.into_bytes(), - num_id: n.id, - nic_list: map_bee_msg_nics(res.1.iter().filter(|e| e.node_uid == n.uid).cloned()) - .collect(), - port: n.port, - _unused_tcp_port: n.port, - node_type: n.node_type, - }) - .collect(); - - nodes.sort_by(|a, b| a.num_id.cmp(&b.num_id)); - - let resp = GetNodesResp { - nodes, - root_num_id: match res.2 { - MetaRoot::Unknown => 0, - MetaRoot::Normal(node_id, _) => node_id, - MetaRoot::Mirrored(group_id) => group_id.into(), - }, - is_root_mirrored: match res.2 { - MetaRoot::Unknown => 0, - MetaRoot::Normal(_, _) => 0, - MetaRoot::Mirrored(_) => 1, - }, - }; - - Ok(resp) - } -} - -impl HandleWithResponse for Heartbeat { - type Response = Ack; - - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; - - update_node( - RegisterNode { - instance_version: self.instance_version, - nic_list_version: self.nic_list_version, - node_alias: self.node_alias, - nics: self.nic_list, - node_type: self.node_type, - node_id: self.node_num_id, - root_num_id: self.root_num_id, - is_root_mirrored: self.is_root_mirrored, - port: self.port, - port_tcp_unused: self.port_tcp_unused, - machine_uuid: self.machine_uuid, - }, - ctx, - ) - .await?; - - Ok(Ack { - ack_id: self.ack_id, - }) - } -} - -impl HandleWithResponse for HeartbeatRequest { - type Response = Heartbeat; - - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - let (alias, nics) = ctx - .db - .read_tx(|tx| { - Ok(( - db::entity::get_alias(tx, MGMTD_UID)? - .ok_or_else(|| TypedError::value_not_found("management uid", MGMTD_UID))?, - db::node_nic::get_with_node(tx, MGMTD_UID)?, - )) - }) - .await - .unwrap_or_default(); - - let (alias, nics) = (alias, map_bee_msg_nics(nics).collect()); - - let resp = Heartbeat { - instance_version: 0, - nic_list_version: 0, - node_type: shared::types::NodeType::Management, - node_alias: alias.into_bytes(), - ack_id: "".into(), - node_num_id: MGMTD_ID, - root_num_id: 0, - is_root_mirrored: 0, - port: ctx.info.user_config.beemsg_port, - port_tcp_unused: ctx.info.user_config.beemsg_port, - nic_list: nics, - machine_uuid: vec![], // No need for the other nodes to know machine UUIDs - }; - - Ok(resp) - } -} - -impl HandleWithResponse for RegisterNode { - type Response = RegisterNodeResp; - - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; - - let node_id = update_node(self, ctx).await?; - - let fs_uuid: String = ctx - .db - .read_tx(|tx| db::config::get(tx, db::config::Config::FsUuid)) - .await? - .ok_or_else(|| anyhow!("Could not read file system UUID from database"))?; - - Ok(RegisterNodeResp { - node_num_id: node_id, - grpc_port: ctx.info.user_config.grpc_port, - fs_uuid: fs_uuid.into_bytes(), - }) - } -} - -/// Processes incoming node information. Registers new nodes if config allows it -async fn update_node(msg: RegisterNode, ctx: &Context) -> Result { - let nics = msg.nics.clone(); - let requested_node_id = msg.node_id; - let info = ctx.info; - - let licensed_machines = match ctx.license.get_num_machines() { - Ok(n) => n, - Err(err) => { - log::debug!( - "Could not obtain number of licensed machines, defaulting to unlimited: {err:#}" - ); - u32::MAX - } - }; - - let (node, meta_root, is_new) = ctx - .db - .write_tx_no_sync(move |tx| { - let node = if msg.node_id == 0 { - // No node ID given => new node - None - } else { - // If Some is returned, the node with node_id already exists - try_resolve_num_id(tx, EntityType::Node, msg.node_type, msg.node_id)? - }; - - let machine_uuid = if matches!(msg.node_type, NodeType::Meta | NodeType::Storage) - && !msg.machine_uuid.is_empty() - { - Some(std::str::from_utf8(&msg.machine_uuid)?) - } else { - None - }; - - if let Some(machine_uuid) = machine_uuid { - if db::node::count_machines(tx, machine_uuid, node.as_ref().map(|n| n.uid))? - >= licensed_machines - { - bail!("Licensed machine limit reached. Node registration denied."); - } - } - - let (node, is_new) = if let Some(node) = node { - // Existing node, update data - db::node::update(tx, node.uid, msg.port, machine_uuid)?; - - (node, false) - } else { - // New node, do additional checks and insert data - - // Check node registration is allowed. This should ignore registering client - // nodes. - if msg.node_type != NodeType::Client && info.user_config.registration_disable { - bail!("Registration of new nodes is not allowed"); - } - - let new_alias = if msg.node_type == NodeType::Client { - // In versions prior to 8.0 the string node ID generated by the client - // started with a number which is not allowed by the new alias schema. - // As part of BeeGFS 8 the nodeID generated for each client mount was - // updated to no longer start with a number, thus it is unlikely this - // would happen unless BeeGFS 8 was mounted by a BeeGFS 7 client. - - let new_alias = String::from_utf8(msg.node_alias) - .ok() - .and_then(|s| Alias::try_from(s).ok()); - - if new_alias.is_none() { - log::warn!( - "Unable to use alias requested by client (possibly the\ -client version < 8.0)" - ); - } - new_alias - } else { - None - }; - - // Insert new node entry - let node = db::node::insert(tx, msg.node_id, new_alias, msg.node_type, msg.port)?; - - // if this is a meta node, auto-add a corresponding meta target after the node. - if msg.node_type == NodeType::Meta { - // Convert the NodeID to a TargetID. Due to the difference in bitsize, meta - // node IDs are not allowed to be bigger than u16 - let Ok(target_id) = TargetId::try_from(node.num_id()) else { - bail!( - "{} is not a valid numeric meta node id\ -(must be between 1 and 65535)", - node.num_id() - ); - }; - - db::target::insert_meta(tx, target_id, None)?; - } - - (node, true) - }; - - // Update the corresponding nic lists - db::node_nic::replace( - tx, - node.uid, - msg.nics.iter().map(|e| ReplaceNic { - nic_type: e.nic_type, - addr: &e.addr, - name: String::from_utf8_lossy(&e.name), - }), - )?; - - let meta_root = match node.node_type() { - // In case this is a meta node, the requester expects info about the meta - // root - NodeType::Meta => db::misc::get_meta_root(tx)?, - _ => MetaRoot::Unknown, - }; - - Ok((node, meta_root, is_new)) - }) - .await?; - - ctx.conn.replace_node_addrs( - node.uid, - nics.clone() - .into_iter() - .map(|e| SocketAddr::new(e.addr, msg.port)) - .collect::>(), - ); - - if is_new { - log::info!("Registered new node {node} (Requested Numeric Id: {requested_node_id})",); - } else { - log::debug!("Updated node {node} node",); - } - - let node_num_id = node.num_id(); - - // notify all nodes - notify_nodes( - ctx, - match node.node_type() { - NodeType::Meta => &[NodeType::Meta, NodeType::Client], - NodeType::Storage => &[NodeType::Meta, NodeType::Storage, NodeType::Client], - NodeType::Client => &[NodeType::Meta], - _ => &[], - }, - &Heartbeat { - instance_version: 0, - nic_list_version: 0, - node_type: node.node_type(), - node_alias: String::from(node.alias).into_bytes(), - ack_id: "".into(), - node_num_id, - root_num_id: match meta_root { - MetaRoot::Unknown => 0, - MetaRoot::Normal(node_id, _) => node_id, - MetaRoot::Mirrored(group_id) => group_id.into(), - }, - is_root_mirrored: match meta_root { - MetaRoot::Unknown | MetaRoot::Normal(_, _) => 0, - MetaRoot::Mirrored(_) => 1, - }, - port: msg.port, - port_tcp_unused: msg.port, - nic_list: nics, - machine_uuid: vec![], // No need for the other nodes to know machine UUIDs - }, - ) - .await; - - Ok(node_num_id) -} - -impl HandleWithResponse for RemoveNode { - type Response = RemoveNodeResp; - - fn error_response() -> Self::Response { - RemoveNodeResp { - result: OpsErr::INTERNAL, - } - } - - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; - - let node = ctx - .db - .write_tx(move |tx| { - if self.node_type != NodeType::Client { - bail!( - "This BeeMsg handler can only delete client nodes. \ -For server nodes, the grpc handler must be used." - ); - } - - let node = LegacyId { - node_type: self.node_type, - num_id: self.node_id, - } - .resolve(tx, EntityType::Node)?; - - db::node::delete(tx, node.uid)?; - - Ok(node) - }) - .await?; - - log::info!("Node deleted: {node}"); - - notify_nodes( - ctx, - match self.node_type { - shared::types::NodeType::Meta => &[NodeType::Meta, NodeType::Client], - shared::types::NodeType::Storage => { - &[NodeType::Meta, NodeType::Storage, NodeType::Client] - } - _ => &[], - }, - &RemoveNode { - ack_id: "".into(), - ..self - }, - ) - .await; - - Ok(RemoveNodeResp { - result: OpsErr::SUCCESS, - }) - } -} - -impl HandleNoResponse for RemoveNodeResp { - async fn handle(self, _ctx: &Context, req: &mut impl Request) -> Result<()> { - // response from server nodes to the RemoveNode notification - log::debug!("Ignoring RemoveNodeResp msg from {:?}", req.addr()); - Ok(()) - } -} diff --git a/mgmtd/src/bee_msg/peer_info.rs b/mgmtd/src/bee_msg/peer_info.rs new file mode 100644 index 0000000..f54280b --- /dev/null +++ b/mgmtd/src/bee_msg/peer_info.rs @@ -0,0 +1,10 @@ +use super::*; +use shared::bee_msg::misc::*; + +impl HandleNoResponse for PeerInfo { + async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result<()> { + // This is supposed to give some information about a connection, but it looks + // like this isnt used at all + Ok(()) + } +} diff --git a/mgmtd/src/bee_msg/refresh_capacity_pools.rs b/mgmtd/src/bee_msg/refresh_capacity_pools.rs new file mode 100644 index 0000000..8d56235 --- /dev/null +++ b/mgmtd/src/bee_msg/refresh_capacity_pools.rs @@ -0,0 +1,17 @@ +use super::*; +use shared::bee_msg::misc::*; + +impl HandleWithResponse for RefreshCapacityPools { + type Response = Ack; + + async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result { + // This message is superfluous and therefore ignored. It is meant to tell the + // mgmtd to trigger a capacity pool pull immediately after a node starts. + // meta and storage send a SetTargetInfo before this msg though, + // so we handle triggering pulls there. + + Ok(Ack { + ack_id: self.ack_id, + }) + } +} diff --git a/mgmtd/src/bee_msg/register_node.rs b/mgmtd/src/bee_msg/register_node.rs new file mode 100644 index 0000000..6e77dca --- /dev/null +++ b/mgmtd/src/bee_msg/register_node.rs @@ -0,0 +1,25 @@ +use super::*; +use common::update_node; +use shared::bee_msg::node::*; + +impl HandleWithResponse for RegisterNode { + type Response = RegisterNodeResp; + + async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(ctx)?; + + let node_id = update_node(self, ctx).await?; + + let fs_uuid: String = ctx + .db + .read_tx(|tx| db::config::get(tx, db::config::Config::FsUuid)) + .await? + .ok_or_else(|| anyhow!("Could not read file system UUID from database"))?; + + Ok(RegisterNodeResp { + node_num_id: node_id, + grpc_port: ctx.info.user_config.grpc_port, + fs_uuid: fs_uuid.into_bytes(), + }) + } +} diff --git a/mgmtd/src/bee_msg/register_target.rs b/mgmtd/src/bee_msg/register_target.rs new file mode 100644 index 0000000..c96e4c7 --- /dev/null +++ b/mgmtd/src/bee_msg/register_target.rs @@ -0,0 +1,48 @@ +use super::*; +use shared::bee_msg::target::*; + +impl HandleWithResponse for RegisterTarget { + type Response = RegisterTargetResp; + + async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(ctx)?; + + let ctx2 = ctx.clone(); + + let (id, is_new) = ctx + .db + .write_tx(move |tx| { + // Do not do anything if the target already exists + if let Some(id) = try_resolve_num_id( + tx, + EntityType::Target, + NodeType::Storage, + self.target_id.into(), + )? { + return Ok((id.num_id().try_into()?, false)); + } + + if ctx2.info.user_config.registration_disable { + bail!("Registration of new targets is not allowed"); + } + + Ok(( + db::target::insert_storage( + tx, + self.target_id, + Some(format!("target_{}", std::str::from_utf8(&self.alias)?).try_into()?), + )?, + true, + )) + }) + .await?; + + if is_new { + log::info!("Registered new storage target with Id {id}"); + } else { + log::debug!("Re-registered existing storage target with Id {id}"); + } + + Ok(RegisterTargetResp { id }) + } +} diff --git a/mgmtd/src/bee_msg/remove_node.rs b/mgmtd/src/bee_msg/remove_node.rs new file mode 100644 index 0000000..861cd80 --- /dev/null +++ b/mgmtd/src/bee_msg/remove_node.rs @@ -0,0 +1,60 @@ +use super::*; +use shared::bee_msg::node::*; + +impl HandleWithResponse for RemoveNode { + type Response = RemoveNodeResp; + + fn error_response() -> Self::Response { + RemoveNodeResp { + result: OpsErr::INTERNAL, + } + } + + async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(ctx)?; + + let node = ctx + .db + .write_tx(move |tx| { + if self.node_type != NodeType::Client { + bail!( + "This BeeMsg handler can only delete client nodes. \ +For server nodes, the grpc handler must be used." + ); + } + + let node = LegacyId { + node_type: self.node_type, + num_id: self.node_id, + } + .resolve(tx, EntityType::Node)?; + + db::node::delete(tx, node.uid)?; + + Ok(node) + }) + .await?; + + log::info!("Node deleted: {node}"); + + notify_nodes( + ctx, + match self.node_type { + shared::types::NodeType::Meta => &[NodeType::Meta, NodeType::Client], + shared::types::NodeType::Storage => { + &[NodeType::Meta, NodeType::Storage, NodeType::Client] + } + _ => &[], + }, + &RemoveNode { + ack_id: "".into(), + ..self + }, + ) + .await; + + Ok(RemoveNodeResp { + result: OpsErr::SUCCESS, + }) + } +} diff --git a/mgmtd/src/bee_msg/remove_node_resp.rs b/mgmtd/src/bee_msg/remove_node_resp.rs new file mode 100644 index 0000000..72761bf --- /dev/null +++ b/mgmtd/src/bee_msg/remove_node_resp.rs @@ -0,0 +1,10 @@ +use super::*; +use shared::bee_msg::node::*; + +impl HandleNoResponse for RemoveNodeResp { + async fn handle(self, _ctx: &Context, req: &mut impl Request) -> Result<()> { + // response from server nodes to the RemoveNode notification + log::debug!("Ignoring RemoveNodeResp msg from {:?}", req.addr()); + Ok(()) + } +} diff --git a/mgmtd/src/bee_msg/quota.rs b/mgmtd/src/bee_msg/request_exceeded_quota.rs similarity index 100% rename from mgmtd/src/bee_msg/quota.rs rename to mgmtd/src/bee_msg/request_exceeded_quota.rs diff --git a/mgmtd/src/bee_msg/set_channel_direct.rs b/mgmtd/src/bee_msg/set_channel_direct.rs new file mode 100644 index 0000000..12f1e4e --- /dev/null +++ b/mgmtd/src/bee_msg/set_channel_direct.rs @@ -0,0 +1,9 @@ +use super::*; +use shared::bee_msg::misc::*; + +impl HandleNoResponse for SetChannelDirect { + async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result<()> { + // do nothing + Ok(()) + } +} diff --git a/mgmtd/src/bee_msg/set_mirror_buddy_groups_resp.rs b/mgmtd/src/bee_msg/set_mirror_buddy_groups_resp.rs new file mode 100644 index 0000000..8630e46 --- /dev/null +++ b/mgmtd/src/bee_msg/set_mirror_buddy_groups_resp.rs @@ -0,0 +1,9 @@ +use super::*; +use shared::bee_msg::buddy_group::*; + +impl HandleNoResponse for SetMirrorBuddyGroupResp { + async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result<()> { + // response from server nodes to SetMirrorBuddyGroup notification + Ok(()) + } +} diff --git a/mgmtd/src/bee_msg/set_storage_target_info.rs b/mgmtd/src/bee_msg/set_storage_target_info.rs new file mode 100644 index 0000000..4d71205 --- /dev/null +++ b/mgmtd/src/bee_msg/set_storage_target_info.rs @@ -0,0 +1,48 @@ +use super::*; +use db::target::TargetCapacities; +use shared::bee_msg::target::*; + +impl HandleWithResponse for SetStorageTargetInfo { + type Response = SetStorageTargetInfoResp; + + fn error_response() -> Self::Response { + SetStorageTargetInfoResp { + result: OpsErr::INTERNAL, + } + } + + async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(ctx)?; + + let node_type = self.node_type; + ctx.db + .write_tx(move |tx| { + db::target::get_and_update_capacities( + tx, + self.info.into_iter().map(|e| { + Ok(( + e.target_id, + TargetCapacities { + total_space: Some(e.total_space.try_into()?), + total_inodes: Some(e.total_inodes.try_into()?), + free_space: Some(e.free_space.try_into()?), + free_inodes: Some(e.free_inodes.try_into()?), + }, + )) + }), + self.node_type.try_into()?, + ) + }) + .await?; + + log::debug!("Updated {node_type:?} target info"); + + // in the old mgmtd, a notice to refresh cap pools is sent out here if a cap pool + // changed I consider this being to expensive to check here and just don't + // do it. Nodes refresh their cap pool every two minutes (by default) anyway + + Ok(SetStorageTargetInfoResp { + result: OpsErr::SUCCESS, + }) + } +} diff --git a/mgmtd/src/bee_msg/set_target_consistency_states.rs b/mgmtd/src/bee_msg/set_target_consistency_states.rs new file mode 100644 index 0000000..fac2db0 --- /dev/null +++ b/mgmtd/src/bee_msg/set_target_consistency_states.rs @@ -0,0 +1,50 @@ +use super::*; +use common::update_last_contact_times; +use shared::bee_msg::target::*; + +impl HandleWithResponse for SetTargetConsistencyStates { + type Response = SetTargetConsistencyStatesResp; + + fn error_response() -> Self::Response { + SetTargetConsistencyStatesResp { + result: OpsErr::INTERNAL, + } + } + + async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(ctx)?; + + let node_type = self.node_type.try_into()?; + let msg = self.clone(); + + ctx.db + .write_tx(move |tx| { + // Check given target Ids exist + db::target::validate_ids(tx, &msg.target_ids, node_type)?; + + if msg.set_online > 0 { + update_last_contact_times(tx, &msg.target_ids, node_type)?; + } + + db::target::update_consistency_states( + tx, + msg.target_ids.into_iter().zip(msg.states.iter().copied()), + node_type, + ) + }) + .await?; + + log::info!("Set consistency state for targets {:?}", self.target_ids,); + + notify_nodes( + ctx, + &[NodeType::Meta, NodeType::Storage, NodeType::Client], + &RefreshTargetStates { ack_id: "".into() }, + ) + .await; + + Ok(SetTargetConsistencyStatesResp { + result: OpsErr::SUCCESS, + }) + } +} diff --git a/mgmtd/src/bee_msg/target.rs b/mgmtd/src/bee_msg/target.rs deleted file mode 100644 index a616045..0000000 --- a/mgmtd/src/bee_msg/target.rs +++ /dev/null @@ -1,412 +0,0 @@ -use super::*; -use crate::db::target::TargetCapacities; -use crate::types::ResolveEntityId; -use rusqlite::Transaction; -use shared::bee_msg::target::*; -use std::time::Duration; - -impl HandleWithResponse for GetTargetMappings { - type Response = GetTargetMappingsResp; - - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - let mapping: HashMap = ctx - .db - .read_tx(move |tx| { - tx.query_map_collect( - sql!( - "SELECT target_id, node_id - FROM storage_targets - WHERE node_id IS NOT NULL" - ), - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .map_err(Into::into) - }) - .await?; - - let resp = GetTargetMappingsResp { mapping }; - - Ok(resp) - } -} - -impl HandleWithResponse for GetTargetStates { - type Response = GetTargetStatesResp; - - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - let pre_shutdown = ctx.run_state.pre_shutdown(); - let node_offline_timeout = ctx.info.user_config.node_offline_timeout; - - let targets = ctx - .db - .read_tx(move |tx| { - get_targets_with_states( - tx, - pre_shutdown, - self.node_type.try_into()?, - node_offline_timeout, - ) - }) - .await?; - - let mut target_ids = Vec::with_capacity(targets.len()); - let mut reachability_states = Vec::with_capacity(targets.len()); - let mut consistency_states = Vec::with_capacity(targets.len()); - - for e in targets { - target_ids.push(e.0); - consistency_states.push(e.1); - reachability_states.push(e.2); - } - - let resp = GetTargetStatesResp { - targets: target_ids, - consistency_states, - reachability_states, - }; - - Ok(resp) - } -} - -pub(crate) fn get_targets_with_states( - tx: &Transaction, - pre_shutdown: bool, - node_type: NodeTypeServer, - node_offline_timeout: Duration, -) -> Result> { - let targets = tx.query_map_collect( - sql!( - "SELECT t.target_id, t.consistency, - (UNIXEPOCH('now') - UNIXEPOCH(t.last_update)), gp.p_target_id, gs.s_target_id - FROM targets AS t - INNER JOIN nodes AS n USING(node_type, node_id) - LEFT JOIN buddy_groups AS gp ON gp.p_target_id = t.target_id AND gp.node_type = t.node_type - LEFT JOIN buddy_groups AS gs ON gs.s_target_id = t.target_id AND gs.node_type = t.node_type - WHERE t.node_type = ?1" - ), - [node_type.sql_variant()], - |row| { - let is_primary = row.get::<_, Option>(3)?.is_some(); - let is_secondary = row.get::<_, Option>(4)?.is_some(); - - Ok(( - row.get(0)?, - TargetConsistencyState::from_row(row, 1)?, - if !pre_shutdown || is_secondary { - let age = Duration::from_secs(row.get(2)?); - - // We never want to report a primary node of a buddy group as offline since this - // is considered invalid. Instead we just report ProbablyOffline and wait for the switchover. - if !is_primary && age > node_offline_timeout { - TargetReachabilityState::Offline - } else if age > node_offline_timeout / 2 { - TargetReachabilityState::ProbablyOffline - } else { - TargetReachabilityState::Online - } - } else { - TargetReachabilityState::ProbablyOffline - }, - )) - }, - )?; - - Ok(targets) -} - -impl HandleWithResponse for RegisterTarget { - type Response = RegisterTargetResp; - - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; - - let ctx2 = ctx.clone(); - - let (id, is_new) = ctx - .db - .write_tx(move |tx| { - // Do not do anything if the target already exists - if let Some(id) = try_resolve_num_id( - tx, - EntityType::Target, - NodeType::Storage, - self.target_id.into(), - )? { - return Ok((id.num_id().try_into()?, false)); - } - - if ctx2.info.user_config.registration_disable { - bail!("Registration of new targets is not allowed"); - } - - Ok(( - db::target::insert_storage( - tx, - self.target_id, - Some(format!("target_{}", std::str::from_utf8(&self.alias)?).try_into()?), - )?, - true, - )) - }) - .await?; - - if is_new { - log::info!("Registered new storage target with Id {id}"); - } else { - log::debug!("Re-registered existing storage target with Id {id}"); - } - - Ok(RegisterTargetResp { id }) - } -} - -impl HandleWithResponse for MapTargets { - type Response = MapTargetsResp; - - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; - - let target_ids = self.target_ids.keys().copied().collect::>(); - - ctx.db - .write_tx(move |tx| { - // Check node Id exists - let node = LegacyId { - node_type: NodeType::Storage, - num_id: self.node_id, - } - .resolve(tx, EntityType::Node)?; - // Check all target Ids exist - db::target::validate_ids(tx, &target_ids, NodeTypeServer::Storage)?; - // Due to the check above, this must always match all the given ids - db::target::update_storage_node_mappings(tx, &target_ids, node.num_id())?; - Ok(()) - }) - .await?; - - // At this point, all mappings must have been successful - - log::info!( - "Mapped storage targets with Ids {:?} to node {}", - self.target_ids.keys(), - self.node_id - ); - - notify_nodes( - ctx, - &[NodeType::Meta, NodeType::Storage, NodeType::Client], - &MapTargets { - target_ids: self.target_ids.clone(), - node_id: self.node_id, - ack_id: "".into(), - }, - ) - .await; - - // Storage server expects a separate status code for each target map requested. We, however, - // do a all-or-nothing approach. If e.g. one target id doesn't exist (which is an - // exceptional error and should usually not happen anyway), we fail the whole - // operation. Therefore we can just send a list of successes. - let resp = MapTargetsResp { - results: self - .target_ids - .into_iter() - .map(|e| (e.0, OpsErr::SUCCESS)) - .collect(), - }; - - Ok(resp) - } -} - -impl HandleNoResponse for MapTargetsResp { - async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result<()> { - // This is sent from the nodes as a result of the MapTargets notification after - // map_targets was called. We just ignore it. - Ok(()) - } -} - -impl HandleWithResponse for SetStorageTargetInfo { - type Response = SetStorageTargetInfoResp; - - fn error_response() -> Self::Response { - SetStorageTargetInfoResp { - result: OpsErr::INTERNAL, - } - } - - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; - - let node_type = self.node_type; - ctx.db - .write_tx(move |tx| { - db::target::get_and_update_capacities( - tx, - self.info.into_iter().map(|e| { - Ok(( - e.target_id, - TargetCapacities { - total_space: Some(e.total_space.try_into()?), - total_inodes: Some(e.total_inodes.try_into()?), - free_space: Some(e.free_space.try_into()?), - free_inodes: Some(e.free_inodes.try_into()?), - }, - )) - }), - self.node_type.try_into()?, - ) - }) - .await?; - - log::debug!("Updated {node_type:?} target info"); - - // in the old mgmtd, a notice to refresh cap pools is sent out here if a cap pool - // changed I consider this being to expensive to check here and just don't - // do it. Nodes refresh their cap pool every two minutes (by default) anyway - - Ok(SetStorageTargetInfoResp { - result: OpsErr::SUCCESS, - }) - } -} - -impl HandleWithResponse for ChangeTargetConsistencyStates { - type Response = ChangeTargetConsistencyStatesResp; - - fn error_response() -> Self::Response { - ChangeTargetConsistencyStatesResp { - result: OpsErr::INTERNAL, - } - } - - async fn handle(self, ctx: &Context, __req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; - - // self.old_states is currently completely ignored. If something reports a non-GOOD state, I - // see no apparent reason to that the old state matches before setting. We have the - // authority, whatever nodes think their old state was doesn't matter. - - let changed = ctx - .db - .write_tx(move |tx| { - let node_type = self.node_type.try_into()?; - - // Check given target Ids exist - db::target::validate_ids(tx, &self.target_ids, node_type)?; - - // Old management updates contact time while handling this message (comes usually in - // every 30 seconds), so we do it as well - update_times(tx, &self.target_ids, node_type)?; - - let affected = db::target::update_consistency_states( - tx, - self.target_ids - .into_iter() - .zip(self.new_states.iter().copied()), - node_type, - )?; - - Ok(affected > 0) - }) - .await?; - - log::debug!( - "Updated target consistency states for {:?} nodes", - self.node_type - ); - - if changed { - notify_nodes( - ctx, - &[NodeType::Meta, NodeType::Storage, NodeType::Client], - &RefreshTargetStates { ack_id: "".into() }, - ) - .await; - } - - Ok(ChangeTargetConsistencyStatesResp { - result: OpsErr::SUCCESS, - }) - } -} - -impl HandleWithResponse for SetTargetConsistencyStates { - type Response = SetTargetConsistencyStatesResp; - - fn error_response() -> Self::Response { - SetTargetConsistencyStatesResp { - result: OpsErr::INTERNAL, - } - } - - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; - - let node_type = self.node_type.try_into()?; - let msg = self.clone(); - - ctx.db - .write_tx(move |tx| { - // Check given target Ids exist - db::target::validate_ids(tx, &msg.target_ids, node_type)?; - - if msg.set_online > 0 { - update_times(tx, &msg.target_ids, node_type)?; - } - - db::target::update_consistency_states( - tx, - msg.target_ids.into_iter().zip(msg.states.iter().copied()), - node_type, - ) - }) - .await?; - - log::info!("Set consistency state for targets {:?}", self.target_ids,); - - notify_nodes( - ctx, - &[NodeType::Meta, NodeType::Storage, NodeType::Client], - &RefreshTargetStates { ack_id: "".into() }, - ) - .await; - - Ok(SetTargetConsistencyStatesResp { - result: OpsErr::SUCCESS, - }) - } -} - -/// Updates the `last_contact` time for all the nodes belonging to the passed targets and the -/// targets `last_update` times themselves. -fn update_times( - tx: &Transaction, - target_ids: &[TargetId], - node_type: NodeTypeServer, -) -> Result<()> { - let target_ids_param = sqlite::rarray_param(target_ids.iter().copied()); - - tx.execute_cached( - sql!( - "UPDATE nodes AS n SET last_contact = DATETIME('now') - WHERE n.node_uid IN ( - SELECT DISTINCT node_uid FROM targets_ext - WHERE target_id IN rarray(?1) AND node_type = ?2)" - ), - rusqlite::params![&target_ids_param, node_type.sql_variant()], - )?; - - tx.execute_cached( - sql!( - "UPDATE targets SET last_update = DATETIME('now') - WHERE target_id IN rarray(?1) AND node_type = ?2" - ), - rusqlite::params![&target_ids_param, node_type.sql_variant()], - )?; - - Ok(()) -} From f82b1cf467c890f0d1c9423dae9e2ba7ddaaecc1 Mon Sep 17 00:00:00 2001 From: Rusty Bee <145002912+rustybee42@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:56:02 +0100 Subject: [PATCH 5/9] test: add in memory database setup function for handler testing --- mgmtd/src/db.rs | 20 ++++++++++++++++++++ sqlite/src/connection.rs | 18 ++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/mgmtd/src/db.rs b/mgmtd/src/db.rs index e6fb0ab..3b5da75 100644 --- a/mgmtd/src/db.rs +++ b/mgmtd/src/db.rs @@ -54,6 +54,26 @@ pub(crate) mod test { use super::*; use rusqlite::{Connection, Transaction}; + pub(crate) async fn setup_with_test_data() -> Connections { + let conns = Connections::new_in_memory(); + + conns + .conn(|conn| { + let tx = conn.transaction().unwrap(); + sqlite::migrate_schema(&tx, MIGRATIONS).unwrap(); + // Setup test data + tx.execute_batch(include_str!("db/schema/test_data.sql")) + .unwrap(); + tx.commit().unwrap(); + + Ok(()) + }) + .await + .unwrap(); + + conns + } + /// Sets ups a fresh database instance in memory and fills, with the test data set and provides /// a transaction handle. pub(crate) fn with_test_data(op: impl FnOnce(&Transaction)) { diff --git a/sqlite/src/connection.rs b/sqlite/src/connection.rs index 1679c28..4451e35 100644 --- a/sqlite/src/connection.rs +++ b/sqlite/src/connection.rs @@ -3,6 +3,7 @@ use rusqlite::config::DbConfig; use rusqlite::{Connection, Transaction, TransactionBehavior}; use std::ops::Deref; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -79,7 +80,12 @@ pub struct InnerConnections { db_file: PathBuf, } +/// Increased whenever new_in_memory is called. Makes sure that the test binary can run multiple +/// tests in parallel with distinct in memory db instances (otherwise they would clash) +static MEMORY_COUNTER: AtomicU64 = AtomicU64::new(0); + impl Connections { + /// Create a new db connection pool using the given db file pub fn new(db_file: impl AsRef) -> Self { Self { inner: Arc::new(InnerConnections { @@ -89,6 +95,18 @@ impl Connections { } } + /// Create a new db connection pool using an in memory db + pub fn new_in_memory() -> Self { + let count = MEMORY_COUNTER.fetch_add(1, Ordering::Relaxed); + + Self { + inner: Arc::new(InnerConnections { + conns: Mutex::new(vec![]), + db_file: format!("file:memdb{count}?mode=memory&cache=shared").into(), + }), + } + } + /// Start a new write (immediate) transaction. If doing writes, it is important to use this /// instead of `.read()` because here the busy timeout / busy handler actually works as it is /// applied before the transaction starts. From 00ae0e1b5bd7b112f5c520b51f56101c8ee627e6 Mon Sep 17 00:00:00 2001 From: Rusty Bee <145002912+rustybee42@users.noreply.github.com> Date: Fri, 15 Nov 2024 12:47:23 +0100 Subject: [PATCH 6/9] refactor: implement `from for pb::EntityIdSet` for convenience --- shared/src/types/entity.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/shared/src/types/entity.rs b/shared/src/types/entity.rs index e346fe9..2f7600f 100644 --- a/shared/src/types/entity.rs +++ b/shared/src/types/entity.rs @@ -159,6 +159,26 @@ impl TryFrom for EntityId { } } +#[cfg(feature = "grpc")] +impl From for pb::EntityIdSet { + fn from(value: EntityId) -> Self { + match value { + EntityId::Alias(a) => pb::EntityIdSet { + alias: Some(a.into()), + ..Default::default() + }, + EntityId::LegacyID(id) => pb::EntityIdSet { + legacy_id: Some(id.into()), + ..Default::default() + }, + EntityId::Uid(uid) => pb::EntityIdSet { + uid: Some(uid), + ..Default::default() + }, + } + } +} + #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct EntityIdSet { pub uid: Uid, From 47cc5042e52547ebd7fa25b6594700ed3c7e4997 Mon Sep 17 00:00:00 2001 From: Rusty Bee <145002912+rustybee42@users.noreply.github.com> Date: Fri, 15 Nov 2024 12:18:03 +0100 Subject: [PATCH 7/9] refactor: dependency injection on grpc and BeeMsg handlers * Accept a trait impl (`App*` / `AppAll`) to instead of conrete `Context` type on handlers * Implement a runtime `App` that satifies the above and is passed to the handlers at runtime * Add a `TestApp` implementation with rudimentary functionality for testing the handlers --- mgmtd/src/app.rs | 97 +++++++++ mgmtd/src/app/runtime.rs | 188 ++++++++++++++++++ mgmtd/src/app/test.rs | 162 +++++++++++++++ mgmtd/src/bee_msg.rs | 43 +--- mgmtd/src/bee_msg/ack.rs | 2 +- mgmtd/src/bee_msg/authenticate_channel.rs | 4 +- .../change_target_consistency_states.rs | 12 +- mgmtd/src/bee_msg/common.rs | 18 +- mgmtd/src/bee_msg/get_mirror_buddy_groups.rs | 7 +- mgmtd/src/bee_msg/get_node_capacity_pools.rs | 50 +++-- mgmtd/src/bee_msg/get_nodes.rs | 7 +- .../bee_msg/get_states_and_buddy_groups.rs | 13 +- mgmtd/src/bee_msg/get_storage_pools.rs | 21 +- mgmtd/src/bee_msg/get_target_mappings.rs | 7 +- mgmtd/src/bee_msg/get_target_states.rs | 11 +- mgmtd/src/bee_msg/heartbeat.rs | 6 +- mgmtd/src/bee_msg/heartbeat_request.rs | 11 +- mgmtd/src/bee_msg/map_targets.rs | 36 ++-- mgmtd/src/bee_msg/map_targets_resp.rs | 2 +- mgmtd/src/bee_msg/peer_info.rs | 2 +- mgmtd/src/bee_msg/refresh_capacity_pools.rs | 2 +- mgmtd/src/bee_msg/register_node.rs | 13 +- mgmtd/src/bee_msg/register_target.rs | 13 +- mgmtd/src/bee_msg/remove_node.rs | 12 +- mgmtd/src/bee_msg/remove_node_resp.rs | 2 +- mgmtd/src/bee_msg/request_exceeded_quota.rs | 7 +- mgmtd/src/bee_msg/set_channel_direct.rs | 2 +- .../bee_msg/set_mirror_buddy_groups_resp.rs | 2 +- mgmtd/src/bee_msg/set_storage_target_info.rs | 41 ++-- .../bee_msg/set_target_consistency_states.rs | 34 ++-- mgmtd/src/context.rs | 82 -------- mgmtd/src/grpc.rs | 46 ++--- mgmtd/src/grpc/assign_pool.rs | 14 +- mgmtd/src/grpc/create_buddy_group.rs | 14 +- mgmtd/src/grpc/create_pool.rs | 14 +- mgmtd/src/grpc/delete_buddy_group.rs | 34 ++-- mgmtd/src/grpc/delete_node.rs | 12 +- mgmtd/src/grpc/delete_pool.rs | 14 +- mgmtd/src/grpc/delete_target.rs | 12 +- mgmtd/src/grpc/get_buddy_groups.rs | 7 +- mgmtd/src/grpc/get_license.rs | 7 +- mgmtd/src/grpc/get_nodes.rs | 7 +- mgmtd/src/grpc/get_pools.rs | 7 +- mgmtd/src/grpc/get_quota_limits.rs | 18 +- mgmtd/src/grpc/get_quota_usage.rs | 22 +- mgmtd/src/grpc/get_targets.rs | 171 ++++++++-------- mgmtd/src/grpc/mirror_root_inode.rs | 20 +- mgmtd/src/grpc/set_alias.rs | 15 +- mgmtd/src/grpc/set_default_quota_limits.rs | 45 ++--- mgmtd/src/grpc/set_quota_limits.rs | 111 +++++------ mgmtd/src/grpc/set_target_state.rs | 17 +- mgmtd/src/grpc/start_resync.rs | 59 +++--- mgmtd/src/lib.rs | 42 ++-- mgmtd/src/quota.rs | 79 ++++---- mgmtd/src/timer.rs | 38 ++-- shared/src/grpc.rs | 2 +- 56 files changed, 1017 insertions(+), 719 deletions(-) create mode 100644 mgmtd/src/app.rs create mode 100644 mgmtd/src/app/runtime.rs create mode 100644 mgmtd/src/app/test.rs delete mode 100644 mgmtd/src/context.rs diff --git a/mgmtd/src/app.rs b/mgmtd/src/app.rs new file mode 100644 index 0000000..4da6031 --- /dev/null +++ b/mgmtd/src/app.rs @@ -0,0 +1,97 @@ +//! Interfaces and implementations for in-app interaction between tasks or threads. + +mod runtime; +#[cfg(test)] +pub(crate) mod test; + +use crate::StaticInfo; +use crate::license::LicensedFeature; +use anyhow::Result; +use protobuf::license::GetCertDataResult; +pub(crate) use runtime::RuntimeApp; +use rusqlite::{Connection, Transaction}; +use shared::bee_msg::Msg; +use shared::bee_serde::{Deserializable, Serializable}; +use shared::types::{NodeId, NodeType, Uid}; +use std::fmt::Debug; +use std::future::Future; +use std::net::SocketAddr; +use std::path::Path; +use std::sync::Arc; + +pub(crate) trait App: Debug + Clone + Send + 'static { + /// Return a borrow to the applications static, immutable config and derived info + fn static_info(&self) -> &StaticInfo; + + // Database access + + /// DB Read transaction + fn db_read_tx Result, R: Send + 'static>( + &self, + op: T, + ) -> impl Future> + Send; + + /// DB write transaction + fn db_write_tx Result, R: Send + 'static>( + &self, + op: T, + ) -> impl Future> + Send; + + /// DB write transaction without fsync + fn db_write_tx_no_sync< + T: Send + 'static + FnOnce(&Transaction) -> Result, + R: Send + 'static, + >( + &self, + op: T, + ) -> impl Future> + Send; + + /// Provides access to a DB connection handle, no transaction + fn db_conn Result, R: Send + 'static>( + &self, + op: T, + ) -> impl Future> + Send; + + // BeeMsg communication + + /// Send a [Msg] to a node via TCP and receive the response + fn beemsg_request( + &self, + node_uid: Uid, + msg: &M, + ) -> impl Future> + Send; + + /// Send a [Msg] to all nodes of a type via UDP + fn beemsg_send_notifications( + &self, + node_types: &'static [NodeType], + msg: &M, + ) -> impl Future + Send; + + /// Replace all stored BeeMsg network addresses of a node in the store + fn beemsg_replace_node_addrs(&self, node_uid: Uid, new_addrs: impl Into>); + + // Run state + + /// Check if management is in pre shutdown state + fn rs_pre_shutdown(&self) -> bool; + /// Notify the runtime control that a particular client pulled states of a particular node type + fn rs_notify_client_pulled_state(&self, node_type: NodeType, node_id: NodeId); + + // Licensing control + + /// Load and verify a license certificate + fn lic_load_and_verify_cert( + &self, + cert_path: &Path, + ) -> impl Future> + Send; + + /// Get license certificate data + fn lic_get_cert_data(&self) -> Result; + + /// Get licensed number of machines + fn lic_get_num_machines(&self) -> Result; + + /// Verify a feature is licensed + fn lic_verify_feature(&self, feature: LicensedFeature) -> Result<()>; +} diff --git a/mgmtd/src/app/runtime.rs b/mgmtd/src/app/runtime.rs new file mode 100644 index 0000000..e0d7d45 --- /dev/null +++ b/mgmtd/src/app/runtime.rs @@ -0,0 +1,188 @@ +use super::*; +use crate::ClientPulledStateNotification; +use crate::bee_msg::dispatch_request; +use crate::license::LicenseVerifier; +use anyhow::Result; +use protobuf::license::GetCertDataResult; +use rusqlite::{Connection, Transaction}; +use shared::conn::msg_dispatch::{DispatchRequest, Request}; +use shared::conn::outgoing::Pool; +use shared::run_state::WeakRunStateHandle; +use sqlite::Connections; +use std::fmt::Debug; +use std::ops::Deref; +use tokio::sync::mpsc; + +/// A collection of Handles used for interacting and accessing the different components of the app. +/// +/// This is the actual runtime object that can be shared between tasks. Interfaces should, however, +/// accept any implementation of the AppContext trait instead. +#[derive(Clone, Debug)] +pub(crate) struct RuntimeApp(Arc); + +/// Stores the actual handles. +#[derive(Debug)] +pub(crate) struct InnerAppHandles { + pub conn: Pool, + pub db: Connections, + pub license: LicenseVerifier, + pub info: &'static StaticInfo, + pub run_state: WeakRunStateHandle, + shutdown_client_id: mpsc::Sender, +} + +impl RuntimeApp { + /// Creates a new AppHandles object. + /// + /// Takes all the stored handles. + pub(crate) fn new( + conn: Pool, + db: Connections, + license: LicenseVerifier, + info: &'static StaticInfo, + run_state: WeakRunStateHandle, + shutdown_client_id: mpsc::Sender, + ) -> Self { + Self(Arc::new(InnerAppHandles { + conn, + db, + license, + info, + run_state, + shutdown_client_id, + })) + } +} + +/// Derefs to InnerAppHandle which stores all the handles. +/// +/// Allows transparent access. +impl Deref for RuntimeApp { + type Target = InnerAppHandles; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// Adds BeeMsg dispatching functionality to AppHandles +impl DispatchRequest for RuntimeApp { + async fn dispatch_request(&self, req: impl Request) -> Result<()> { + dispatch_request(self, req).await + } +} + +impl App for RuntimeApp { + fn static_info(&self) -> &StaticInfo { + self.info + } + + async fn db_read_tx< + T: Send + 'static + FnOnce(&Transaction) -> Result, + R: Send + 'static, + >( + &self, + op: T, + ) -> Result { + Connections::read_tx(&self.db, op).await + } + + async fn db_write_tx< + T: Send + 'static + FnOnce(&Transaction) -> Result, + R: Send + 'static, + >( + &self, + op: T, + ) -> Result { + Connections::write_tx(&self.db, op).await + } + + async fn db_write_tx_no_sync< + T: Send + 'static + FnOnce(&Transaction) -> Result, + R: Send + 'static, + >( + &self, + op: T, + ) -> Result { + Connections::write_tx_no_sync(&self.db, op).await + } + + async fn db_conn< + T: Send + 'static + FnOnce(&mut Connection) -> Result, + R: Send + 'static, + >( + &self, + op: T, + ) -> Result { + Connections::conn(&self.db, op).await + } + + async fn beemsg_request( + &self, + node_uid: Uid, + msg: &M, + ) -> Result { + Pool::request(&self.conn, node_uid, msg).await + } + + async fn beemsg_send_notifications( + &self, + node_types: &'static [NodeType], + msg: &M, + ) { + log::trace!("NOTIFICATION to {node_types:?}: {msg:?}"); + + for t in node_types { + if let Err(err) = async { + let nodes = self + .db_read_tx(move |tx| crate::db::node::get_with_type(tx, *t)) + .await?; + + self.conn + .broadcast_datagram(nodes.into_iter().map(|e| e.uid), msg) + .await?; + + Ok(()) as Result<_> + } + .await + { + log::error!("Notification could not be sent to all {t} nodes: {err:#}"); + } + } + } + + fn beemsg_replace_node_addrs(&self, node_uid: Uid, new_addrs: impl Into>) { + Pool::replace_node_addrs(&self.conn, node_uid, new_addrs) + } + + fn rs_pre_shutdown(&self) -> bool { + WeakRunStateHandle::pre_shutdown(&self.run_state) + } + + fn rs_notify_client_pulled_state(&self, node_type: NodeType, node_id: NodeId) { + if self.run_state.pre_shutdown() { + let tx = self.shutdown_client_id.clone(); + + // We don't want to block the task calling this and are not interested by the results + tokio::spawn(async move { + let _ = tx.send((node_type, node_id)).await; + }); + } + } + + async fn lic_load_and_verify_cert(&self, cert_path: &Path) -> Result { + LicenseVerifier::load_and_verify_cert(&self.license, cert_path).await + } + + fn lic_get_cert_data(&self) -> Result { + LicenseVerifier::get_cert_data(&self.license) + } + + fn lic_get_num_machines(&self) -> Result { + LicenseVerifier::get_num_machines(&self.license) + } + + fn lic_verify_feature(&self, feature: LicensedFeature) -> Result<()> { + self.license.verify_feature(feature) + } +} diff --git a/mgmtd/src/app/test.rs b/mgmtd/src/app/test.rs new file mode 100644 index 0000000..1a63282 --- /dev/null +++ b/mgmtd/src/app/test.rs @@ -0,0 +1,162 @@ +use super::*; +use crate::config::Config; +use shared::bee_msg::MsgId; +use shared::nic::Nic; +use shared::types::{AuthSecret, NicType}; +use sqlite::Connections; +use std::net::Ipv4Addr; +use std::sync::Mutex; + +#[derive(Debug, Clone)] +pub struct TestApp { + pub db: Connections, + pub info: Arc, + #[allow(clippy::type_complexity)] + pub notifications: Arc)>>>, +} + +impl TestApp { + pub async fn new() -> Self { + let db = crate::db::test::setup_with_test_data().await; + Self { + db, + info: Arc::new(StaticInfo { + user_config: Config::default(), + auth_secret: Some(AuthSecret::hash_from_bytes("secret")), + network_addrs: vec![Nic { + address: Ipv4Addr::LOCALHOST.into(), + nic_type: NicType::Ethernet, + name: "localhost".to_string(), + priority: 0, + }], + use_ipv6: false, + }), + notifications: Arc::new(Mutex::new(vec![])), + } + } + + pub fn has_sent_notification(&self, receivers: &[NodeType]) -> bool { + self.notifications + .lock() + .unwrap() + .contains(&(M::ID, receivers.to_vec())) + } +} + +impl App for TestApp { + fn static_info(&self) -> &StaticInfo { + &self.info + } + + async fn db_read_tx< + T: Send + 'static + FnOnce(&Transaction) -> Result, + R: Send + 'static, + >( + &self, + op: T, + ) -> Result { + Connections::read_tx(&self.db, op).await + } + + async fn db_write_tx< + T: Send + 'static + FnOnce(&Transaction) -> Result, + R: Send + 'static, + >( + &self, + op: T, + ) -> Result { + Connections::write_tx(&self.db, op).await + } + async fn db_write_tx_no_sync< + T: Send + 'static + FnOnce(&Transaction) -> Result, + R: Send + 'static, + >( + &self, + op: T, + ) -> Result { + Connections::write_tx_no_sync(&self.db, op).await + } + async fn db_conn< + T: Send + 'static + FnOnce(&mut rusqlite::Connection) -> Result, + R: Send + 'static, + >( + &self, + op: T, + ) -> Result { + Connections::conn(&self.db, op).await + } + + async fn beemsg_request( + &self, + _node_uid: Uid, + _msg: &M, + ) -> Result { + // TODO + Ok(R::default()) + } + + async fn beemsg_send_notifications( + &self, + node_types: &'static [NodeType], + _msg: &M, + ) { + self.notifications + .lock() + .unwrap() + .push((M::ID, node_types.to_owned())); + // nop + } + + fn beemsg_replace_node_addrs(&self, _node_uid: Uid, _new_addrs: impl Into>) { + // nop + } + + fn rs_pre_shutdown(&self) -> bool { + false + } + + fn rs_notify_client_pulled_state(&self, _node_type: NodeType, _node_id: NodeId) { + // nop + } + + async fn lic_load_and_verify_cert(&self, _cert_path: &std::path::Path) -> Result { + Ok("dummy cert".to_string()) + } + + fn lic_get_cert_data(&self) -> Result { + Ok(protobuf::license::GetCertDataResult::default()) + } + + fn lic_get_num_machines(&self) -> Result { + Ok(128) + } + + fn lic_verify_feature(&self, _feature: LicensedFeature) -> Result<()> { + Ok(()) + } +} + +/// Queries a single value from the db and asserts on it +macro_rules! assert_eq_db { + ($handle:expr, $sql:literal, [$($params:expr),* $(,)?], $expect:expr $(,$arg:tt)* $(,)?) => {{ + // Little trick to "detect" the type of $expect + #[allow(unused_assignments)] + let mut res = $expect.to_owned(); + + res = $handle + .db_read_tx(|tx| { + tx.query_row( + ::sqlite_check::sql!($sql), + ::rusqlite::params![$($params),*], + |row| row.get(0), + ) + .map_err(Into::into) + }) + .await + .unwrap(); + + assert_eq!(res, $expect $(,$arg)*); + }}; +} + +pub(crate) use assert_eq_db; diff --git a/mgmtd/src/bee_msg.rs b/mgmtd/src/bee_msg.rs index 6aa9218..a2f8903 100644 --- a/mgmtd/src/bee_msg.rs +++ b/mgmtd/src/bee_msg.rs @@ -3,7 +3,7 @@ //! Dispatches the incoming requests (coming from the BeeMsg connection pool), takes appropriate //! action in the matching handler and provides a response. -use crate::context::Context; +use crate::app::*; use crate::db; use crate::error::TypedError; use crate::types::*; @@ -49,14 +49,14 @@ mod set_target_consistency_states; /// Msg request handler for requests where no response is expected. /// To handle a message, implement this and add it to the dispatch list with `=> _`. trait HandleNoResponse: Msg + Deserializable { - async fn handle(self, ctx: &Context, req: &mut impl Request) -> Result<()>; + async fn handle(self, app: &impl App, req: &mut impl Request) -> Result<()>; } /// Msg request handler for requests where a response is expected. /// To handle a message, implement this and add it to the dispatch list with `=> R`. trait HandleWithResponse: Msg + Deserializable { type Response: Msg + Serializable; - async fn handle(self, ctx: &Context, req: &mut impl Request) -> Result; + async fn handle(self, app: &impl App, req: &mut impl Request) -> Result; /// Defines the message to send back on an error during `handle()`. Defaults to /// `Response::default()`. @@ -87,7 +87,7 @@ impl Display for PreShutdownError { /// The `=> R` tells the macro that this message handler returns a response. For non-response /// messages, put a `=> _` there. Implement the appropriate handler trait for that message or you /// will get errors -pub(crate) async fn dispatch_request(ctx: &Context, mut req: impl Request) -> Result<()> { +pub(crate) async fn dispatch_request(app: &RuntimeApp, mut req: impl Request) -> Result<()> { /// Creates the dispatching match statement macro_rules! dispatch_msg { ($({$msg_type:path => $r:tt, $ctx_str:literal})*) => { @@ -106,7 +106,7 @@ pub(crate) async fn dispatch_request(ctx: &Context, mut req: impl Request) -> Re log::trace!("INCOMING from {:?}: {:?}", req.addr(), des); - let res = des.handle(ctx, &mut req).await; + let res = des.handle(app, &mut req).await; dispatch_msg!(@HANDLE res, $msg_type => $r, $ctx_str) } ),* @@ -201,38 +201,11 @@ async fn handle_unspecified_msg(req: impl Request) -> Result<()> { Ok(()) } -/// Checks if the management is in pre shutdown state -fn fail_on_pre_shutdown(ctx: &Context) -> Result<()> { - if ctx.run_state.pre_shutdown() { +/// Fails if the management is in pre shutdown state +fn fail_on_pre_shutdown(app: &impl App) -> Result<()> { + if app.rs_pre_shutdown() { return Err(anyhow!(PreShutdownError {})); } Ok(()) } - -pub async fn notify_nodes( - ctx: &Context, - node_types: &'static [NodeType], - msg: &M, -) { - log::trace!("NOTIFICATION to {node_types:?}: {msg:?}"); - - for t in node_types { - if let Err(err) = async { - let nodes = ctx - .db - .read_tx(move |tx| db::node::get_with_type(tx, *t)) - .await?; - - ctx.conn - .broadcast_datagram(nodes.into_iter().map(|e| e.uid), msg) - .await?; - - Ok(()) as Result<_> - } - .await - { - log::error!("Notification could not be sent to all {t} nodes: {err:#}"); - } - } -} diff --git a/mgmtd/src/bee_msg/ack.rs b/mgmtd/src/bee_msg/ack.rs index 3eb9bad..7bc03bc 100644 --- a/mgmtd/src/bee_msg/ack.rs +++ b/mgmtd/src/bee_msg/ack.rs @@ -2,7 +2,7 @@ use super::*; use shared::bee_msg::misc::*; impl HandleNoResponse for Ack { - async fn handle(self, _ctx: &Context, req: &mut impl Request) -> Result<()> { + async fn handle(self, _app: &impl App, req: &mut impl Request) -> Result<()> { log::debug!("Ignoring Ack from {:?}: Id: {:?}", req.addr(), self.ack_id); Ok(()) } diff --git a/mgmtd/src/bee_msg/authenticate_channel.rs b/mgmtd/src/bee_msg/authenticate_channel.rs index 9d8a5e3..95693da 100644 --- a/mgmtd/src/bee_msg/authenticate_channel.rs +++ b/mgmtd/src/bee_msg/authenticate_channel.rs @@ -2,8 +2,8 @@ use super::*; use shared::bee_msg::misc::*; impl HandleNoResponse for AuthenticateChannel { - async fn handle(self, ctx: &Context, req: &mut impl Request) -> Result<()> { - if let Some(ref secret) = ctx.info.auth_secret { + async fn handle(self, app: &impl App, req: &mut impl Request) -> Result<()> { + if let Some(ref secret) = app.static_info().auth_secret { if secret == &self.auth_secret { req.authenticate_connection(); } else { diff --git a/mgmtd/src/bee_msg/change_target_consistency_states.rs b/mgmtd/src/bee_msg/change_target_consistency_states.rs index 20433f4..2856f53 100644 --- a/mgmtd/src/bee_msg/change_target_consistency_states.rs +++ b/mgmtd/src/bee_msg/change_target_consistency_states.rs @@ -11,16 +11,15 @@ impl HandleWithResponse for ChangeTargetConsistencyStates { } } - async fn handle(self, ctx: &Context, __req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(app)?; // self.old_states is currently completely ignored. If something reports a non-GOOD state, I // see no apparent reason to that the old state matches before setting. We have the // authority, whatever nodes think their old state was doesn't matter. - let changed = ctx - .db - .write_tx(move |tx| { + let changed = app + .db_write_tx(move |tx| { let node_type = self.node_type.try_into()?; // Check given target Ids exist @@ -48,8 +47,7 @@ impl HandleWithResponse for ChangeTargetConsistencyStates { ); if changed { - notify_nodes( - ctx, + app.beemsg_send_notifications( &[NodeType::Meta, NodeType::Storage, NodeType::Client], &RefreshTargetStates { ack_id: "".into() }, ) diff --git a/mgmtd/src/bee_msg/common.rs b/mgmtd/src/bee_msg/common.rs index 28ff36a..278cf3c 100644 --- a/mgmtd/src/bee_msg/common.rs +++ b/mgmtd/src/bee_msg/common.rs @@ -10,12 +10,12 @@ use std::sync::Arc; use std::time::Duration; /// Processes incoming node information. Registers new nodes if config allows it -pub(super) async fn update_node(msg: RegisterNode, ctx: &Context) -> Result { +pub(super) async fn update_node(msg: RegisterNode, app: &impl App) -> Result { let nics = msg.nics.clone(); let requested_node_id = msg.node_id; - let info = ctx.info; + let registration_disable = app.static_info().user_config.registration_disable; - let licensed_machines = match ctx.license.get_num_machines() { + let licensed_machines = match app.lic_get_num_machines() { Ok(n) => n, Err(err) => { log::debug!( @@ -25,9 +25,8 @@ pub(super) async fn update_node(msg: RegisterNode, ctx: &Context) -> Result new node None @@ -62,7 +61,7 @@ pub(super) async fn update_node(msg: RegisterNode, ctx: &Context) -> Result &[NodeType::Meta, NodeType::Client], NodeType::Storage => &[NodeType::Meta, NodeType::Storage, NodeType::Client], diff --git a/mgmtd/src/bee_msg/get_mirror_buddy_groups.rs b/mgmtd/src/bee_msg/get_mirror_buddy_groups.rs index 1995826..f04c42f 100644 --- a/mgmtd/src/bee_msg/get_mirror_buddy_groups.rs +++ b/mgmtd/src/bee_msg/get_mirror_buddy_groups.rs @@ -4,10 +4,9 @@ use shared::bee_msg::buddy_group::*; impl HandleWithResponse for GetMirrorBuddyGroups { type Response = GetMirrorBuddyGroupsResp; - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - let groups: Vec<(BuddyGroupId, TargetId, TargetId)> = ctx - .db - .read_tx(move |tx| { + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + let groups: Vec<(BuddyGroupId, TargetId, TargetId)> = app + .db_read_tx(move |tx| { tx.query_map_collect( sql!( "SELECT group_id, p_target_id, s_target_id FROM buddy_groups_ext diff --git a/mgmtd/src/bee_msg/get_node_capacity_pools.rs b/mgmtd/src/bee_msg/get_node_capacity_pools.rs index 50e2918..6e26a0b 100644 --- a/mgmtd/src/bee_msg/get_node_capacity_pools.rs +++ b/mgmtd/src/bee_msg/get_node_capacity_pools.rs @@ -76,20 +76,22 @@ fn load_buddy_groups_info_by_type( impl HandleWithResponse for GetNodeCapacityPools { type Response = GetNodeCapacityPoolsResp; - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { // We return raw u16 here as ID because BeeGFS expects a u16 that can be // either a NodeNUmID, TargetNumID or BuddyGroupID let pools: HashMap>> = match self.query_type { CapacityPoolQueryType::Meta => { - let targets = ctx - .db - .read_tx(|tx| load_targets_info_by_type(tx, NodeTypeServer::Meta)) + let targets = app + .db_read_tx(|tx| load_targets_info_by_type(tx, NodeTypeServer::Meta)) .await?; let cp_calc = CapPoolCalculator::new( - ctx.info.user_config.cap_pool_meta_limits.clone(), - ctx.info.user_config.cap_pool_dynamic_meta_limits.as_ref(), + app.static_info().user_config.cap_pool_meta_limits.clone(), + app.static_info() + .user_config + .cap_pool_dynamic_meta_limits + .as_ref(), &targets, )?; @@ -105,9 +107,8 @@ impl HandleWithResponse for GetNodeCapacityPools { } CapacityPoolQueryType::Storage => { - let (targets, pools) = ctx - .db - .read_tx(|tx| { + let (targets, pools) = app + .db_read_tx(|tx| { let targets = load_targets_info_by_type(tx, NodeTypeServer::Storage)?; let pools: Vec = tx.query_map_collect( @@ -125,8 +126,11 @@ impl HandleWithResponse for GetNodeCapacityPools { let f_targets = targets.iter().filter(|e| e.pool_id == Some(sp)); let cp_calc = CapPoolCalculator::new( - ctx.info.user_config.cap_pool_storage_limits.clone(), - ctx.info + app.static_info() + .user_config + .cap_pool_storage_limits + .clone(), + app.static_info() .user_config .cap_pool_dynamic_storage_limits .as_ref(), @@ -146,14 +150,16 @@ impl HandleWithResponse for GetNodeCapacityPools { } CapacityPoolQueryType::MetaMirrored => { - let groups = ctx - .db - .read_tx(|tx| load_buddy_groups_info_by_type(tx, NodeTypeServer::Meta)) + let groups = app + .db_read_tx(|tx| load_buddy_groups_info_by_type(tx, NodeTypeServer::Meta)) .await?; let cp_calc = CapPoolCalculator::new( - ctx.info.user_config.cap_pool_meta_limits.clone(), - ctx.info.user_config.cap_pool_dynamic_meta_limits.as_ref(), + app.static_info().user_config.cap_pool_meta_limits.clone(), + app.static_info() + .user_config + .cap_pool_dynamic_meta_limits + .as_ref(), &groups, )?; @@ -170,9 +176,8 @@ impl HandleWithResponse for GetNodeCapacityPools { } CapacityPoolQueryType::StorageMirrored => { - let (groups, pools) = ctx - .db - .read_tx(|tx| { + let (groups, pools) = app + .db_read_tx(|tx| { let groups = load_buddy_groups_info_by_type(tx, NodeTypeServer::Storage)?; let pools: Vec = tx.query_map_collect( @@ -190,8 +195,11 @@ impl HandleWithResponse for GetNodeCapacityPools { let f_groups = groups.iter().filter(|e| e.pool_id == Some(sp)); let cp_calc = CapPoolCalculator::new( - ctx.info.user_config.cap_pool_storage_limits.clone(), - ctx.info + app.static_info() + .user_config + .cap_pool_storage_limits + .clone(), + app.static_info() .user_config .cap_pool_dynamic_storage_limits .as_ref(), diff --git a/mgmtd/src/bee_msg/get_nodes.rs b/mgmtd/src/bee_msg/get_nodes.rs index 165e755..da735f1 100644 --- a/mgmtd/src/bee_msg/get_nodes.rs +++ b/mgmtd/src/bee_msg/get_nodes.rs @@ -6,10 +6,9 @@ use shared::bee_msg::node::*; impl HandleWithResponse for GetNodes { type Response = GetNodesResp; - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - let res = ctx - .db - .read_tx(move |tx| { + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + let res = app + .db_read_tx(move |tx| { let node_type = self.node_type; let res = ( db::node::get_with_type(tx, node_type)?, diff --git a/mgmtd/src/bee_msg/get_states_and_buddy_groups.rs b/mgmtd/src/bee_msg/get_states_and_buddy_groups.rs index 54459a3..db06c8d 100644 --- a/mgmtd/src/bee_msg/get_states_and_buddy_groups.rs +++ b/mgmtd/src/bee_msg/get_states_and_buddy_groups.rs @@ -5,15 +5,14 @@ use shared::bee_msg::buddy_group::*; impl HandleWithResponse for GetStatesAndBuddyGroups { type Response = GetStatesAndBuddyGroupsResp; - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { let node_type: NodeTypeServer = self.node_type.try_into()?; - let pre_shutdown = ctx.run_state.pre_shutdown(); - let node_offline_timeout = ctx.info.user_config.node_offline_timeout; + let pre_shutdown = app.rs_pre_shutdown(); + let node_offline_timeout = app.static_info().user_config.node_offline_timeout; - let (targets, groups) = ctx - .db - .read_tx(move |tx| { + let (targets, groups) = app + .db_read_tx(move |tx| { let targets = get_targets_with_states( tx, pre_shutdown, @@ -65,7 +64,7 @@ impl HandleWithResponse for GetStatesAndBuddyGroups { // If it's a client that requested it, notify the run controller that it pulled states if self.requested_by_client_id != 0 { - ctx.notify_client_pulled_state(self.node_type, self.requested_by_client_id); + app.rs_notify_client_pulled_state(self.node_type, self.requested_by_client_id); } Ok(resp) diff --git a/mgmtd/src/bee_msg/get_storage_pools.rs b/mgmtd/src/bee_msg/get_storage_pools.rs index 3531910..db85287 100644 --- a/mgmtd/src/bee_msg/get_storage_pools.rs +++ b/mgmtd/src/bee_msg/get_storage_pools.rs @@ -24,10 +24,9 @@ impl CapacityInfo for &TargetOrBuddyGroup { impl HandleWithResponse for GetStoragePools { type Response = GetStoragePoolsResp; - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - let (pools, targets, buddy_groups) = ctx - .db - .read_tx(move |tx| { + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + let (pools, targets, buddy_groups) = app + .db_read_tx(move |tx| { let pools: Vec<(PoolId, String)> = tx.query_map_collect( sql!( "SELECT pool_id, alias FROM storage_pools @@ -105,8 +104,11 @@ impl HandleWithResponse for GetStoragePools { let f_buddy_groups = buddy_groups.iter().filter(|t| t.pool_id == pool.0); let cp_targets_calc = CapPoolCalculator::new( - ctx.info.user_config.cap_pool_storage_limits.clone(), - ctx.info + app.static_info() + .user_config + .cap_pool_storage_limits + .clone(), + app.static_info() .user_config .cap_pool_dynamic_storage_limits .as_ref(), @@ -114,8 +116,11 @@ impl HandleWithResponse for GetStoragePools { )?; let cp_buddy_groups_calc = CapPoolCalculator::new( - ctx.info.user_config.cap_pool_storage_limits.clone(), - ctx.info + app.static_info() + .user_config + .cap_pool_storage_limits + .clone(), + app.static_info() .user_config .cap_pool_dynamic_storage_limits .as_ref(), diff --git a/mgmtd/src/bee_msg/get_target_mappings.rs b/mgmtd/src/bee_msg/get_target_mappings.rs index ba8743d..a707600 100644 --- a/mgmtd/src/bee_msg/get_target_mappings.rs +++ b/mgmtd/src/bee_msg/get_target_mappings.rs @@ -4,10 +4,9 @@ use shared::bee_msg::target::*; impl HandleWithResponse for GetTargetMappings { type Response = GetTargetMappingsResp; - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - let mapping: HashMap = ctx - .db - .read_tx(move |tx| { + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + let mapping: HashMap = app + .db_read_tx(move |tx| { tx.query_map_collect( sql!( "SELECT target_id, node_id diff --git a/mgmtd/src/bee_msg/get_target_states.rs b/mgmtd/src/bee_msg/get_target_states.rs index a9349c3..2f0674d 100644 --- a/mgmtd/src/bee_msg/get_target_states.rs +++ b/mgmtd/src/bee_msg/get_target_states.rs @@ -5,13 +5,12 @@ use shared::bee_msg::target::*; impl HandleWithResponse for GetTargetStates { type Response = GetTargetStatesResp; - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - let pre_shutdown = ctx.run_state.pre_shutdown(); - let node_offline_timeout = ctx.info.user_config.node_offline_timeout; + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + let pre_shutdown = app.rs_pre_shutdown(); + let node_offline_timeout = app.static_info().user_config.node_offline_timeout; - let targets = ctx - .db - .read_tx(move |tx| { + let targets = app + .db_read_tx(move |tx| { get_targets_with_states( tx, pre_shutdown, diff --git a/mgmtd/src/bee_msg/heartbeat.rs b/mgmtd/src/bee_msg/heartbeat.rs index 4417a28..f3127fd 100644 --- a/mgmtd/src/bee_msg/heartbeat.rs +++ b/mgmtd/src/bee_msg/heartbeat.rs @@ -6,8 +6,8 @@ use shared::bee_msg::node::*; impl HandleWithResponse for Heartbeat { type Response = Ack; - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(app)?; update_node( RegisterNode { @@ -23,7 +23,7 @@ impl HandleWithResponse for Heartbeat { port_tcp_unused: self.port_tcp_unused, machine_uuid: self.machine_uuid, }, - ctx, + app, ) .await?; diff --git a/mgmtd/src/bee_msg/heartbeat_request.rs b/mgmtd/src/bee_msg/heartbeat_request.rs index 095156b..f180aa7 100644 --- a/mgmtd/src/bee_msg/heartbeat_request.rs +++ b/mgmtd/src/bee_msg/heartbeat_request.rs @@ -5,10 +5,9 @@ use shared::bee_msg::node::*; impl HandleWithResponse for HeartbeatRequest { type Response = Heartbeat; - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - let (alias, nics) = ctx - .db - .read_tx(|tx| { + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + let (alias, nics) = app + .db_read_tx(|tx| { Ok(( db::entity::get_alias(tx, MGMTD_UID)? .ok_or_else(|| TypedError::value_not_found("management uid", MGMTD_UID))?, @@ -29,8 +28,8 @@ impl HandleWithResponse for HeartbeatRequest { node_num_id: MGMTD_ID, root_num_id: 0, is_root_mirrored: 0, - port: ctx.info.user_config.beemsg_port, - port_tcp_unused: ctx.info.user_config.beemsg_port, + port: app.static_info().user_config.beemsg_port, + port_tcp_unused: app.static_info().user_config.beemsg_port, nic_list: nics, machine_uuid: vec![], // No need for the other nodes to know machine UUIDs }; diff --git a/mgmtd/src/bee_msg/map_targets.rs b/mgmtd/src/bee_msg/map_targets.rs index a421ac4..4512233 100644 --- a/mgmtd/src/bee_msg/map_targets.rs +++ b/mgmtd/src/bee_msg/map_targets.rs @@ -4,26 +4,25 @@ use shared::bee_msg::target::*; impl HandleWithResponse for MapTargets { type Response = MapTargetsResp; - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(app)?; let target_ids = self.target_ids.keys().copied().collect::>(); - ctx.db - .write_tx(move |tx| { - // Check node Id exists - let node = LegacyId { - node_type: NodeType::Storage, - num_id: self.node_id, - } - .resolve(tx, EntityType::Node)?; - // Check all target Ids exist - db::target::validate_ids(tx, &target_ids, NodeTypeServer::Storage)?; - // Due to the check above, this must always match all the given ids - db::target::update_storage_node_mappings(tx, &target_ids, node.num_id())?; - Ok(()) - }) - .await?; + app.db_write_tx(move |tx| { + // Check node Id exists + let node = LegacyId { + node_type: NodeType::Storage, + num_id: self.node_id, + } + .resolve(tx, EntityType::Node)?; + // Check all target Ids exist + db::target::validate_ids(tx, &target_ids, NodeTypeServer::Storage)?; + // Due to the check above, this must always match all the given ids + db::target::update_storage_node_mappings(tx, &target_ids, node.num_id())?; + Ok(()) + }) + .await?; // At this point, all mappings must have been successful @@ -33,8 +32,7 @@ impl HandleWithResponse for MapTargets { self.node_id ); - notify_nodes( - ctx, + app.beemsg_send_notifications( &[NodeType::Meta, NodeType::Storage, NodeType::Client], &MapTargets { target_ids: self.target_ids.clone(), diff --git a/mgmtd/src/bee_msg/map_targets_resp.rs b/mgmtd/src/bee_msg/map_targets_resp.rs index 1b52481..798b9c5 100644 --- a/mgmtd/src/bee_msg/map_targets_resp.rs +++ b/mgmtd/src/bee_msg/map_targets_resp.rs @@ -2,7 +2,7 @@ use super::*; use shared::bee_msg::target::*; impl HandleNoResponse for MapTargetsResp { - async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result<()> { + async fn handle(self, _app: &impl App, _req: &mut impl Request) -> Result<()> { // This is sent from the nodes as a result of the MapTargets notification after // map_targets was called. We just ignore it. Ok(()) diff --git a/mgmtd/src/bee_msg/peer_info.rs b/mgmtd/src/bee_msg/peer_info.rs index f54280b..055bc80 100644 --- a/mgmtd/src/bee_msg/peer_info.rs +++ b/mgmtd/src/bee_msg/peer_info.rs @@ -2,7 +2,7 @@ use super::*; use shared::bee_msg::misc::*; impl HandleNoResponse for PeerInfo { - async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result<()> { + async fn handle(self, _app: &impl App, _req: &mut impl Request) -> Result<()> { // This is supposed to give some information about a connection, but it looks // like this isnt used at all Ok(()) diff --git a/mgmtd/src/bee_msg/refresh_capacity_pools.rs b/mgmtd/src/bee_msg/refresh_capacity_pools.rs index 8d56235..09304b0 100644 --- a/mgmtd/src/bee_msg/refresh_capacity_pools.rs +++ b/mgmtd/src/bee_msg/refresh_capacity_pools.rs @@ -4,7 +4,7 @@ use shared::bee_msg::misc::*; impl HandleWithResponse for RefreshCapacityPools { type Response = Ack; - async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result { + async fn handle(self, _app: &impl App, _req: &mut impl Request) -> Result { // This message is superfluous and therefore ignored. It is meant to tell the // mgmtd to trigger a capacity pool pull immediately after a node starts. // meta and storage send a SetTargetInfo before this msg though, diff --git a/mgmtd/src/bee_msg/register_node.rs b/mgmtd/src/bee_msg/register_node.rs index 6e77dca..53a0d3d 100644 --- a/mgmtd/src/bee_msg/register_node.rs +++ b/mgmtd/src/bee_msg/register_node.rs @@ -5,20 +5,19 @@ use shared::bee_msg::node::*; impl HandleWithResponse for RegisterNode { type Response = RegisterNodeResp; - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(app)?; - let node_id = update_node(self, ctx).await?; + let node_id = update_node(self, app).await?; - let fs_uuid: String = ctx - .db - .read_tx(|tx| db::config::get(tx, db::config::Config::FsUuid)) + let fs_uuid: String = app + .db_read_tx(|tx| db::config::get(tx, db::config::Config::FsUuid)) .await? .ok_or_else(|| anyhow!("Could not read file system UUID from database"))?; Ok(RegisterNodeResp { node_num_id: node_id, - grpc_port: ctx.info.user_config.grpc_port, + grpc_port: app.static_info().user_config.grpc_port, fs_uuid: fs_uuid.into_bytes(), }) } diff --git a/mgmtd/src/bee_msg/register_target.rs b/mgmtd/src/bee_msg/register_target.rs index c96e4c7..5190f73 100644 --- a/mgmtd/src/bee_msg/register_target.rs +++ b/mgmtd/src/bee_msg/register_target.rs @@ -4,14 +4,13 @@ use shared::bee_msg::target::*; impl HandleWithResponse for RegisterTarget { type Response = RegisterTargetResp; - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(app)?; - let ctx2 = ctx.clone(); + let registration_disable = app.static_info().user_config.registration_disable; - let (id, is_new) = ctx - .db - .write_tx(move |tx| { + let (id, is_new) = app + .db_write_tx(move |tx| { // Do not do anything if the target already exists if let Some(id) = try_resolve_num_id( tx, @@ -22,7 +21,7 @@ impl HandleWithResponse for RegisterTarget { return Ok((id.num_id().try_into()?, false)); } - if ctx2.info.user_config.registration_disable { + if registration_disable { bail!("Registration of new targets is not allowed"); } diff --git a/mgmtd/src/bee_msg/remove_node.rs b/mgmtd/src/bee_msg/remove_node.rs index 861cd80..321d897 100644 --- a/mgmtd/src/bee_msg/remove_node.rs +++ b/mgmtd/src/bee_msg/remove_node.rs @@ -10,12 +10,11 @@ impl HandleWithResponse for RemoveNode { } } - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(app)?; - let node = ctx - .db - .write_tx(move |tx| { + let node = app + .db_write_tx(move |tx| { if self.node_type != NodeType::Client { bail!( "This BeeMsg handler can only delete client nodes. \ @@ -37,8 +36,7 @@ For server nodes, the grpc handler must be used." log::info!("Node deleted: {node}"); - notify_nodes( - ctx, + app.beemsg_send_notifications( match self.node_type { shared::types::NodeType::Meta => &[NodeType::Meta, NodeType::Client], shared::types::NodeType::Storage => { diff --git a/mgmtd/src/bee_msg/remove_node_resp.rs b/mgmtd/src/bee_msg/remove_node_resp.rs index 72761bf..fbffdf3 100644 --- a/mgmtd/src/bee_msg/remove_node_resp.rs +++ b/mgmtd/src/bee_msg/remove_node_resp.rs @@ -2,7 +2,7 @@ use super::*; use shared::bee_msg::node::*; impl HandleNoResponse for RemoveNodeResp { - async fn handle(self, _ctx: &Context, req: &mut impl Request) -> Result<()> { + async fn handle(self, _app: &impl App, req: &mut impl Request) -> Result<()> { // response from server nodes to the RemoveNode notification log::debug!("Ignoring RemoveNodeResp msg from {:?}", req.addr()); Ok(()) diff --git a/mgmtd/src/bee_msg/request_exceeded_quota.rs b/mgmtd/src/bee_msg/request_exceeded_quota.rs index 59afeb3..b667f91 100644 --- a/mgmtd/src/bee_msg/request_exceeded_quota.rs +++ b/mgmtd/src/bee_msg/request_exceeded_quota.rs @@ -12,10 +12,9 @@ impl HandleWithResponse for RequestExceededQuota { } } - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - let inner = ctx - .db - .read_tx(move |tx| { + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + let inner = app + .db_read_tx(move |tx| { let exceeded_ids = db::quota_usage::exceeded_quota_ids( tx, if self.pool_id != 0 { diff --git a/mgmtd/src/bee_msg/set_channel_direct.rs b/mgmtd/src/bee_msg/set_channel_direct.rs index 12f1e4e..cd88fe0 100644 --- a/mgmtd/src/bee_msg/set_channel_direct.rs +++ b/mgmtd/src/bee_msg/set_channel_direct.rs @@ -2,7 +2,7 @@ use super::*; use shared::bee_msg::misc::*; impl HandleNoResponse for SetChannelDirect { - async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result<()> { + async fn handle(self, _app: &impl App, _req: &mut impl Request) -> Result<()> { // do nothing Ok(()) } diff --git a/mgmtd/src/bee_msg/set_mirror_buddy_groups_resp.rs b/mgmtd/src/bee_msg/set_mirror_buddy_groups_resp.rs index 8630e46..7b6e104 100644 --- a/mgmtd/src/bee_msg/set_mirror_buddy_groups_resp.rs +++ b/mgmtd/src/bee_msg/set_mirror_buddy_groups_resp.rs @@ -2,7 +2,7 @@ use super::*; use shared::bee_msg::buddy_group::*; impl HandleNoResponse for SetMirrorBuddyGroupResp { - async fn handle(self, _ctx: &Context, _req: &mut impl Request) -> Result<()> { + async fn handle(self, _app: &impl App, _req: &mut impl Request) -> Result<()> { // response from server nodes to SetMirrorBuddyGroup notification Ok(()) } diff --git a/mgmtd/src/bee_msg/set_storage_target_info.rs b/mgmtd/src/bee_msg/set_storage_target_info.rs index 4d71205..901fe2a 100644 --- a/mgmtd/src/bee_msg/set_storage_target_info.rs +++ b/mgmtd/src/bee_msg/set_storage_target_info.rs @@ -11,29 +11,28 @@ impl HandleWithResponse for SetStorageTargetInfo { } } - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(app)?; let node_type = self.node_type; - ctx.db - .write_tx(move |tx| { - db::target::get_and_update_capacities( - tx, - self.info.into_iter().map(|e| { - Ok(( - e.target_id, - TargetCapacities { - total_space: Some(e.total_space.try_into()?), - total_inodes: Some(e.total_inodes.try_into()?), - free_space: Some(e.free_space.try_into()?), - free_inodes: Some(e.free_inodes.try_into()?), - }, - )) - }), - self.node_type.try_into()?, - ) - }) - .await?; + app.db_write_tx(move |tx| { + db::target::get_and_update_capacities( + tx, + self.info.into_iter().map(|e| { + Ok(( + e.target_id, + TargetCapacities { + total_space: Some(e.total_space.try_into()?), + total_inodes: Some(e.total_inodes.try_into()?), + free_space: Some(e.free_space.try_into()?), + free_inodes: Some(e.free_inodes.try_into()?), + }, + )) + }), + self.node_type.try_into()?, + ) + }) + .await?; log::debug!("Updated {node_type:?} target info"); diff --git a/mgmtd/src/bee_msg/set_target_consistency_states.rs b/mgmtd/src/bee_msg/set_target_consistency_states.rs index fac2db0..75079a2 100644 --- a/mgmtd/src/bee_msg/set_target_consistency_states.rs +++ b/mgmtd/src/bee_msg/set_target_consistency_states.rs @@ -11,33 +11,31 @@ impl HandleWithResponse for SetTargetConsistencyStates { } } - async fn handle(self, ctx: &Context, _req: &mut impl Request) -> Result { - fail_on_pre_shutdown(ctx)?; + async fn handle(self, app: &impl App, _req: &mut impl Request) -> Result { + fail_on_pre_shutdown(app)?; let node_type = self.node_type.try_into()?; let msg = self.clone(); - ctx.db - .write_tx(move |tx| { - // Check given target Ids exist - db::target::validate_ids(tx, &msg.target_ids, node_type)?; + app.db_write_tx(move |tx| { + // Check given target Ids exist + db::target::validate_ids(tx, &msg.target_ids, node_type)?; - if msg.set_online > 0 { - update_last_contact_times(tx, &msg.target_ids, node_type)?; - } + if msg.set_online > 0 { + update_last_contact_times(tx, &msg.target_ids, node_type)?; + } - db::target::update_consistency_states( - tx, - msg.target_ids.into_iter().zip(msg.states.iter().copied()), - node_type, - ) - }) - .await?; + db::target::update_consistency_states( + tx, + msg.target_ids.into_iter().zip(msg.states.iter().copied()), + node_type, + ) + }) + .await?; log::info!("Set consistency state for targets {:?}", self.target_ids,); - notify_nodes( - ctx, + app.beemsg_send_notifications( &[NodeType::Meta, NodeType::Storage, NodeType::Client], &RefreshTargetStates { ack_id: "".into() }, ) diff --git a/mgmtd/src/context.rs b/mgmtd/src/context.rs deleted file mode 100644 index ed7321a..0000000 --- a/mgmtd/src/context.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! Interfaces and implementations for in-app interaction between tasks or threads. - -use crate::bee_msg::dispatch_request; -use crate::license::LicenseVerifier; -use crate::{ClientPulledStateNotification, StaticInfo}; -use anyhow::Result; -use shared::conn::msg_dispatch::*; -use shared::conn::outgoing::Pool; -use shared::run_state::WeakRunStateHandle; -use shared::types::{NodeId, NodeType}; -use std::ops::Deref; -use std::sync::Arc; -use tokio::sync::mpsc; - -/// A collection of Handles used for interacting and accessing the different components of the app. -/// -/// This is the actual runtime object that can be shared between tasks. Interfaces should, however, -/// accept any implementation of the AppContext trait instead. -#[derive(Clone, Debug)] -pub(crate) struct Context(Arc); - -/// Stores the actual handles. -#[derive(Debug)] -pub(crate) struct InnerContext { - pub conn: Pool, - pub db: sqlite::Connections, - pub license: LicenseVerifier, - pub info: &'static StaticInfo, - pub run_state: WeakRunStateHandle, - shutdown_client_id: mpsc::Sender, -} - -impl Context { - /// Creates a new AppHandles object. - /// - /// Takes all the stored handles. - pub(crate) fn new( - conn: Pool, - db: sqlite::Connections, - license: LicenseVerifier, - info: &'static StaticInfo, - run_state: WeakRunStateHandle, - shutdown_client_id: mpsc::Sender, - ) -> Self { - Self(Arc::new(InnerContext { - conn, - db, - license, - info, - run_state, - shutdown_client_id, - })) - } - - pub(crate) fn notify_client_pulled_state(&self, node_type: NodeType, node_id: NodeId) { - if self.run_state.pre_shutdown() { - let tx = self.shutdown_client_id.clone(); - - // We don't want to block the task calling this and are not interested by the results - tokio::spawn(async move { - let _ = tx.send((node_type, node_id)).await; - }); - } - } -} - -/// Derefs to InnerAppHandle which stores all the handles. -/// -/// Allows transparent access. -impl Deref for Context { - type Target = InnerContext; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DispatchRequest for Context { - async fn dispatch_request(&self, req: impl Request) -> Result<()> { - dispatch_request(self, req).await - } -} diff --git a/mgmtd/src/grpc.rs b/mgmtd/src/grpc.rs index 1a0d4ba..b561172 100644 --- a/mgmtd/src/grpc.rs +++ b/mgmtd/src/grpc.rs @@ -1,7 +1,6 @@ //! gRPC server and handlers -use crate::bee_msg::notify_nodes; -use crate::context::Context; +use crate::app::*; use crate::db; use crate::license::LicensedFeature; use crate::types::{ResolveEntityId, SqliteEnumExt}; @@ -47,7 +46,7 @@ mod start_resync; /// Management gRPC service implementation struct #[derive(Debug)] pub(crate) struct ManagementService { - pub ctx: Context, + pub app: RuntimeApp, } /// Implementation of the management gRPC service. Use the shared::impl_grpc_handler! macro to @@ -172,21 +171,21 @@ impl pm::management_server::Management for ManagementService { } /// Serve gRPC requests on the `grpc_port` extracted from the config -pub(crate) fn serve(ctx: Context, mut shutdown: RunStateHandle) -> Result<()> { +pub(crate) fn serve(app: RuntimeApp, mut shutdown: RunStateHandle) -> Result<()> { let builder = Server::builder(); // If gRPC TLS is enabled, configure the server accordingly - let mut builder = if !ctx.info.user_config.tls_disable { - let tls_cert = std::fs::read(&ctx.info.user_config.tls_cert_file).with_context(|| { + let mut builder = if !app.info.user_config.tls_disable { + let tls_cert = std::fs::read(&app.info.user_config.tls_cert_file).with_context(|| { format!( "Could not read TLS certificate file {:?}", - &ctx.info.user_config.tls_cert_file + &app.info.user_config.tls_cert_file ) })?; - let tls_key = std::fs::read(&ctx.info.user_config.tls_key_file).with_context(|| { + let tls_key = std::fs::read(&app.info.user_config.tls_key_file).with_context(|| { format!( "Could not read TLS key file {:?}", - &ctx.info.user_config.tls_key_file + &app.info.user_config.tls_key_file ) })?; @@ -197,12 +196,12 @@ pub(crate) fn serve(ctx: Context, mut shutdown: RunStateHandle) -> Result<()> { builder }; - let ctx2 = ctx.clone(); + let app2 = app.clone(); let service = pm::management_server::ManagementServer::with_interceptor( - ManagementService { ctx: ctx.clone() }, + ManagementService { app: app.clone() }, move |req: Request<()>| { // If authentication is enabled, require the secret passed with every request - if let Some(required_secret) = ctx2.info.auth_secret { + if let Some(required_secret) = app2.info.auth_secret { let check = || -> Result<()> { let Some(request_secret) = req.metadata().get("auth-secret") else { bail!("Request requires authentication but no secret was provided") @@ -227,12 +226,12 @@ pub(crate) fn serve(ctx: Context, mut shutdown: RunStateHandle) -> Result<()> { ); let serve_addr = SocketAddr::new( - if ctx.info.use_ipv6 { + if app.info.use_ipv6 { Ipv6Addr::UNSPECIFIED.into() } else { Ipv4Addr::UNSPECIFIED.into() }, - ctx.info.user_config.grpc_port, + app.info.user_config.grpc_port, ); log::info!("Serving gRPC requests on {serve_addr}"); @@ -250,18 +249,17 @@ pub(crate) fn serve(ctx: Context, mut shutdown: RunStateHandle) -> Result<()> { Ok(()) } -/// Checks if the given license feature is enabled or fails with "Unauthenticated" if not -fn needs_license(ctx: &Context, feature: LicensedFeature) -> Result<()> { - ctx.license - .verify_feature(feature) - .status_code(Code::Unauthenticated) -} - -/// Checks if the management is in pre shutdown state -fn fail_on_pre_shutdown(ctx: &Context) -> Result<()> { - if ctx.run_state.pre_shutdown() { +/// Fails if the management is in pre shutdown state +fn fail_on_pre_shutdown(app: &impl App) -> Result<()> { + if app.rs_pre_shutdown() { return Err(anyhow!("Management is shutting down")).status_code(Code::Unavailable); } Ok(()) } + +/// Fails with "Unauthenticated" if the given license feature is not enabled +fn fail_on_missing_license(app: &impl App, feature: LicensedFeature) -> Result<()> { + app.lic_verify_feature(feature) + .status_code(Code::Unauthenticated) +} diff --git a/mgmtd/src/grpc/assign_pool.rs b/mgmtd/src/grpc/assign_pool.rs index c04de9f..35a3995 100644 --- a/mgmtd/src/grpc/assign_pool.rs +++ b/mgmtd/src/grpc/assign_pool.rs @@ -3,17 +3,16 @@ use shared::bee_msg::storage_pool::RefreshStoragePools; /// Assigns a pool to a list of targets and buddy groups. pub(crate) async fn assign_pool( - ctx: Context, + app: &impl App, req: pm::AssignPoolRequest, ) -> Result { - needs_license(&ctx, LicensedFeature::Storagepool)?; - fail_on_pre_shutdown(&ctx)?; + fail_on_missing_license(app, LicensedFeature::Storagepool)?; + fail_on_pre_shutdown(app)?; let pool: EntityId = required_field(req.pool)?.try_into()?; - let pool = ctx - .db - .write_tx(move |tx| { + let pool = app + .db_write_tx(move |tx| { let pool = pool.resolve(tx, EntityType::Pool)?; do_assign(tx, pool.num_id().try_into()?, req.targets, req.buddy_groups)?; Ok(pool) @@ -22,8 +21,7 @@ pub(crate) async fn assign_pool( log::info!("Pool assigned: {pool}"); - notify_nodes( - &ctx, + app.beemsg_send_notifications( &[NodeType::Meta, NodeType::Storage], &RefreshStoragePools { ack_id: "".into() }, ) diff --git a/mgmtd/src/grpc/create_buddy_group.rs b/mgmtd/src/grpc/create_buddy_group.rs index e26772a..4ecddaa 100644 --- a/mgmtd/src/grpc/create_buddy_group.rs +++ b/mgmtd/src/grpc/create_buddy_group.rs @@ -3,11 +3,11 @@ use shared::bee_msg::buddy_group::SetMirrorBuddyGroup; /// Creates a new buddy group pub(crate) async fn create_buddy_group( - ctx: Context, + app: &impl App, req: pm::CreateBuddyGroupRequest, ) -> Result { - needs_license(&ctx, LicensedFeature::Mirroring)?; - fail_on_pre_shutdown(&ctx)?; + fail_on_missing_license(app, LicensedFeature::Mirroring)?; + fail_on_pre_shutdown(app)?; let node_type: NodeTypeServer = req.node_type().try_into()?; let alias: Alias = required_field(req.alias)?.try_into()?; @@ -15,9 +15,8 @@ pub(crate) async fn create_buddy_group( let p_target: EntityId = required_field(req.primary_target)?.try_into()?; let s_target: EntityId = required_field(req.secondary_target)?.try_into()?; - let (group, p_target, s_target) = ctx - .db - .write_tx(move |tx| { + let (group, p_target, s_target) = app + .db_write_tx(move |tx| { let p_target = p_target.resolve(tx, EntityType::Target)?; let s_target = s_target.resolve(tx, EntityType::Target)?; @@ -46,8 +45,7 @@ pub(crate) async fn create_buddy_group( log::info!("Buddy group created: {group}"); - notify_nodes( - &ctx, + app.beemsg_send_notifications( &[NodeType::Meta, NodeType::Storage, NodeType::Client], &SetMirrorBuddyGroup { ack_id: "".into(), diff --git a/mgmtd/src/grpc/create_pool.rs b/mgmtd/src/grpc/create_pool.rs index d8267f9..c8a97ca 100644 --- a/mgmtd/src/grpc/create_pool.rs +++ b/mgmtd/src/grpc/create_pool.rs @@ -4,11 +4,11 @@ use shared::bee_msg::storage_pool::RefreshStoragePools; /// Creates a new pool, optionally assigning targets and groups pub(crate) async fn create_pool( - ctx: Context, + app: &impl App, req: pm::CreatePoolRequest, ) -> Result { - needs_license(&ctx, LicensedFeature::Storagepool)?; - fail_on_pre_shutdown(&ctx)?; + fail_on_missing_license(app, LicensedFeature::Storagepool)?; + fail_on_pre_shutdown(app)?; if req.node_type() != pb::NodeType::Storage { bail!("node type must be storage"); @@ -17,9 +17,8 @@ pub(crate) async fn create_pool( let alias: Alias = required_field(req.alias)?.try_into()?; let num_id: PoolId = req.num_id.unwrap_or_default().try_into()?; - let (pool_uid, alias, pool_id) = ctx - .db - .write_tx(move |tx| { + let (pool_uid, alias, pool_id) = app + .db_write_tx(move |tx| { let (pool_uid, pool_id) = db::storage_pool::insert(tx, num_id, &alias)?; do_assign(tx, pool_id, req.targets, req.buddy_groups)?; Ok((pool_uid, alias, pool_id)) @@ -37,8 +36,7 @@ pub(crate) async fn create_pool( log::info!("Pool created: {pool}"); - notify_nodes( - &ctx, + app.beemsg_send_notifications( &[NodeType::Meta, NodeType::Storage], &RefreshStoragePools { ack_id: "".into() }, ) diff --git a/mgmtd/src/grpc/delete_buddy_group.rs b/mgmtd/src/grpc/delete_buddy_group.rs index 2a67daf..9b314a9 100644 --- a/mgmtd/src/grpc/delete_buddy_group.rs +++ b/mgmtd/src/grpc/delete_buddy_group.rs @@ -5,19 +5,18 @@ use shared::bee_msg::buddy_group::{RemoveBuddyGroup, RemoveBuddyGroupResp}; /// Deletes a buddy group. This function is racy as it is a two step process, talking to other /// nodes in between. Since it is rarely used, that's ok though. pub(crate) async fn delete_buddy_group( - ctx: Context, + app: &impl App, req: pm::DeleteBuddyGroupRequest, ) -> Result { - needs_license(&ctx, LicensedFeature::Mirroring)?; - fail_on_pre_shutdown(&ctx)?; + fail_on_missing_license(app, LicensedFeature::Mirroring)?; + fail_on_pre_shutdown(app)?; let group: EntityId = required_field(req.group)?.try_into()?; let execute: bool = required_field(req.execute)?; // 1. Check deletion is allowed - let (group, p_node_uid, s_node_uid) = ctx - .db - .conn(move |conn| { + let (group, p_node_uid, s_node_uid) = app + .db_conn(move |conn| { let tx = conn.transaction()?; let group = group.resolve(&tx, EntityType::BuddyGroup)?; @@ -45,8 +44,8 @@ pub(crate) async fn delete_buddy_group( force: 0, }; - let p_res: RemoveBuddyGroupResp = ctx.conn.request(p_node_uid, &remove_bee_msg).await?; - let s_res: RemoveBuddyGroupResp = ctx.conn.request(s_node_uid, &remove_bee_msg).await?; + let p_res: RemoveBuddyGroupResp = app.beemsg_request(p_node_uid, &remove_bee_msg).await?; + let s_res: RemoveBuddyGroupResp = app.beemsg_request(s_node_uid, &remove_bee_msg).await?; if p_res.result != OpsErr::SUCCESS || s_res.result != OpsErr::SUCCESS { bail!( @@ -58,18 +57,17 @@ Primary result: {:?}, Secondary result: {:?}", } // 3. If the deletion request succeeded, remove the group from the database - ctx.db - .conn(move |conn| { - let tx = conn.transaction()?; + app.db_conn(move |conn| { + let tx = conn.transaction()?; - db::buddy_group::delete_storage(&tx, group_id)?; + db::buddy_group::delete_storage(&tx, group_id)?; - if execute { - tx.commit()?; - } - Ok(()) - }) - .await?; + if execute { + tx.commit()?; + } + Ok(()) + }) + .await?; if execute { log::info!("Buddy group deleted: {group}"); diff --git a/mgmtd/src/grpc/delete_node.rs b/mgmtd/src/grpc/delete_node.rs index f3fb407..ae05d27 100644 --- a/mgmtd/src/grpc/delete_node.rs +++ b/mgmtd/src/grpc/delete_node.rs @@ -3,17 +3,16 @@ use shared::bee_msg::node::RemoveNode; /// Deletes a node. If it is a meta node, deletes its target first. pub(crate) async fn delete_node( - ctx: Context, + app: &impl App, req: pm::DeleteNodeRequest, ) -> Result { - fail_on_pre_shutdown(&ctx)?; + fail_on_pre_shutdown(app)?; let node: EntityId = required_field(req.node)?.try_into()?; let execute: bool = required_field(req.execute)?; - let node = ctx - .db - .conn(move |conn| { + let node = app + .db_conn(move |conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let node = node.resolve(&tx, EntityType::Node)?; @@ -76,8 +75,7 @@ pub(crate) async fn delete_node( if execute { log::info!("Node deleted: {node}"); - notify_nodes( - &ctx, + app.beemsg_send_notifications( match node.node_type() { NodeType::Meta => &[NodeType::Meta, NodeType::Client], NodeType::Storage => &[NodeType::Meta, NodeType::Storage, NodeType::Client], diff --git a/mgmtd/src/grpc/delete_pool.rs b/mgmtd/src/grpc/delete_pool.rs index 7810074..197f096 100644 --- a/mgmtd/src/grpc/delete_pool.rs +++ b/mgmtd/src/grpc/delete_pool.rs @@ -3,18 +3,17 @@ use shared::bee_msg::storage_pool::RefreshStoragePools; /// Deletes a pool. The pool must be empty. pub(crate) async fn delete_pool( - ctx: Context, + app: &impl App, req: pm::DeletePoolRequest, ) -> Result { - needs_license(&ctx, LicensedFeature::Storagepool)?; - fail_on_pre_shutdown(&ctx)?; + fail_on_missing_license(app, LicensedFeature::Storagepool)?; + fail_on_pre_shutdown(app)?; let pool: EntityId = required_field(req.pool)?.try_into()?; let execute: bool = required_field(req.execute)?; - let pool = ctx - .db - .conn(move |conn| { + let pool = app + .db_conn(move |conn| { let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; let pool = pool.resolve(&tx, EntityType::Pool)?; @@ -51,8 +50,7 @@ are still assigned to this pool" if execute { log::info!("Pool deleted: {pool}"); - notify_nodes( - &ctx, + app.beemsg_send_notifications( &[NodeType::Meta, NodeType::Storage], &RefreshStoragePools { ack_id: "".into() }, ) diff --git a/mgmtd/src/grpc/delete_target.rs b/mgmtd/src/grpc/delete_target.rs index ed0bc5c..dbbd5d5 100644 --- a/mgmtd/src/grpc/delete_target.rs +++ b/mgmtd/src/grpc/delete_target.rs @@ -3,17 +3,16 @@ use shared::bee_msg::misc::RefreshCapacityPools; /// Deletes a target pub(crate) async fn delete_target( - ctx: Context, + app: &impl App, req: pm::DeleteTargetRequest, ) -> Result { - fail_on_pre_shutdown(&ctx)?; + fail_on_pre_shutdown(app)?; let target: EntityId = required_field(req.target)?.try_into()?; let execute: bool = required_field(req.execute)?; - let target = ctx - .db - .conn(move |conn| { + let target = app + .db_conn(move |conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let target = target.resolve(&tx, EntityType::Target)?; @@ -47,8 +46,7 @@ pub(crate) async fn delete_target( if execute { log::info!("Target deleted: {target}"); - notify_nodes( - &ctx, + app.beemsg_send_notifications( &[NodeType::Meta], &RefreshCapacityPools { ack_id: "".into() }, ) diff --git a/mgmtd/src/grpc/get_buddy_groups.rs b/mgmtd/src/grpc/get_buddy_groups.rs index 65667b2..56264db 100644 --- a/mgmtd/src/grpc/get_buddy_groups.rs +++ b/mgmtd/src/grpc/get_buddy_groups.rs @@ -2,12 +2,11 @@ use super::*; /// Delivers the list of buddy groups pub(crate) async fn get_buddy_groups( - ctx: Context, + app: &impl App, _req: pm::GetBuddyGroupsRequest, ) -> Result { - let buddy_groups = ctx - .db - .read_tx(|tx| { + let buddy_groups = app + .db_read_tx(|tx| { Ok(tx.query_map_collect( sql!( "SELECT group_uid, group_id, bg.alias, bg.node_type, diff --git a/mgmtd/src/grpc/get_license.rs b/mgmtd/src/grpc/get_license.rs index 2343b9d..422d6d1 100644 --- a/mgmtd/src/grpc/get_license.rs +++ b/mgmtd/src/grpc/get_license.rs @@ -2,16 +2,15 @@ use super::*; use protobuf::management::{self as pm, GetLicenseResponse}; pub(crate) async fn get_license( - ctx: Context, + app: &impl App, req: pm::GetLicenseRequest, ) -> Result { let reload: bool = required_field(req.reload)?; if reload { - ctx.license - .load_and_verify_cert(&ctx.info.user_config.license_cert_file) + app.lic_load_and_verify_cert(&app.static_info().user_config.license_cert_file) .await?; } - let cert_data = ctx.license.get_cert_data()?; + let cert_data = app.lic_get_cert_data()?; Ok(GetLicenseResponse { cert_data: Some(cert_data), }) diff --git a/mgmtd/src/grpc/get_nodes.rs b/mgmtd/src/grpc/get_nodes.rs index 5877cf1..ad0559e 100644 --- a/mgmtd/src/grpc/get_nodes.rs +++ b/mgmtd/src/grpc/get_nodes.rs @@ -4,12 +4,11 @@ use std::str::FromStr; /// Delivers a list of nodes pub(crate) async fn get_nodes( - ctx: Context, + app: &impl App, req: pm::GetNodesRequest, ) -> Result { - let (mut nodes, nics, meta_root_node, meta_root_buddy_group, fs_uuid) = ctx - .db - .read_tx(move |tx| { + let (mut nodes, nics, meta_root_node, meta_root_buddy_group, fs_uuid) = app + .db_read_tx(move |tx| { // Fetching the nic list is optional as it causes additional load let nics: Vec<(Uid, pm::get_nodes_response::node::Nic)> = if req.include_nics { tx.prepare_cached(sql!( diff --git a/mgmtd/src/grpc/get_pools.rs b/mgmtd/src/grpc/get_pools.rs index 52f8df3..de2d69c 100644 --- a/mgmtd/src/grpc/get_pools.rs +++ b/mgmtd/src/grpc/get_pools.rs @@ -2,12 +2,11 @@ use super::*; /// Delivers the list of pools pub(crate) async fn get_pools( - ctx: Context, + app: &impl App, req: pm::GetPoolsRequest, ) -> Result { - let (mut pools, targets, buddy_groups) = ctx - .db - .read_tx(move |tx| { + let (mut pools, targets, buddy_groups) = app + .db_read_tx(move |tx| { let make_sp = |row: &Row| -> rusqlite::Result { Ok(pm::get_pools_response::StoragePool { id: Some(pb::EntityIdSet { diff --git a/mgmtd/src/grpc/get_quota_limits.rs b/mgmtd/src/grpc/get_quota_limits.rs index 9219301..7026f88 100644 --- a/mgmtd/src/grpc/get_quota_limits.rs +++ b/mgmtd/src/grpc/get_quota_limits.rs @@ -4,20 +4,19 @@ use itertools::Itertools; use std::fmt::Write; pub(crate) async fn get_quota_limits( - ctx: Context, + app: &impl App, req: pm::GetQuotaLimitsRequest, ) -> Result> { - needs_license(&ctx, LicensedFeature::Quota)?; + fail_on_missing_license(app, LicensedFeature::Quota)?; - if !ctx.info.user_config.quota_enable { + if !app.static_info().user_config.quota_enable { bail!(QUOTA_NOT_ENABLED_STR); } let pool_id = if let Some(pool) = req.pool { let pool: EntityId = pool.try_into()?; - let pool_id = ctx - .db - .read_tx(move |tx| pool.resolve(tx, EntityType::Pool)) + let pool_id = app + .db_read_tx(move |tx| pool.resolve(tx, EntityType::Pool)) .await? .num_id(); Some(pool_id) @@ -80,14 +79,15 @@ pub(crate) async fn get_quota_limits( inode = QuotaType::Inode.sql_variant() ); + let app = app.clone(); let stream = resp_stream(QUOTA_STREAM_BUF_SIZE, async move |stream| { let mut offset = 0; loop { let sql = sql.clone(); - let entries: Vec<_> = ctx - .db - .read_tx(move |tx| { + let entries: Vec<_> = app + .clone() + .db_read_tx(move |tx| { tx.query_map_collect(&sql, [offset, QUOTA_STREAM_PAGE_LIMIT], |row| { Ok(pm::QuotaInfo { pool: Some(pb::EntityIdSet { diff --git a/mgmtd/src/grpc/get_quota_usage.rs b/mgmtd/src/grpc/get_quota_usage.rs index cdb05f3..738931e 100644 --- a/mgmtd/src/grpc/get_quota_usage.rs +++ b/mgmtd/src/grpc/get_quota_usage.rs @@ -4,12 +4,12 @@ use itertools::Itertools; use std::fmt::Write; pub(crate) async fn get_quota_usage( - ctx: Context, + app: &impl App, req: pm::GetQuotaUsageRequest, ) -> Result> { - needs_license(&ctx, LicensedFeature::Quota)?; + fail_on_missing_license(app, LicensedFeature::Quota)?; - if !ctx.info.user_config.quota_enable { + if !app.static_info().user_config.quota_enable { bail!(QUOTA_NOT_ENABLED_STR); } @@ -56,9 +56,8 @@ pub(crate) async fn get_quota_usage( if let Some(pool) = req.pool { let pool: EntityId = pool.try_into()?; - let pool_uid = ctx - .db - .read_tx(move |tx| pool.resolve(tx, EntityType::Pool)) + let pool_uid = app + .db_read_tx(move |tx| pool.resolve(tx, EntityType::Pool)) .await? .uid; @@ -97,14 +96,14 @@ pub(crate) async fn get_quota_usage( inode = QuotaType::Inode.sql_variant() ); + let app = app.clone(); let stream = resp_stream(QUOTA_STREAM_BUF_SIZE, async move |stream| { let mut offset = 0; loop { let sql = sql.clone(); - let entries: Vec<_> = ctx - .db - .read_tx(move |tx| { + let entries: Vec<_> = app + .db_read_tx(move |tx| { tx.query_map_collect(&sql, [offset, QUOTA_STREAM_PAGE_LIMIT], |row| { Ok(pm::QuotaInfo { pool: Some(pb::EntityIdSet { @@ -138,7 +137,10 @@ pub(crate) async fn get_quota_usage( .send(pm::GetQuotaUsageResponse { entry: Some(entry), refresh_period_s: Some( - ctx.info.user_config.quota_update_interval.as_secs(), + app.static_info() + .user_config + .quota_update_interval + .as_secs(), ), }) .await?; diff --git a/mgmtd/src/grpc/get_targets.rs b/mgmtd/src/grpc/get_targets.rs index af9985e..31fda48 100644 --- a/mgmtd/src/grpc/get_targets.rs +++ b/mgmtd/src/grpc/get_targets.rs @@ -14,101 +14,105 @@ impl CapacityInfo for &pm::get_targets_response::Target { /// Delivers the list of targets pub(crate) async fn get_targets( - ctx: Context, + app: &impl App, _req: pm::GetTargetsRequest, ) -> Result { - let node_offline_timeout = ctx.info.user_config.node_offline_timeout; - let pre_shutdown = ctx.run_state.pre_shutdown(); + let node_offline_timeout = app.static_info().user_config.node_offline_timeout; + let pre_shutdown = app.rs_pre_shutdown(); - let targets_q = sql!( - "SELECT t.target_uid, t.alias, t.target_id, t.node_type, - n.node_uid, n.alias, n.node_id, - p.pool_uid, p.alias, p.pool_id, - t.consistency, (UNIXEPOCH('now') - UNIXEPOCH(t.last_update)), - t.free_space, t.free_inodes, t.total_space, t.total_inodes, - gp.p_target_id, gs.s_target_id - FROM targets_ext AS t - INNER JOIN nodes_ext AS n USING(node_uid) - LEFT JOIN pools_ext AS p USING(node_type, pool_id) - LEFT JOIN buddy_groups AS gp ON gp.p_target_id = t.target_id AND gp.node_type = t.node_type - LEFT JOIN buddy_groups AS gs ON gs.s_target_id = t.target_id AND gs.node_type = t.node_type" - ); + let fetch_op = move |tx: &Transaction| { + let targets_q = sql!( + "SELECT t.target_uid, t.alias, t.target_id, t.node_type, + n.node_uid, n.alias, n.node_id, + p.pool_uid, p.alias, p.pool_id, + t.consistency, (UNIXEPOCH('now') - UNIXEPOCH(t.last_update)), + t.free_space, t.free_inodes, t.total_space, t.total_inodes, + gp.p_target_id, gs.s_target_id + FROM targets_ext AS t + INNER JOIN nodes_ext AS n USING(node_uid) + LEFT JOIN pools_ext AS p USING(node_type, pool_id) + LEFT JOIN buddy_groups AS gp ON gp.p_target_id = t.target_id + AND gp.node_type = t.node_type + LEFT JOIN buddy_groups AS gs ON gs.s_target_id = t.target_id + AND gs.node_type = t.node_type" + ); - let targets_f = move |row: &rusqlite::Row| { - let node_type = NodeType::from_row(row, 3)?.into_proto_i32(); - let age = Duration::from_secs(row.get(11)?); - let is_primary = row.get::<_, Option>(16)?.is_some(); - let is_secondary = row.get::<_, Option>(17)?.is_some(); + let targets_f = move |row: &rusqlite::Row| { + let node_type = NodeType::from_row(row, 3)?.into_proto_i32(); + let age = Duration::from_secs(row.get(11)?); + let is_primary = row.get::<_, Option>(16)?.is_some(); + let is_secondary = row.get::<_, Option>(17)?.is_some(); - Ok(pm::get_targets_response::Target { - id: Some(pb::EntityIdSet { - uid: row.get(0)?, - legacy_id: Some(pb::LegacyId { - num_id: row.get(2)?, - node_type, - }), - alias: row.get(1)?, - }), - node_type, - node: Some(pb::EntityIdSet { - uid: row.get(4)?, - legacy_id: Some(pb::LegacyId { - num_id: row.get(6)?, - node_type, + Ok(pm::get_targets_response::Target { + id: Some(pb::EntityIdSet { + uid: row.get(0)?, + legacy_id: Some(pb::LegacyId { + num_id: row.get(2)?, + node_type, + }), + alias: row.get(1)?, }), - alias: row.get(5)?, - }), - storage_pool: if let Some(uid) = row.get::<_, Option>(7)? { - Some(pb::EntityIdSet { - uid: Some(uid), + node_type, + node: Some(pb::EntityIdSet { + uid: row.get(4)?, legacy_id: Some(pb::LegacyId { - num_id: row.get(9)?, + num_id: row.get(6)?, node_type, }), - alias: row.get(8)?, - }) - } else { - None - }, + alias: row.get(5)?, + }), + storage_pool: if let Some(uid) = row.get::<_, Option>(7)? { + Some(pb::EntityIdSet { + uid: Some(uid), + legacy_id: Some(pb::LegacyId { + num_id: row.get(9)?, + node_type, + }), + alias: row.get(8)?, + }) + } else { + None + }, - reachability_state: if !pre_shutdown || is_secondary { - if !is_primary && age > node_offline_timeout { - pb::ReachabilityState::Offline - } else if age > node_offline_timeout / 2 { - pb::ReachabilityState::Poffline + reachability_state: if !pre_shutdown || is_secondary { + if !is_primary && age > node_offline_timeout { + pb::ReachabilityState::Offline + } else if age > node_offline_timeout / 2 { + pb::ReachabilityState::Poffline + } else { + pb::ReachabilityState::Online + } + .into() } else { - pb::ReachabilityState::Online - } - .into() - } else { - pb::ReachabilityState::Poffline.into() - }, - consistency_state: TargetConsistencyState::from_row(row, 10)?.into_proto_i32(), - last_contact_s: age.as_secs().into(), - free_space_bytes: row.get(12)?, - free_inodes: row.get(13)?, - cap_pool: pb::CapacityPool::Unspecified.into(), - total_space_bytes: row.get(14)?, - total_inodes: row.get(15)?, - }) - }; + pb::ReachabilityState::Poffline.into() + }, + consistency_state: TargetConsistencyState::from_row(row, 10)?.into_proto_i32(), + last_contact_s: row.get(11)?, + free_space_bytes: row.get(12)?, + free_inodes: row.get(13)?, + cap_pool: pb::CapacityPool::Unspecified.into(), + total_space_bytes: row.get(14)?, + total_inodes: row.get(15)?, + }) + }; - let pools_q = sql!("SELECT pool_uid FROM storage_pools"); + let pools_q = sql!("SELECT pool_uid FROM storage_pools"); - let (mut targets, pools): (Vec, Vec) = ctx - .db - .read_tx(move |tx| { - Ok(( - tx.query_map_collect(targets_q, [], targets_f)?, - tx.query_map_collect(pools_q, [], |row| row.get(0))?, - )) - }) - .await - .status_code(Code::Internal)?; + Ok(( + tx.query_map_collect(targets_q, [], targets_f)?, + tx.query_map_collect(pools_q, [], |row| row.get(0))?, + )) + }; + + let (mut targets, pools): (Vec, Vec) = + app.db_read_tx(fetch_op).await.status_code(Code::Internal)?; let cap_pool_meta_calc = CapPoolCalculator::new( - ctx.info.user_config.cap_pool_meta_limits.clone(), - ctx.info.user_config.cap_pool_dynamic_meta_limits.as_ref(), + app.static_info().user_config.cap_pool_meta_limits.clone(), + app.static_info() + .user_config + .cap_pool_dynamic_meta_limits + .as_ref(), targets .iter() .filter(|t| t.node_type() == pb::NodeType::Meta), @@ -128,8 +132,11 @@ pub(crate) async fn get_targets( for sp_uid in pools { let cap_pool_storage_calc = CapPoolCalculator::new( - ctx.info.user_config.cap_pool_storage_limits.clone(), - ctx.info + app.static_info() + .user_config + .cap_pool_storage_limits + .clone(), + app.static_info() .user_config .cap_pool_dynamic_storage_limits .as_ref(), diff --git a/mgmtd/src/grpc/mirror_root_inode.rs b/mgmtd/src/grpc/mirror_root_inode.rs index cc02558..d52f358 100644 --- a/mgmtd/src/grpc/mirror_root_inode.rs +++ b/mgmtd/src/grpc/mirror_root_inode.rs @@ -5,16 +5,15 @@ use shared::bee_msg::buddy_group::{SetMetadataMirroring, SetMetadataMirroringRes /// Enable metadata mirroring for the root directory pub(crate) async fn mirror_root_inode( - ctx: Context, + app: &impl App, _req: pm::MirrorRootInodeRequest, ) -> Result { - needs_license(&ctx, LicensedFeature::Mirroring)?; - fail_on_pre_shutdown(&ctx)?; + fail_on_missing_license(app, LicensedFeature::Mirroring)?; + fail_on_pre_shutdown(app)?; - let offline_timeout = ctx.info.user_config.node_offline_timeout.as_secs(); - let meta_root = ctx - .db - .read_tx(move |tx| { + let offline_timeout = app.static_info().user_config.node_offline_timeout.as_secs(); + let meta_root = app + .db_read_tx(move |tx| { let node_uid = match db::misc::get_meta_root(tx)? { MetaRoot::Normal(_, node_uid) => node_uid, MetaRoot::Mirrored(_) => bail!("Root inode is already mirrored"), @@ -77,13 +76,12 @@ communicated during the last {offline_timeout}s." }) .await?; - let resp: SetMetadataMirroringResp = ctx - .conn - .request(meta_root, &SetMetadataMirroring {}) + let resp: SetMetadataMirroringResp = app + .beemsg_request(meta_root, &SetMetadataMirroring {}) .await?; match resp.result { - OpsErr::SUCCESS => ctx.db.write_tx(db::misc::enable_metadata_mirroring).await?, + OpsErr::SUCCESS => app.db_write_tx(db::misc::enable_metadata_mirroring).await?, _ => bail!( "The root meta server failed to mirror the root inode: {:?}", resp.result diff --git a/mgmtd/src/grpc/set_alias.rs b/mgmtd/src/grpc/set_alias.rs index 2a07c67..9b8a242 100644 --- a/mgmtd/src/grpc/set_alias.rs +++ b/mgmtd/src/grpc/set_alias.rs @@ -4,10 +4,10 @@ use shared::bee_msg::node::Heartbeat; /// Sets the entity alias for any entity pub(crate) async fn set_alias( - ctx: Context, + app: &impl App, req: pm::SetAliasRequest, ) -> Result { - fail_on_pre_shutdown(&ctx)?; + fail_on_pre_shutdown(app)?; // Parse proto msg let entity_type: EntityType = req.entity_type().try_into()?; @@ -48,9 +48,8 @@ pub(crate) async fn set_alias( // If the entity is a node, notify all nodes about the changed alias if entity_type == EntityType::Node { - let (entity, node, nic_list) = ctx - .db - .write_tx(move |tx| { + let (entity, node, nic_list) = app + .db_write_tx(move |tx| { let entity = update_alias_fn(tx, &new_alias)?; let node = db::node::get_by_alias(tx, new_alias.as_ref())?; @@ -60,8 +59,7 @@ pub(crate) async fn set_alias( }) .await?; - notify_nodes( - &ctx, + app.beemsg_send_notifications( &[NodeType::Meta, NodeType::Storage, NodeType::Client], &Heartbeat { instance_version: 0, @@ -82,8 +80,7 @@ pub(crate) async fn set_alias( // If not a node, just update the alias } else { - ctx.db - .write_tx(move |tx| update_alias_fn(tx, &new_alias)) + app.db_write_tx(move |tx| update_alias_fn(tx, &new_alias)) .await?; } diff --git a/mgmtd/src/grpc/set_default_quota_limits.rs b/mgmtd/src/grpc/set_default_quota_limits.rs index 9da61d9..8d74c2f 100644 --- a/mgmtd/src/grpc/set_default_quota_limits.rs +++ b/mgmtd/src/grpc/set_default_quota_limits.rs @@ -3,13 +3,13 @@ use super::*; use std::cmp::Ordering; pub(crate) async fn set_default_quota_limits( - ctx: Context, + app: &impl App, req: pm::SetDefaultQuotaLimitsRequest, ) -> Result { - needs_license(&ctx, LicensedFeature::Quota)?; - fail_on_pre_shutdown(&ctx)?; + fail_on_missing_license(app, LicensedFeature::Quota)?; + fail_on_pre_shutdown(app)?; - if !ctx.info.user_config.quota_enable { + if !app.static_info().user_config.quota_enable { bail!(QUOTA_NOT_ENABLED_STR); } @@ -52,27 +52,26 @@ pub(crate) async fn set_default_quota_limits( Ok(()) } - ctx.db - .write_tx(move |tx| { - let pool = pool.resolve(tx, EntityType::Pool)?; - let pool_id: PoolId = pool.num_id().try_into()?; + app.db_write_tx(move |tx| { + let pool = pool.resolve(tx, EntityType::Pool)?; + let pool_id: PoolId = pool.num_id().try_into()?; - if let Some(l) = req.user_space_limit { - update(tx, l, pool_id, QuotaIdType::User, QuotaType::Space)?; - } - if let Some(l) = req.user_inode_limit { - update(tx, l, pool_id, QuotaIdType::User, QuotaType::Inode)?; - } - if let Some(l) = req.group_space_limit { - update(tx, l, pool_id, QuotaIdType::Group, QuotaType::Space)?; - } - if let Some(l) = req.group_inode_limit { - update(tx, l, pool_id, QuotaIdType::Group, QuotaType::Inode)?; - } + if let Some(l) = req.user_space_limit { + update(tx, l, pool_id, QuotaIdType::User, QuotaType::Space)?; + } + if let Some(l) = req.user_inode_limit { + update(tx, l, pool_id, QuotaIdType::User, QuotaType::Inode)?; + } + if let Some(l) = req.group_space_limit { + update(tx, l, pool_id, QuotaIdType::Group, QuotaType::Space)?; + } + if let Some(l) = req.group_inode_limit { + update(tx, l, pool_id, QuotaIdType::Group, QuotaType::Inode)?; + } - Ok(()) - }) - .await?; + Ok(()) + }) + .await?; Ok(pm::SetDefaultQuotaLimitsResponse {}) } diff --git a/mgmtd/src/grpc/set_quota_limits.rs b/mgmtd/src/grpc/set_quota_limits.rs index 1bb35d9..d8a9a21 100644 --- a/mgmtd/src/grpc/set_quota_limits.rs +++ b/mgmtd/src/grpc/set_quota_limits.rs @@ -2,78 +2,77 @@ use super::common::QUOTA_NOT_ENABLED_STR; use super::*; pub(crate) async fn set_quota_limits( - ctx: Context, + app: &impl App, req: pm::SetQuotaLimitsRequest, ) -> Result { - needs_license(&ctx, LicensedFeature::Quota)?; - fail_on_pre_shutdown(&ctx)?; + fail_on_missing_license(app, LicensedFeature::Quota)?; + fail_on_pre_shutdown(app)?; - if !ctx.info.user_config.quota_enable { + if !app.static_info().user_config.quota_enable { bail!(QUOTA_NOT_ENABLED_STR); } - ctx.db - .write_tx(|tx| { - let mut insert_stmt = tx.prepare_cached(sql!( - "REPLACE INTO quota_limits + app.db_write_tx(|tx| { + let mut insert_stmt = tx.prepare_cached(sql!( + "REPLACE INTO quota_limits (quota_id, id_type, quota_type, pool_id, value) VALUES (?1, ?2, ?3, ?4, ?5)" - ))?; + ))?; - let mut delete_stmt = tx.prepare_cached(sql!( - "DELETE FROM quota_limits + let mut delete_stmt = tx.prepare_cached(sql!( + "DELETE FROM quota_limits WHERE quota_id = ?1 AND id_type = ?2 AND quota_type = ?3 AND pool_id = ?4" - ))?; + ))?; - for lim in req.limits { - let id_type: QuotaIdType = lim.id_type().try_into()?; - let quota_id = required_field(lim.quota_id)?; + for lim in req.limits { + let id_type: QuotaIdType = lim.id_type().try_into()?; + let quota_id = required_field(lim.quota_id)?; - let pool: EntityId = required_field(lim.pool)?.try_into()?; - let pool_id = pool.resolve(tx, EntityType::Pool)?.num_id(); + let pool: EntityId = required_field(lim.pool)?.try_into()?; + let pool_id = pool.resolve(tx, EntityType::Pool)?.num_id(); - if let Some(l) = lim.space_limit { - if l > -1 { - insert_stmt.execute(params![ - quota_id, - id_type.sql_variant(), - QuotaType::Space.sql_variant(), - pool_id, - l - ])? - } else { - delete_stmt.execute(params![ - quota_id, - id_type.sql_variant(), - QuotaType::Space.sql_variant(), - pool_id, - ])? - }; - } + if let Some(l) = lim.space_limit { + if l > -1 { + insert_stmt.execute(params![ + quota_id, + id_type.sql_variant(), + QuotaType::Space.sql_variant(), + pool_id, + l + ])? + } else { + delete_stmt.execute(params![ + quota_id, + id_type.sql_variant(), + QuotaType::Space.sql_variant(), + pool_id, + ])? + }; + } - if let Some(l) = lim.inode_limit { - if l > -1 { - insert_stmt.execute(params![ - quota_id, - id_type.sql_variant(), - QuotaType::Inode.sql_variant(), - pool_id, - l - ])? - } else { - delete_stmt.execute(params![ - quota_id, - id_type.sql_variant(), - QuotaType::Inode.sql_variant(), - pool_id, - ])? - }; - } + if let Some(l) = lim.inode_limit { + if l > -1 { + insert_stmt.execute(params![ + quota_id, + id_type.sql_variant(), + QuotaType::Inode.sql_variant(), + pool_id, + l + ])? + } else { + delete_stmt.execute(params![ + quota_id, + id_type.sql_variant(), + QuotaType::Inode.sql_variant(), + pool_id, + ])? + }; } + } - Ok(()) - }) - .await?; + Ok(()) + }) + .await?; Ok(pm::SetQuotaLimitsResponse {}) } diff --git a/mgmtd/src/grpc/set_target_state.rs b/mgmtd/src/grpc/set_target_state.rs index bf98e95..a621d99 100644 --- a/mgmtd/src/grpc/set_target_state.rs +++ b/mgmtd/src/grpc/set_target_state.rs @@ -6,17 +6,16 @@ use shared::bee_msg::target::{ /// Set consistency state for a target pub(crate) async fn set_target_state( - ctx: Context, + app: &impl App, req: pm::SetTargetStateRequest, ) -> Result { - fail_on_pre_shutdown(&ctx)?; + fail_on_pre_shutdown(app)?; let state: TargetConsistencyState = req.consistency_state().try_into()?; let target: EntityId = required_field(req.target)?.try_into()?; - let (target, node_uid) = ctx - .db - .write_tx(move |tx| { + let (target, node_uid) = app + .db_write_tx(move |tx| { let target = target.resolve(tx, EntityType::Target)?; let node: i64 = tx.query_row_cached( @@ -35,9 +34,8 @@ pub(crate) async fn set_target_state( }) .await?; - let resp: SetTargetConsistencyStatesResp = ctx - .conn - .request( + let resp: SetTargetConsistencyStatesResp = app + .beemsg_request( node_uid, &SetTargetConsistencyStates { node_type: target.node_type(), @@ -55,8 +53,7 @@ pub(crate) async fn set_target_state( ); } - notify_nodes( - &ctx, + app.beemsg_send_notifications( &[NodeType::Meta, NodeType::Storage, NodeType::Client], &RefreshTargetStates { ack_id: "".into() }, ) diff --git a/mgmtd/src/grpc/start_resync.rs b/mgmtd/src/grpc/start_resync.rs index ded2407..284a6be 100644 --- a/mgmtd/src/grpc/start_resync.rs +++ b/mgmtd/src/grpc/start_resync.rs @@ -10,20 +10,19 @@ use tokio::time::sleep; /// Starts a resync of a storage or metadata target from its buddy target pub(crate) async fn start_resync( - ctx: Context, + app: &impl App, req: pm::StartResyncRequest, ) -> Result { - needs_license(&ctx, LicensedFeature::Mirroring)?; - fail_on_pre_shutdown(&ctx)?; + fail_on_missing_license(app, LicensedFeature::Mirroring)?; + fail_on_pre_shutdown(app)?; let buddy_group: EntityId = required_field(req.buddy_group)?.try_into()?; let timestamp: i64 = required_field(req.timestamp)?; let restart: bool = required_field(req.restart)?; // For resync source is always primary target and destination is secondary target - let (src_target_id, dest_target_id, src_node_uid, node_type, group) = ctx - .db - .read_tx(move |tx| { + let (src_target_id, dest_target_id, src_node_uid, node_type, group) = app + .db_read_tx(move |tx| { let group = buddy_group.resolve(tx, EntityType::BuddyGroup)?; let node_type: NodeTypeServer = group.node_type().try_into()?; @@ -64,9 +63,8 @@ not supported." bail!("Resync cannot be restarted or aborted for metadata servers."); } - let resp: GetMetaResyncStatsResp = ctx - .conn - .request( + let resp: GetMetaResyncStatsResp = app + .beemsg_request( src_node_uid, &GetMetaResyncStats { target_id: src_target_id, @@ -80,9 +78,8 @@ not supported." } NodeTypeServer::Storage => { if !restart { - let resp: GetStorageResyncStatsResp = ctx - .conn - .request( + let resp: GetStorageResyncStatsResp = app + .beemsg_request( src_node_uid, &GetStorageResyncStats { target_id: src_target_id, @@ -95,7 +92,7 @@ not supported." } if timestamp > -1 { - override_last_buddy_comm(&ctx, src_node_uid, src_target_id, &group, timestamp) + override_last_buddy_comm(app, src_node_uid, src_target_id, &group, timestamp) .await?; } } else { @@ -103,7 +100,7 @@ not supported." bail!("Resync for storage targets can only be restarted with timestamp."); } - override_last_buddy_comm(&ctx, src_node_uid, src_target_id, &group, timestamp) + override_last_buddy_comm(app, src_node_uid, src_target_id, &group, timestamp) .await?; log::info!("Waiting for the already running resync operations to abort."); @@ -116,9 +113,8 @@ not supported." // tells us the resync is finished, but that is a bit more complex and, with the // current system, still unreliable. loop { - let resp: GetStorageResyncStatsResp = ctx - .conn - .request( + let resp: GetStorageResyncStatsResp = app + .beemsg_request( src_node_uid, &GetStorageResyncStats { target_id: src_target_id, @@ -141,16 +137,15 @@ not supported." } // set destination target state as needs-resync in mgmtd database - ctx.db - .write_tx(move |tx| { - db::target::update_consistency_states( - tx, - [(dest_target_id, TargetConsistencyState::NeedsResync)], - node_type, - )?; - Ok(()) - }) - .await?; + app.db_write_tx(move |tx| { + db::target::update_consistency_states( + tx, + [(dest_target_id, TargetConsistencyState::NeedsResync)], + node_type, + )?; + Ok(()) + }) + .await?; // This also triggers the source node to fetch the new needs resync state and start the resync // using the internode syncer loop. In case of overriding last buddy communication on storage @@ -160,8 +155,7 @@ not supported." // // Note that sending a SetTargetConsistencyStateMsg does have no effect on making this quicker, // so we omit it. - notify_nodes( - &ctx, + app.beemsg_send_notifications( &[NodeType::Meta, NodeType::Storage, NodeType::Client], &RefreshTargetStates { ack_id: "".into() }, ) @@ -172,15 +166,14 @@ not supported." /// Override last buddy communication timestamp on source storage node /// Note that this might be overwritten again on the storage server between async fn override_last_buddy_comm( - ctx: &Context, + app: &impl App, src_node_uid: Uid, src_target_id: TargetId, group: &EntityIdSet, timestamp: i64, ) -> Result<()> { - let resp: SetTargetConsistencyStatesResp = ctx - .conn - .request( + let resp: SetTargetConsistencyStatesResp = app + .beemsg_request( src_node_uid, &SetLastBuddyCommOverride { target_id: src_target_id, diff --git a/mgmtd/src/lib.rs b/mgmtd/src/lib.rs index e4aaf19..cd78937 100644 --- a/mgmtd/src/lib.rs +++ b/mgmtd/src/lib.rs @@ -1,9 +1,9 @@ //! The BeeGFS management service +mod app; mod bee_msg; mod cap_pool; pub mod config; -mod context; pub mod db; mod error; mod grpc; @@ -12,10 +12,10 @@ mod quota; mod timer; mod types; +use crate::app::RuntimeApp; use crate::config::Config; -use crate::context::Context; use anyhow::Result; -use bee_msg::notify_nodes; +use app::App; use db::node_nic::ReplaceNic; use license::LicenseVerifier; use shared::bee_msg::target::RefreshTargetStates; @@ -117,7 +117,7 @@ pub async fn start(info: StaticInfo, license: LicenseVerifier) -> Result Result Result, } @@ -181,7 +181,7 @@ impl RunControl { self.run_state_control.pre_shutdown(); let client_list: HashSet = self - .ctx + .app .db .read_tx(move |tx| { let buddy_groups: i64 = @@ -217,18 +217,18 @@ impl RunControl { log::warn!( "Buddy groups are in use and clients are registered - \ waiting for all clients to pull state (timeout after {:?}) ...", - self.ctx.info.user_config.node_offline_timeout + self.app.info.user_config.node_offline_timeout ); // Let the nodes pull the new states as soon as possible - notify_nodes( - &self.ctx, - &[NodeType::Client, NodeType::Meta, NodeType::Storage], - &RefreshTargetStates { - ack_id: b"".to_vec(), - }, - ) - .await; + self.app + .beemsg_send_notifications( + &[NodeType::Client, NodeType::Meta, NodeType::Storage], + &RefreshTargetStates { + ack_id: b"".to_vec(), + }, + ) + .await; tokio::select! { // Wait for all clients having downloaded the state @@ -254,7 +254,7 @@ impl RunControl { /// Waits until every client in `client_list` has been received to the `self.shutdown_client_` async fn wait_for_clients(&mut self, mut client_list: HashSet) { - let deadline = Instant::now() + self.ctx.info.user_config.node_offline_timeout; + let deadline = Instant::now() + self.app.info.user_config.node_offline_timeout; let receive_client_ids = async { while let Some(client_id) = self.shutdown_client_rx.recv().await { diff --git a/mgmtd/src/quota.rs b/mgmtd/src/quota.rs index 3285289..d56fc41 100644 --- a/mgmtd/src/quota.rs +++ b/mgmtd/src/quota.rs @@ -1,6 +1,6 @@ //! Functionality for fetching and updating quota information from / to nodes and the database. -use crate::context::Context; +use crate::app::*; use crate::db; use crate::db::quota_usage::QuotaData; use crate::types::SqliteEnumExt; @@ -16,12 +16,11 @@ use std::collections::HashSet; use std::path::Path; /// Fetches quota information for all storage targets, calculates exceeded IDs and distributes them. -pub(crate) async fn update_and_distribute(ctx: &Context) -> Result<()> { +pub(crate) async fn update_and_distribute(app: &impl App) -> Result<()> { // Fetch quota data from storage daemons - let targets: Vec<(TargetId, PoolId, Uid)> = ctx - .db - .read_tx(move |tx| { + let targets: Vec<(TargetId, PoolId, Uid)> = app + .db_read_tx(move |tx| { tx.query_map_collect( sql!( "SELECT target_id, pool_id, node_uid @@ -49,7 +48,7 @@ pub(crate) async fn update_and_distribute(ctx: &Context) -> Result<()> { let (mut user_ids, mut group_ids) = (HashSet::new(), HashSet::new()); // If configured, add system User IDS - let user_ids_min = ctx.info.user_config.quota_user_system_ids_min; + let user_ids_min = app.static_info().user_config.quota_user_system_ids_min; if let Some(user_ids_min) = user_ids_min { system_ids::user_ids() @@ -61,7 +60,7 @@ pub(crate) async fn update_and_distribute(ctx: &Context) -> Result<()> { } // If configured, add system Group IDS - let group_ids_min = ctx.info.user_config.quota_group_system_ids_min; + let group_ids_min = app.static_info().user_config.quota_group_system_ids_min; if let Some(group_ids_min) = group_ids_min { system_ids::group_ids() @@ -73,22 +72,22 @@ pub(crate) async fn update_and_distribute(ctx: &Context) -> Result<()> { } // If configured, add user IDs from file - if let Some(ref path) = ctx.info.user_config.quota_user_ids_file { + if let Some(ref path) = app.static_info().user_config.quota_user_ids_file { try_read_quota_ids(path, &mut user_ids)?; } // If configured, add group IDs from file - if let Some(ref path) = ctx.info.user_config.quota_group_ids_file { + if let Some(ref path) = app.static_info().user_config.quota_group_ids_file { try_read_quota_ids(path, &mut group_ids)?; } // If configured, add range based user IDs - if let Some(range) = &ctx.info.user_config.quota_user_ids_range { + if let Some(range) = &app.static_info().user_config.quota_user_ids_range { user_ids.extend(range.clone()); } // If configured, add range based group IDs - if let Some(range) = &ctx.info.user_config.quota_group_ids_range { + if let Some(range) = &app.static_info().user_config.quota_group_ids_range { group_ids.extend(range.clone()); } @@ -96,14 +95,13 @@ pub(crate) async fn update_and_distribute(ctx: &Context) -> Result<()> { // Sends one request per target to the respective owner node // Requesting is done concurrently. for (target_id, pool_id, node_uid) in targets { - let ctx2 = ctx.clone(); + let app2 = app.clone(); let user_ids2 = user_ids.clone(); // Users tasks.push(tokio::spawn(async move { - let resp: Result = ctx2 - .conn - .request( + let resp: Result = app2 + .beemsg_request( node_uid, &GetQuotaInfo::with_user_ids(user_ids2, target_id, pool_id), ) @@ -119,14 +117,13 @@ pub(crate) async fn update_and_distribute(ctx: &Context) -> Result<()> { (target_id, resp) })); - let ctx2 = ctx.clone(); + let app2 = app.clone(); let group_ids2 = group_ids.clone(); // Groups tasks.push(tokio::spawn(async move { - let resp = ctx2 - .conn - .request( + let resp = app2 + .beemsg_request( node_uid, &GetQuotaInfo::with_group_ids(group_ids2, target_id, pool_id), ) @@ -149,37 +146,35 @@ pub(crate) async fn update_and_distribute(ctx: &Context) -> Result<()> { let (target_id, resp) = t.await?; if let Ok(r) = resp { // Insert quota usage data into the database - ctx.db - .write_tx(move |tx| { - db::quota_usage::update( - tx, - target_id, - r.quota_entry.into_iter().map(|e| QuotaData { - quota_id: e.id, - id_type: e.id_type, - space: e.space, - inodes: e.inodes, - }), - ) - }) - .await?; + app.db_write_tx(move |tx| { + db::quota_usage::update( + tx, + target_id, + r.quota_entry.into_iter().map(|e| QuotaData { + quota_id: e.id, + id_type: e.id_type, + space: e.space, + inodes: e.inodes, + }), + ) + }) + .await?; } } - if ctx.info.user_config.quota_enforce { - exceeded_quota(ctx).await?; + if app.static_info().user_config.quota_enforce { + exceeded_quota(app).await?; } Ok(()) } /// Calculate and push exceeded quota info to the nodes -async fn exceeded_quota(ctx: &Context) -> Result<()> { +async fn exceeded_quota(app: &impl App) -> Result<()> { log::info!("Calculating and pushing exceeded quota"); - let (msges, nodes) = ctx - .db - .read_tx(|tx| { + let (msges, nodes) = app + .db_read_tx(|tx| { let pools: Vec<_> = tx.query_map_collect(sql!("SELECT pool_id FROM pools"), [], |row| row.get(0))?; @@ -235,10 +230,10 @@ async fn exceeded_quota(ctx: &Context) -> Result<()> { for msg in msges { let mut request_fails = 0; let mut non_success_count = 0; + for node_uid in &nodes { - match ctx - .conn - .request::<_, SetExceededQuotaResp>(*node_uid, &msg) + match app + .beemsg_request::<_, SetExceededQuotaResp>(*node_uid, &msg) .await { Ok(resp) => { diff --git a/mgmtd/src/timer.rs b/mgmtd/src/timer.rs index c9bb671..9dee08e 100644 --- a/mgmtd/src/timer.rs +++ b/mgmtd/src/timer.rs @@ -1,6 +1,7 @@ //! Contains timers executing periodic tasks. -use crate::context::Context; +use crate::App; +use crate::app::RuntimeApp; use crate::db::{self}; use crate::license::LicensedFeature; use crate::quota::update_and_distribute; @@ -10,27 +11,27 @@ use shared::types::NodeType; use tokio::time::{MissedTickBehavior, sleep}; /// Starts the timed tasks. -pub(crate) fn start_tasks(ctx: Context, run_state: RunStateHandle) { +pub(crate) fn start_tasks(app: RuntimeApp, run_state: RunStateHandle) { // TODO send out timer based RefreshTargetStates notification if a reachability // state changed ? - tokio::spawn(delete_stale_clients(ctx.clone(), run_state.clone())); - tokio::spawn(switchover(ctx.clone(), run_state.clone())); + tokio::spawn(delete_stale_clients(app.clone(), run_state.clone())); + tokio::spawn(switchover(app.clone(), run_state.clone())); - if ctx.info.user_config.quota_enable { - if let Err(err) = ctx.license.verify_feature(LicensedFeature::Quota) { + if app.info.user_config.quota_enable { + if let Err(err) = app.license.verify_feature(LicensedFeature::Quota) { log::error!( "Quota is enabled in the config, but the feature could not be verified. Continuing without quota support: {err}" ); } else { - tokio::spawn(update_quota(ctx, run_state)); + tokio::spawn(update_quota(app, run_state)); } } } /// Deletes client nodes from the database which haven't responded for the configured time. -async fn delete_stale_clients(ctx: Context, mut run_state: RunStateHandle) { - let timeout = ctx.info.user_config.client_auto_remove_timeout; +async fn delete_stale_clients(app: RuntimeApp, mut run_state: RunStateHandle) { + let timeout = app.info.user_config.client_auto_remove_timeout; loop { tokio::select! { @@ -40,7 +41,7 @@ async fn delete_stale_clients(ctx: Context, mut run_state: RunStateHandle) { log::debug!("Running stale client deleter"); - match ctx + match app .db .write_tx(move |tx| db::node::delete_stale_clients(tx, timeout)) .await @@ -58,17 +59,17 @@ async fn delete_stale_clients(ctx: Context, mut run_state: RunStateHandle) { } /// Fetches quota information for all storage targets, calculates exceeded IDs and distributes them. -async fn update_quota(ctx: Context, mut run_state: RunStateHandle) { +async fn update_quota(app: RuntimeApp, mut run_state: RunStateHandle) { loop { log::debug!("Running quota update"); - match update_and_distribute(&ctx).await { + match update_and_distribute(&app).await { Ok(_) => {} Err(err) => log::error!("Updating quota failed: {err:#}"), } tokio::select! { - _ = sleep(ctx.info.user_config.quota_update_interval) => {} + _ = sleep(app.info.user_config.quota_update_interval) => {} _ = run_state.wait_for_pre_shutdown() => { break; } } } @@ -77,7 +78,7 @@ async fn update_quota(ctx: Context, mut run_state: RunStateHandle) { } /// Finds buddy groups with switchover condition, swaps them and notifies nodes. -async fn switchover(ctx: Context, mut run_state: RunStateHandle) { +async fn switchover(app: RuntimeApp, mut run_state: RunStateHandle) { // On the other nodes / old management, the interval in which the switchover checks are done // is determined by "1/6 sysTargetOfflineTimeoutSecs". // This is also the interval the target states are being pushed to management. To avoid an @@ -85,7 +86,7 @@ async fn switchover(ctx: Context, mut run_state: RunStateHandle) { // up-and-running primary doesn't because of their timing, this value should be the same as on // the nodes. If we delay the initial check by that time, then a running primary has enough time // to report in and update the last contact time before the check happens. - let interval = ctx.info.user_config.node_offline_timeout / 6; + let interval = app.info.user_config.node_offline_timeout / 6; let mut timer = tokio::time::interval(interval); timer.set_missed_tick_behavior(MissedTickBehavior::Skip); @@ -100,9 +101,9 @@ async fn switchover(ctx: Context, mut run_state: RunStateHandle) { log::debug!("Running switchover check"); - let timeout = ctx.info.user_config.node_offline_timeout; + let timeout = app.info.user_config.node_offline_timeout; - match ctx + match app .db .write_tx(move |tx| db::buddy_group::check_and_swap_buddies(tx, timeout)) .await @@ -113,8 +114,7 @@ async fn switchover(ctx: Context, mut run_state: RunStateHandle) { "A switchover was triggered for the following buddy groups: {swapped:?}" ); - crate::bee_msg::notify_nodes( - &ctx, + app.beemsg_send_notifications( &[NodeType::Meta, NodeType::Storage, NodeType::Client], &RefreshTargetStates { ack_id: "".into() }, ) diff --git a/shared/src/grpc.rs b/shared/src/grpc.rs index 14fc31f..72365a9 100644 --- a/shared/src/grpc.rs +++ b/shared/src/grpc.rs @@ -50,7 +50,7 @@ macro_rules! impl_grpc_handler { // It assumes what is passed to the handler function and that might be different // for different users of this. // I don't have a quick idea how to fix this in an elegant way, so we keep it for. - let res = $impl_fn::$impl_fn(self.ctx.clone(), req.into_inner()).await; + let res = $impl_fn::$impl_fn(&self.app, req.into_inner()).await; match res { Ok(res) => Ok(Response::new(res)), From 1c3d6475a521e95e1baca8d101520a2d545b5f4f Mon Sep 17 00:00:00 2001 From: Rusty Bee <145002912+rustybee42@users.noreply.github.com> Date: Tue, 16 Sep 2025 14:41:45 +0200 Subject: [PATCH 8/9] test: add TestRequest --- shared/src/conn/msg_dispatch.rs | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/shared/src/conn/msg_dispatch.rs b/shared/src/conn/msg_dispatch.rs index 154f811..2932148 100644 --- a/shared/src/conn/msg_dispatch.rs +++ b/shared/src/conn/msg_dispatch.rs @@ -100,3 +100,45 @@ impl Request for SocketRequest<'_> { self.header.msg_id() } } + +pub mod test { + use super::*; + use std::net::{Ipv4Addr, SocketAddrV4}; + + pub struct TestRequest { + pub msg_id: MsgId, + pub authenticate_connection: bool, + } + + impl TestRequest { + pub fn new(msg_id: MsgId) -> Self { + Self { + msg_id, + authenticate_connection: false, + } + } + } + + impl Request for TestRequest { + async fn respond(self, _msg: &M) -> Result<()> { + // do nothing + Ok(()) + } + + fn authenticate_connection(&mut self) { + self.authenticate_connection = true; + } + + fn addr(&self) -> SocketAddr { + SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0).into() + } + + fn msg_id(&self) -> MsgId { + self.msg_id + } + + fn deserialize_msg(&self) -> Result { + unimplemented!() + } + } +} From f92dbfc7c2bed0b89dfaa63ace9d5ddadf0250b7 Mon Sep 17 00:00:00 2001 From: Rusty Bee <145002912+rustybee42@users.noreply.github.com> Date: Tue, 1 Apr 2025 15:32:12 +0200 Subject: [PATCH 9/9] test: add some handler tests * gRPC: get_nodes, delete_node, set_alias, get_pools * BeeMsg: get_nodes, authenticate_channel --- mgmtd/src/app/test.rs | 1 + mgmtd/src/bee_msg/authenticate_channel.rs | 21 ++++++ mgmtd/src/bee_msg/get_nodes.rs | 23 ++++++ mgmtd/src/grpc/delete_node.rs | 47 ++++++++++++ mgmtd/src/grpc/get_nodes.rs | 47 ++++++++++++ mgmtd/src/grpc/get_pools.rs | 48 ++++++++++++ mgmtd/src/grpc/set_alias.rs | 90 +++++++++++++++++++++++ 7 files changed, 277 insertions(+) diff --git a/mgmtd/src/app/test.rs b/mgmtd/src/app/test.rs index 1a63282..a25762d 100644 --- a/mgmtd/src/app/test.rs +++ b/mgmtd/src/app/test.rs @@ -1,6 +1,7 @@ use super::*; use crate::config::Config; use shared::bee_msg::MsgId; +pub use shared::conn::msg_dispatch::test::TestRequest; use shared::nic::Nic; use shared::types::{AuthSecret, NicType}; use sqlite::Connections; diff --git a/mgmtd/src/bee_msg/authenticate_channel.rs b/mgmtd/src/bee_msg/authenticate_channel.rs index 95693da..bfb053a 100644 --- a/mgmtd/src/bee_msg/authenticate_channel.rs +++ b/mgmtd/src/bee_msg/authenticate_channel.rs @@ -22,3 +22,24 @@ impl HandleNoResponse for AuthenticateChannel { Ok(()) } } + +#[cfg(test)] +mod test { + use super::*; + use crate::app::test::*; + + #[tokio::test] + async fn authenticate_channel() { + let app = TestApp::new().await; + let mut req = TestRequest::new(AuthenticateChannel::ID); + + AuthenticateChannel { + auth_secret: AuthSecret::hash_from_bytes("secret"), + } + .handle(&app, &mut req) + .await + .unwrap(); + + assert!(req.authenticate_connection); + } +} diff --git a/mgmtd/src/bee_msg/get_nodes.rs b/mgmtd/src/bee_msg/get_nodes.rs index da735f1..8f30ef3 100644 --- a/mgmtd/src/bee_msg/get_nodes.rs +++ b/mgmtd/src/bee_msg/get_nodes.rs @@ -56,3 +56,26 @@ impl HandleWithResponse for GetNodes { Ok(resp) } } + +#[cfg(test)] +mod test { + use super::*; + use crate::app::test::*; + + #[tokio::test] + async fn get_nodes() { + let app = TestApp::new().await; + let mut req = TestRequest::new(GetNodes::ID); + + let resp = GetNodes { + node_type: NodeType::Meta, + } + .handle(&app, &mut req) + .await + .unwrap(); + + assert_eq_db!(app, "SELECT COUNT(*) FROM meta_nodes", [], resp.nodes.len()); + assert_eq!(resp.root_num_id, 1); + assert_eq!(resp.is_root_mirrored, 0); + } +} diff --git a/mgmtd/src/grpc/delete_node.rs b/mgmtd/src/grpc/delete_node.rs index ae05d27..4b650f7 100644 --- a/mgmtd/src/grpc/delete_node.rs +++ b/mgmtd/src/grpc/delete_node.rs @@ -94,3 +94,50 @@ pub(crate) async fn delete_node( node: Some(node.into()), }) } +#[cfg(test)] +mod test { + use super::*; + use crate::app::test::*; + + #[tokio::test] + async fn delete_node() { + let h = TestApp::new().await; + let mut req = pm::DeleteNodeRequest { + node: Some(pb::EntityIdSet { + uid: None, + alias: None, + legacy_id: Some(pb::LegacyId { + num_id: 1, + node_type: pb::NodeType::Management.into(), + }), + }), + execute: Some(true), + }; + + // Can't delete management node + super::delete_node(&h, req.clone()).await.unwrap_err(); + + // Can't delete meta buddy group member target (which is on the node) + req.node.as_mut().unwrap().uid = None; + req.node.as_mut().unwrap().legacy_id = Some(pb::LegacyId { + num_id: 1, + node_type: pb::NodeType::Meta.into(), + }); + super::delete_node(&h, req.clone()).await.unwrap_err(); + + // Delete empty node + req.node.as_mut().unwrap().legacy_id = Some(pb::LegacyId { + num_id: 99, + node_type: pb::NodeType::Meta.into(), + }); + let resp = super::delete_node(&h, req.clone()).await.unwrap(); + + assert_eq!(resp.node.unwrap().legacy_id.unwrap().num_id, 99); + assert_eq_db!( + h, + "SELECT COUNT(*) FROM nodes WHERE node_id = ?1 AND node_type = ?2", + [99, NodeType::Meta.sql_variant()], + 0 + ); + } +} diff --git a/mgmtd/src/grpc/get_nodes.rs b/mgmtd/src/grpc/get_nodes.rs index ad0559e..aba44e0 100644 --- a/mgmtd/src/grpc/get_nodes.rs +++ b/mgmtd/src/grpc/get_nodes.rs @@ -159,3 +159,50 @@ pub(crate) async fn get_nodes( fs_uuid, }) } +#[cfg(test)] +mod test { + use super::*; + use crate::app::test::*; + + #[tokio::test] + async fn get_nodes() { + let app = TestApp::new().await; + + let res = super::get_nodes( + &app, + pm::GetNodesRequest { + include_nics: false, + }, + ) + .await + .unwrap(); + + assert_eq!(res.nodes.len(), 14); + assert!(res.nodes.iter().all(|e| e.nics.is_empty())); + + let res = super::get_nodes(&app, pm::GetNodesRequest { include_nics: true }) + .await + .unwrap(); + + assert_eq!(res.nodes.len(), 14); + assert_eq!( + res.nodes + .iter() + .find(|e| e.id.as_ref().unwrap().uid() == 101001) + .unwrap() + .nics + .len(), + 4 + ); + assert_eq!( + res.nodes + .iter() + .find(|e| e.id.as_ref().unwrap().uid() == 103004) + .unwrap() + .nics + .len(), + 2 + ); + assert_eq!(res.meta_root_node.unwrap().uid.unwrap(), 101001); + } +} diff --git a/mgmtd/src/grpc/get_pools.rs b/mgmtd/src/grpc/get_pools.rs index de2d69c..4668936 100644 --- a/mgmtd/src/grpc/get_pools.rs +++ b/mgmtd/src/grpc/get_pools.rs @@ -131,3 +131,51 @@ pub(crate) async fn get_pools( Ok(pm::GetPoolsResponse { pools }) } + +#[cfg(test)] +mod test { + use super::*; + use crate::app::test::*; + + #[tokio::test] + async fn get_pools() { + let app = TestApp::new().await; + + let resp = super::get_pools( + &app, + pm::GetPoolsRequest { + with_quota_limits: true, + }, + ) + .await + .unwrap(); + + assert_eq!(resp.pools.len(), 4); + + let default_pool = resp + .pools + .iter() + .find(|e| e.id.as_ref().unwrap().legacy_id.as_ref().unwrap().num_id == 1) + .unwrap(); + + assert_eq!(default_pool.targets.len(), 5); + assert_eq!(default_pool.buddy_groups.len(), 2); + assert_eq!(default_pool.user_space_limit.unwrap(), 1000); + assert_eq!(default_pool.user_inode_limit.unwrap(), 1000); + assert_eq!(default_pool.group_space_limit.unwrap(), 1000); + assert_eq!(default_pool.group_inode_limit.unwrap(), 1000); + + let other_pool = resp + .pools + .iter() + .find(|e| e.id.as_ref().unwrap().legacy_id.as_ref().unwrap().num_id == 2) + .unwrap(); + + assert_eq!(other_pool.targets.len(), 4); + assert_eq!(other_pool.buddy_groups.len(), 0); + assert_eq!(other_pool.user_space_limit.unwrap(), -1); + assert_eq!(other_pool.user_inode_limit.unwrap(), -1); + assert_eq!(other_pool.group_space_limit.unwrap(), -1); + assert_eq!(other_pool.group_inode_limit.unwrap(), -1); + } +} diff --git a/mgmtd/src/grpc/set_alias.rs b/mgmtd/src/grpc/set_alias.rs index 9b8a242..54078ac 100644 --- a/mgmtd/src/grpc/set_alias.rs +++ b/mgmtd/src/grpc/set_alias.rs @@ -86,3 +86,93 @@ pub(crate) async fn set_alias( Ok(pm::SetAliasResponse {}) } + +#[cfg(test)] +mod test { + use super::*; + use crate::app::test::*; + + #[tokio::test] + async fn set_alias() { + let app = TestApp::new().await; + + // Nonexisting entity + super::set_alias( + &app, + pm::SetAliasRequest { + entity_id: Some(EntityId::Uid(99999999).into()), + entity_type: pb::EntityType::Node.into(), + new_alias: "new_alias".to_string(), + }, + ) + .await + .unwrap_err(); + + // Invalid entity_type / entity_id combination + super::set_alias( + &app, + pm::SetAliasRequest { + entity_id: Some(EntityId::Uid(101001).into()), + entity_type: pb::EntityType::Target.into(), + new_alias: "new_alias".to_string(), + }, + ) + .await + .unwrap_err(); + + // Alias already in use + super::set_alias( + &app, + pm::SetAliasRequest { + entity_id: Some(EntityId::Alias("meta_node_1".try_into().unwrap()).into()), + entity_type: pb::EntityType::Node.into(), + new_alias: "meta_node_2".to_string(), + }, + ) + .await + .unwrap_err(); + + // Deny setting client aliases + super::set_alias( + &app, + pm::SetAliasRequest { + entity_id: Some( + EntityId::LegacyID(LegacyId { + node_type: NodeType::Client, + num_id: 1, + }) + .into(), + ), + entity_type: pb::EntityType::Node.into(), + new_alias: "new_alias".to_string(), + }, + ) + .await + .unwrap_err(); + + // Success + super::set_alias( + &app, + pm::SetAliasRequest { + entity_id: Some(EntityId::Uid(101001).into()), + entity_type: pb::EntityType::Node.into(), + new_alias: "new_alias".to_string(), + }, + ) + .await + .unwrap(); + + assert!(app.has_sent_notification::(&[ + NodeType::Meta, + NodeType::Storage, + NodeType::Client, + ])); + + assert_eq_db!( + app, + "SELECT alias FROM entities WHERE uid = ?1", + [101001], + "new_alias" + ); + } +}