1313// See the License for the specific language governing permissions and
1414// limitations under the License.
1515
16+ // cargo test --test soak integration::main --features integration
17+ //!
18+ //! It will send a batch of requests to the runtime and measure the throughput.
19+ //!
20+ //! It will also measure the latency of the requests.
21+ //!
22+ //! A reasonable soak test configuration to start off is 1 minute duration with 10000 batch load:
23+ //! export DYN_QUEUED_UP_PROCESSING=true
24+ //! export DYN_SOAK_BATCH_LOAD=10000
25+ //! export DYN_SOAK_RUN_DURATION=60s
26+ //! cargo test --test soak integration::main --features integration -- --nocapture
1627#[ cfg( feature = "integration" ) ]
1728mod integration {
1829
@@ -22,13 +33,17 @@ mod integration {
2233 logging,
2334 pipeline:: {
2435 async_trait, network:: Ingress , AsyncEngine , AsyncEngineContextProvider , Error , ManyOut ,
25- ResponseStream , SingleIn ,
36+ PushRouter , ResponseStream , SingleIn ,
2637 } ,
2738 protocols:: annotated:: Annotated ,
28- DistributedRuntime , ErrorContext , Result , Runtime , Worker ,
39+ stream , DistributedRuntime , ErrorContext , Result , Runtime , Worker ,
2940 } ;
3041 use futures:: StreamExt ;
31- use std:: { sync:: Arc , time:: Duration } ;
42+ use std:: {
43+ sync:: atomic:: { AtomicU64 , Ordering } ,
44+ sync:: Arc ,
45+ time:: Duration ,
46+ } ;
3247 use tokio:: time:: Instant ;
3348
3449 #[ test]
@@ -45,16 +60,29 @@ mod integration {
4560
4661 client. await ??;
4762 distributed. shutdown ( ) ;
48- server. await ??;
63+ let handler = server. await ??;
64+
65+ // Print final backend counter value
66+ let final_count = handler. backend_counter . load ( Ordering :: Relaxed ) ;
67+ println ! (
68+ "Final RequestHandler backend_counter: {} requests processed" ,
69+ final_count
70+ ) ;
4971
5072 Ok ( ( ) )
5173 }
5274
53- struct RequestHandler { }
75+ struct RequestHandler {
76+ backend_counter : AtomicU64 ,
77+ queued_up_processing : bool ,
78+ }
5479
5580 impl RequestHandler {
56- fn new ( ) -> Arc < Self > {
57- Arc :: new ( Self { } )
81+ fn new ( queued_up_processing : bool ) -> Arc < Self > {
82+ Arc :: new ( Self {
83+ backend_counter : AtomicU64 :: new ( 0 ) ,
84+ queued_up_processing,
85+ } )
5886 }
5987 }
6088
@@ -63,25 +91,40 @@ mod integration {
6391 async fn generate ( & self , input : SingleIn < String > ) -> Result < ManyOut < Annotated < String > > > {
6492 let ( data, ctx) = input. into_parts ( ) ;
6593
94+ // Increment backend counter
95+ self . backend_counter . fetch_add ( 1 , Ordering :: Relaxed ) ;
96+
6697 let chars = data
6798 . chars ( )
6899 . map ( |c| Annotated :: from_data ( c. to_string ( ) ) )
69100 . collect :: < Vec < _ > > ( ) ;
70101
71- let stream = async_stream:: stream! {
72- for c in chars {
73- yield c;
74- tokio:: time:: sleep( tokio:: time:: Duration :: from_millis( 100 ) ) . await ;
75- }
76- } ;
77-
78- Ok ( ResponseStream :: new ( Box :: pin ( stream) , ctx. context ( ) ) )
102+ if self . queued_up_processing {
103+ // queued up processing - delayed response to saturate the queue
104+ let async_stream = async_stream:: stream! {
105+ for c in chars {
106+ yield c;
107+ tokio:: time:: sleep( tokio:: time:: Duration :: from_millis( 100 ) ) . await ;
108+ }
109+ } ;
110+ Ok ( ResponseStream :: new ( Box :: pin ( async_stream) , ctx. context ( ) ) )
111+ } else {
112+ // normal processing - immediate response
113+ let iter_stream = stream:: iter ( chars) ;
114+ Ok ( ResponseStream :: new ( Box :: pin ( iter_stream) , ctx. context ( ) ) )
115+ }
79116 }
80117 }
81118
82- async fn backend ( runtime : DistributedRuntime ) -> Result < ( ) > {
119+ async fn backend ( runtime : DistributedRuntime ) -> Result < Arc < RequestHandler > > {
120+ // get the queued up processing setting from env (not delayed)
121+ let queued_up_processing =
122+ std:: env:: var ( "DYN_QUEUED_UP_PROCESSING" ) . unwrap_or ( "false" . to_string ( ) ) ;
123+ let queued_up_processing: bool = queued_up_processing. parse ( ) . unwrap_or ( false ) ;
124+
83125 // attach an ingress to an engine
84- let ingress = Ingress :: for_engine ( RequestHandler :: new ( ) ) ?;
126+ let handler = RequestHandler :: new ( queued_up_processing) ;
127+ let ingress = Ingress :: for_engine ( handler. clone ( ) ) ?;
85128
86129 // // make the ingress discoverable via a component service
87130 // // we must first create a service, then we can attach one more more endpoints
@@ -95,39 +138,44 @@ mod integration {
95138 . endpoint_builder ( )
96139 . handler ( ingress)
97140 . start ( )
98- . await
141+ . await ?;
142+
143+ Ok ( handler)
99144 }
100145
101146 async fn client ( runtime : DistributedRuntime ) -> Result < ( ) > {
102147 // get the run duration from env
103- let run_duration = std:: env:: var ( "DYN_SOAK_RUN_DURATION" ) . unwrap_or ( "1m " . to_string ( ) ) ;
148+ let run_duration = std:: env:: var ( "DYN_SOAK_RUN_DURATION" ) . unwrap_or ( "3s " . to_string ( ) ) ;
104149 let run_duration =
105- humantime:: parse_duration ( & run_duration) . unwrap_or ( Duration :: from_secs ( 60 ) ) ;
150+ humantime:: parse_duration ( & run_duration) . unwrap_or ( Duration :: from_secs ( 3 ) ) ;
106151
107- let batch_load = std:: env:: var ( "DYN_SOAK_BATCH_LOAD" ) . unwrap_or ( "10000 " . to_string ( ) ) ;
108- let batch_load: usize = batch_load. parse ( ) . unwrap_or ( 10000 ) ;
152+ let batch_load = std:: env:: var ( "DYN_SOAK_BATCH_LOAD" ) . unwrap_or ( "100 " . to_string ( ) ) ;
153+ let batch_load: usize = batch_load. parse ( ) . unwrap_or ( 100 ) ;
109154
110155 let client = runtime
111156 . namespace ( DEFAULT_NAMESPACE ) ?
112157 . component ( "backend" ) ?
113158 . endpoint ( "generate" )
114- . client :: < String , Annotated < String > > ( )
159+ . client ( )
115160 . await ?;
116161
117162 client. wait_for_instances ( ) . await ?;
118- let client = Arc :: new ( client) ;
163+ let router =
164+ PushRouter :: < String , Annotated < String > > :: from_client ( client, Default :: default ( ) )
165+ . await ?;
166+ let router = Arc :: new ( router) ;
119167
120168 let start = Instant :: now ( ) ;
121169 let mut count = 0 ;
122170
123171 loop {
124172 let mut tasks = Vec :: new ( ) ;
125173 for _ in 0 ..batch_load {
126- let client = client . clone ( ) ;
174+ let router = router . clone ( ) ;
127175 tasks. push ( tokio:: spawn ( async move {
128176 let mut stream = tokio:: time:: timeout (
129- Duration :: from_secs ( 30 ) ,
130- client . random ( "hello world" . to_string ( ) . into ( ) ) ,
177+ Duration :: from_secs ( 5 ) ,
178+ router . random ( "hello world" . to_string ( ) . into ( ) ) ,
131179 )
132180 . await
133181 . context ( "request timed out" ) ??;
@@ -147,7 +195,9 @@ mod integration {
147195
148196 let elapsed = start. elapsed ( ) ;
149197 count += batch_load;
150- println ! ( "elapsed: {:?}; count: {}" , elapsed, count) ;
198+ if count % 1000 == 0 {
199+ println ! ( "elapsed: {:?}; count: {}" , elapsed, count) ;
200+ }
151201
152202 if elapsed > run_duration {
153203 println ! ( "done" ) ;
0 commit comments