Skip to content

Commit 41c689e

Browse files
rmccorm4rmccorm4
authored
Add ensemble parameter forwarding test (#6284)
1 parent e0dc4aa commit 41c689e

File tree

4 files changed

+138
-28
lines changed

4 files changed

+138
-28
lines changed

‎qa/L0_parameters/model_repository/ensemble/config.pbtxt‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# * Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# * Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
# * Neither the name of NVIDIA CORPORATION nor the names of its
12+
# contributors may be used to endorse or promote products derived
13+
# from this software without specific prior written permission.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
16+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
27+
platform: "ensemble"
28+
max_batch_size: 0
29+
30+
input [
31+
{
32+
name: "INPUT0"
33+
data_type: TYPE_FP32
34+
dims: [ 1 ]
35+
}
36+
]
37+
38+
output [
39+
{
40+
name: "key"
41+
data_type: TYPE_STRING
42+
dims: [ -1 ]
43+
},
44+
{
45+
name: "value"
46+
data_type: TYPE_STRING
47+
dims: [ -1 ]
48+
}
49+
]
50+
51+
ensemble_scheduling
52+
{
53+
step [
54+
{
55+
model_name: "identity"
56+
model_version: -1
57+
input_map { key: "INPUT0", value: "INPUT0" }
58+
output_map { key: "OUTPUT0", value: "OUTPUT0" }
59+
},
60+
{
61+
model_name: "parameter"
62+
model_version: -1
63+
input_map { key: "INPUT0", value: "OUTPUT0" }
64+
output_map { key: "key", value: "key" }
65+
output_map { key: "value", value: "value" }
66+
}
67+
]
68+
}

‎qa/L0_parameters/model_repository/identity/config.pbtxt‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# * Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# * Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
# * Neither the name of NVIDIA CORPORATION nor the names of its
12+
# contributors may be used to endorse or promote products derived
13+
# from this software without specific prior written permission.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
16+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
27+
backend: "identity"
28+
max_batch_size: 0
29+
30+
input [
31+
{
32+
name: "INPUT0"
33+
data_type: TYPE_FP32
34+
dims: [ 1 ]
35+
}
36+
]
37+
38+
output [
39+
{
40+
name: "OUTPUT0"
41+
data_type: TYPE_FP32
42+
dims: [ 1 ]
43+
}
44+
]

‎qa/L0_parameters/parameters_test.py‎

Lines changed: 19 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,20 @@ def create_inputs(self, client_type):
9494
inputs[0].set_data_from_numpy(np.asarray([1], dtype=np.float32))
9595
return inputs
9696

97-
async def send_request_and_verify(self, client_type, client, is_async=False):
97+
async def send_request_and_verify(
98+
self, client_type, client, is_async=False, model_name="parameter"
99+
):
98100
inputs = self.create_inputs(client_type)
99101
for parameters in self.parameter_list:
102+
# Setup infer callable to re-use below for brevity
103+
infer_callable = partial(
104+
client.infer,
105+
model_name=model_name,
106+
inputs=inputs,
107+
parameters=parameters,
108+
headers=self.headers,
109+
)
110+
100111
# The `triton_` prefix is reserved for Triton usage
101112
should_error = False
102113
if "triton_" in parameters.keys():
@@ -105,38 +116,17 @@ async def send_request_and_verify(self, client_type, client, is_async=False):
105116
if is_async:
106117
if should_error:
107118
with self.assertRaises(InferenceServerException):
108-
result = await client.infer(
109-
model_name="parameter",
110-
inputs=inputs,
111-
parameters=parameters,
112-
headers=self.headers,
113-
)
119+
await infer_callable()
114120
return
115121
else:
116-
result = await client.infer(
117-
model_name="parameter",
118-
inputs=inputs,
119-
parameters=parameters,
120-
headers=self.headers,
121-
)
122-
122+
result = await infer_callable()
123123
else:
124124
if should_error:
125125
with self.assertRaises(InferenceServerException):
126-
result = client.infer(
127-
model_name="parameter",
128-
inputs=inputs,
129-
parameters=parameters,
130-
headers=self.headers,
131-
)
126+
infer_callable()
132127
return
133128
else:
134-
result = client.infer(
135-
model_name="parameter",
136-
inputs=inputs,
137-
parameters=parameters,
138-
headers=self.headers,
139-
)
129+
result = infer_callable()
140130

141131
self.verify_outputs(result, parameters)
142132

@@ -219,6 +209,9 @@ def test_grpc_stream_parameter(self):
219209
self.verify_outputs(result, parameters)
220210
self.grpc.stop_stream()
221211

212+
async def test_ensemble_parameter_forwarding(self):
213+
await self.send_request_and_verify(httpclient, self.http, model_name="ensemble")
214+
222215
async def asyncTearDown(self):
223216
self.http.close()
224217
self.grpc.close()

‎qa/L0_parameters/test.sh‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ SERVER=/opt/tritonserver/bin/tritonserver
4545
SERVER_LOG="./inference_server.log"
4646
source ../common/util.sh
4747

48+
MODELDIR="model_repository"
49+
# Use identity model as dummy step to ensure parameters pass through each step
50+
mkdir -p "${MODELDIR}/identity/1"
51+
mkdir -p "${MODELDIR}/ensemble/1"
52+
4853
# TODO: Add support and testing for C++ client parameters:
4954
# https://jirasw.nvidia.com/browse/DLIS-4673
5055

@@ -54,9 +59,9 @@ for i in {0..1}; do
5459
# TEST_HEADER is a parameter used by `parameters_test.py` that controls
5560
# whether the script will test for inclusion of headers in parameters or not.
5661
if [ $i == 1 ]; then
57-
SERVER_ARGS="--model-repository=model_repository --exit-timeout-secs=120 --grpc-header-forward-pattern my_header.* --http-header-forward-pattern my_header.*"
62+
SERVER_ARGS="--model-repository=${MODELDIR} --exit-timeout-secs=120 --grpc-header-forward-pattern my_header.* --http-header-forward-pattern my_header.*"
5863
else
59-
SERVER_ARGS="--model-repository=model_repository --exit-timeout-secs=120"
64+
SERVER_ARGS="--model-repository=${MODELDIR} --exit-timeout-secs=120"
6065
fi
6166
run_server
6267
if [ "$SERVER_PID" == "0" ]; then

0 commit comments

Comments
 (0)