Skip to content

Commit d2fc025

Browse files
FrozenGenetqchen
FrozenGene
authored and
tqchen
committed
[RUTNIME] Support C++ RPC (#4281)
1 parent 2f65a87 commit d2fc025

File tree

12 files changed

+1766
-2
lines changed

12 files changed

+1766
-2
lines changed

‎apps/cpp_rpc/Makefile‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Makefile to compile RPC Server.
19+
TVM_ROOT=$(shell cd ../..; pwd)
20+
DMLC_CORE=${TVM_ROOT}/3rdparty/dmlc-core
21+
TVM_RUNTIME_DIR?=
22+
OS?=
23+
24+
# Android can not link pthrad, but Linux need.
25+
ifeq ($(OS), Linux)
26+
LINK_PTHREAD=-lpthread
27+
else
28+
LINK_PTHREAD=
29+
endif
30+
31+
PKG_CFLAGS = -std=c++11 -O2 -fPIC -Wall\
32+
-I${TVM_ROOT}/include\
33+
-I${DMLC_CORE}/include\
34+
-I${TVM_ROOT}/3rdparty/dlpack/include
35+
36+
PKG_LDFLAGS = -L$(TVM_RUNTIME_DIR) $(LINK_PTHREAD) -ltvm_runtime -ldl -Wl,-R$(TVM_RUNTIME_DIR)
37+
38+
ifeq ($(USE_GLOG), 1)
39+
PKG_CFLAGS += -DDMLC_USE_GLOG=1
40+
PKG_LDFLAGS += -lglog
41+
endif
42+
43+
.PHONY: clean all
44+
45+
all: tvm_rpc
46+
47+
# Build rule for all in one TVM package library
48+
tvm_rpc: *.cc
49+
@mkdir -p $(@D)
50+
$(CXX) $(PKG_CFLAGS) -o $@ $(filter %.cc %.o %.a, $^) $(PKG_LDFLAGS)
51+
52+
clean:
53+
-rm -f tvm_rpc

‎apps/cpp_rpc/README.md‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
<!--- Licensed to the Apache Software Foundation (ASF) under one -->
2+
<!--- or more contributor license agreements. See the NOTICE file -->
3+
<!--- distributed with this work for additional information -->
4+
<!--- regarding copyright ownership. The ASF licenses this file -->
5+
<!--- to you under the Apache License, Version 2.0 (the -->
6+
<!--- "License"); you may not use this file except in compliance -->
7+
<!--- with the License. You may obtain a copy of the License at -->
8+
9+
<!--- http://www.apache.org/licenses/LICENSE-2.0 -->
10+
11+
<!--- Unless required by applicable law or agreed to in writing, -->
12+
<!--- software distributed under the License is distributed on an -->
13+
<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
14+
<!--- KIND, either express or implied. See the License for the -->
15+
<!--- specific language governing permissions and limitations -->
16+
<!--- under the License. -->
17+
18+
# TVM RPC Server
19+
This folder contains a simple recipe to make RPC server in c++.
20+
21+
## Usage
22+
- Build tvm runtime
23+
- Make the rpc executable [Makefile](Makefile).
24+
`make CXX=/path/to/cross compiler g++/ TVM_RUNTIME_DIR=/path/to/tvm runtime library directory/ OS=Linux`
25+
if you want to compile it for embedded Linux, you should add `OS=Linux`.
26+
if the target os is Android, you doesn't need to pass OS argument.
27+
You could cross compile the TVM runtime like this:
28+
```
29+
cd tvm
30+
mkdir arm_runtime
31+
cp cmake/config.cmake arm_runtime
32+
cd arm_runtime
33+
cmake .. -DCMAKE_CXX_COMPILER="/path/to/cross compiler g++/"
34+
make runtime
35+
```
36+
- Use `./tvm_rpc server` to start the RPC server
37+
38+
## How it works
39+
- The tvm runtime dll is linked along with this executable and when the RPC server starts it will load the tvm runtime library.
40+
41+
```
42+
Command line usage
43+
server - Start the server
44+
--host - The hostname of the server, Default=0.0.0.0
45+
--port - The port of the RPC, Default=9090
46+
--port-end - The end search port of the RPC, Default=9199
47+
--tracker - The RPC tracker address in host:port format e.g. 10.1.1.2:9190 Default=""
48+
--key - The key used to identify the device type in tracker. Default=""
49+
--custom-addr - Custom IP Address to Report to RPC Tracker. Default=""
50+
--silent - Whether to run in silent mode. Default=False
51+
Example
52+
./tvm_rpc server --host=0.0.0.0 --port=9000 --port-end=9090 --tracker=127.0.0.1:9190 --key=rasp
53+
```
54+
55+
## Note
56+
Currently support is only there for Linux / Android environment and proxy mode doesn't be supported currently.

‎apps/cpp_rpc/main.cc‎

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
/*!
21+
* \file rpc_server.cc
22+
* \brief RPC Server for TVM.
23+
*/
24+
#include <stdlib.h>
25+
#include <signal.h>
26+
#include <stdio.h>
27+
#include <unistd.h>
28+
#include <dmlc/logging.h>
29+
#include <iostream>
30+
#include <cstring>
31+
#include <vector>
32+
#include <sstream>
33+
34+
#include "../../src/common/util.h"
35+
#include "../../src/common/socket.h"
36+
#include "rpc_server.h"
37+
38+
using namespace std;
39+
using namespace tvm::runtime;
40+
using namespace tvm::common;
41+
42+
static const string kUSAGE = \
43+
"Command line usage\n" \
44+
" server - Start the server\n" \
45+
"--host - The hostname of the server, Default=0.0.0.0\n" \
46+
"--port - The port of the RPC, Default=9090\n" \
47+
"--port-end - The end search port of the RPC, Default=9199\n" \
48+
"--tracker - The RPC tracker address in host:port format e.g. 10.1.1.2:9190 Default=\"\"\n" \
49+
"--key - The key used to identify the device type in tracker. Default=\"\"\n" \
50+
"--custom-addr - Custom IP Address to Report to RPC Tracker. Default=\"\"\n" \
51+
"--silent - Whether to run in silent mode. Default=False\n" \
52+
"\n" \
53+
" Example\n" \
54+
" ./tvm_rpc server --host=0.0.0.0 --port=9000 --port-end=9090 "
55+
" --tracker=127.0.0.1:9190 --key=rasp" \
56+
"\n";
57+
58+
/*!
59+
* \brief RpcServerArgs.
60+
* \arg host The hostname of the server, Default=0.0.0.0
61+
* \arg port The port of the RPC, Default=9090
62+
* \arg port_end The end search port of the RPC, Default=9199
63+
* \arg tracker The address of RPC tracker in host:port format e.g. 10.77.1.234:9190 Default=""
64+
* \arg key The key used to identify the device type in tracker. Default=""
65+
* \arg custom_addr Custom IP Address to Report to RPC Tracker. Default=""
66+
* \arg silent Whether run in silent mode. Default=False
67+
*/
68+
struct RpcServerArgs {
69+
string host = "0.0.0.0";
70+
int port = 9090;
71+
int port_end = 9099;
72+
string tracker;
73+
string key;
74+
string custom_addr;
75+
bool silent = false;
76+
};
77+
78+
/*!
79+
* \brief PrintArgs print the contents of RpcServerArgs
80+
* \param args RpcServerArgs structure
81+
*/
82+
void PrintArgs(struct RpcServerArgs args) {
83+
LOG(INFO) << "host = " << args.host;
84+
LOG(INFO) << "port = " << args.port;
85+
LOG(INFO) << "port_end = " << args.port_end;
86+
LOG(INFO) << "tracker = " << args.tracker;
87+
LOG(INFO) << "key = " << args.key;
88+
LOG(INFO) << "custom_addr = " << args.custom_addr;
89+
LOG(INFO) << "silent = " << ((args.silent) ? ("True"): ("False"));
90+
}
91+
92+
/*!
93+
* \brief CtrlCHandler, exits if Ctrl+C is pressed
94+
* \param s signal
95+
*/
96+
void CtrlCHandler(int s) {
97+
LOG(INFO) << "\nUser pressed Ctrl+C, Exiting";
98+
exit(1);
99+
}
100+
101+
/*!
102+
* \brief HandleCtrlC Register for handling Ctrl+C event.
103+
*/
104+
void HandleCtrlC() {
105+
// Ctrl+C handler
106+
struct sigaction sigIntHandler;
107+
sigIntHandler.sa_handler = CtrlCHandler;
108+
sigemptyset(&sigIntHandler.sa_mask);
109+
sigIntHandler.sa_flags = 0;
110+
sigaction(SIGINT, &sigIntHandler, nullptr);
111+
}
112+
113+
/*!
114+
* \brief GetCmdOption Parse and find the command option.
115+
* \param argc arg counter
116+
* \param argv arg values
117+
* \param option command line option to search for.
118+
* \param key whether the option itself is key
119+
* \return value corresponding to option.
120+
*/
121+
string GetCmdOption(int argc, char* argv[], string option, bool key = false) {
122+
string cmd;
123+
for (int i = 1; i < argc; ++i) {
124+
string arg = argv[i];
125+
if (arg.find(option) == 0) {
126+
if (key) {
127+
cmd = argv[i];
128+
return cmd;
129+
}
130+
// We assume "=" is the end of option.
131+
CHECK_EQ(*option.rbegin(), '=');
132+
cmd = arg.substr(arg.find("=") + 1);
133+
return cmd;
134+
}
135+
}
136+
return cmd;
137+
}
138+
139+
/*!
140+
* \brief ValidateTracker Check the tracker address format is correct and changes the format.
141+
* \param tracker The tracker input.
142+
* \return result of operation.
143+
*/
144+
bool ValidateTracker(string &tracker) {
145+
vector<string> list = Split(tracker, ':');
146+
if ((list.size() != 2) || (!ValidateIP(list[0])) || (!IsNumber(list[1]))) {
147+
return false;
148+
}
149+
ostringstream ss;
150+
ss << "('" << list[0] << "', " << list[1] << ")";
151+
tracker = ss.str();
152+
return true;
153+
}
154+
155+
/*!
156+
* \brief ParseCmdArgs parses the command line arguments.
157+
* \param argc arg counter
158+
* \param argv arg values
159+
* \param args, the output structure which holds the parsed values
160+
*/
161+
void ParseCmdArgs(int argc, char * argv[], struct RpcServerArgs &args) {
162+
string silent = GetCmdOption(argc, argv, "--silent", true);
163+
if (!silent.empty()) {
164+
args.silent = true;
165+
// Only errors and fatal is logged
166+
dmlc::InitLogging("--minloglevel=2");
167+
}
168+
169+
string host = GetCmdOption(argc, argv, "--host=");
170+
if (!host.empty()) {
171+
if (!ValidateIP(host)) {
172+
LOG(WARNING) << "Wrong host address format.";
173+
LOG(INFO) << kUSAGE;
174+
exit(1);
175+
}
176+
args.host = host;
177+
}
178+
179+
string port = GetCmdOption(argc, argv, "--port=");
180+
if (!port.empty()) {
181+
if (!IsNumber(port) || stoi(port) > 65535) {
182+
LOG(WARNING) << "Wrong port number.";
183+
LOG(INFO) << kUSAGE;
184+
exit(1);
185+
}
186+
args.port = stoi(port);
187+
}
188+
189+
string port_end = GetCmdOption(argc, argv, "--port_end=");
190+
if (!port_end.empty()) {
191+
if (!IsNumber(port_end) || stoi(port_end) > 65535) {
192+
LOG(WARNING) << "Wrong port_end number.";
193+
LOG(INFO) << kUSAGE;
194+
exit(1);
195+
}
196+
args.port_end = stoi(port_end);
197+
}
198+
199+
string tracker = GetCmdOption(argc, argv, "--tracker=");
200+
if (!tracker.empty()) {
201+
if (!ValidateTracker(tracker)) {
202+
LOG(WARNING) << "Wrong tracker address format.";
203+
LOG(INFO) << kUSAGE;
204+
exit(1);
205+
}
206+
args.tracker = tracker;
207+
}
208+
209+
string key = GetCmdOption(argc, argv, "--key=");
210+
if (!key.empty()) {
211+
args.key = key;
212+
}
213+
214+
string custom_addr = GetCmdOption(argc, argv, "--custom_addr=");
215+
if (!custom_addr.empty()) {
216+
if (!ValidateIP(custom_addr)) {
217+
LOG(WARNING) << "Wrong custom address format.";
218+
LOG(INFO) << kUSAGE;
219+
exit(1);
220+
}
221+
args.custom_addr = custom_addr;
222+
}
223+
}
224+
225+
/*!
226+
* \brief RpcServer Starts the RPC server.
227+
* \param argc arg counter
228+
* \param argv arg values
229+
* \return result of operation.
230+
*/
231+
int RpcServer(int argc, char * argv[]) {
232+
struct RpcServerArgs args;
233+
234+
/* parse the command line args */
235+
ParseCmdArgs(argc, argv, args);
236+
PrintArgs(args);
237+
238+
// Ctrl+C handler
239+
LOG(INFO) << "Starting CPP Server, Press Ctrl+C to stop.";
240+
HandleCtrlC();
241+
tvm::runtime::RPCServerCreate(args.host, args.port, args.port_end, args.tracker,
242+
args.key, args.custom_addr, args.silent);
243+
return 0;
244+
}
245+
246+
/*!
247+
* \brief main The main function.
248+
* \param argc arg counter
249+
* \param argv arg values
250+
* \return result of operation.
251+
*/
252+
int main(int argc, char * argv[]) {
253+
if (argc <= 1) {
254+
LOG(INFO) << kUSAGE;
255+
return 0;
256+
}
257+
258+
if (0 == strcmp(argv[1], "server")) {
259+
RpcServer(argc, argv);
260+
} else {
261+
LOG(INFO) << kUSAGE;
262+
}
263+
264+
return 0;
265+
}

0 commit comments

Comments
 (0)