Skip to content

Commit 6b14b44

Browse files
robhoesrobhoes
authored
Merge pull request #10 from robhoes/trunk-pvs-direct
PVS-proxy updates for performance and error handling
2 parents c8a39ee + 03954d6 commit 6b14b44

File tree

2 files changed

+73
-29
lines changed

2 files changed

+73
-29
lines changed

‎scripts/setup-pvs-proxy-rules‎

Lines changed: 71 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,13 @@ TYPE=$2
1515
PVS_VM_INTERFACE=$3
1616
PRIVATE_PATH=$4
1717

18+
IP=/usr/sbin/ip
19+
VSCTL=/usr/bin/ovs-vsctl
20+
OFCTL=/usr/bin/ovs-ofctl
21+
XSREAD=/usr/bin/xenstore-read
22+
XSWRITE=/usr/bin/xenstore-write
23+
XSRM=/usr/bin/xenstore-rm
24+
1825
LOG_TAG="setup-pvs-proxy-rules"
1926

2027
handle_error()
@@ -31,38 +38,62 @@ handle_xs_error()
3138

3239
logger -t "$LOG_TAG" "Called as $0 $*"
3340

41+
path="${PRIVATE_PATH}/pvs-site"
42+
PVS_SITE=$($XSREAD "$path")
43+
if [ $? -ne 0 ] || [ -z "$PVS_SITE" ]; then
44+
handle_xs_error "$path"
45+
fi
46+
47+
path="${PRIVATE_PATH}/vif-uuid"
48+
VIF=$($XSREAD "$path")
49+
if [ $? -ne 0 ] || [ -z "$VIF" ]; then
50+
handle_xs_error "$path"
51+
fi
52+
53+
# Only continue if the proxy state is "started".
54+
path="/xapi/pvs-proxy/$PVS_SITE/$VIF/state"
55+
PVS_PROXY_STATE=$($XSREAD "$path")
56+
if [ "$PVS_PROXY_STATE" != "started" ]; then
57+
handle_error "PVS proxy daemon not configured for this proxy - not installing OVS rules."
58+
fi
59+
3460
path="${PRIVATE_PATH}/pvs-interface"
35-
PVS_PROXY_INTERFACE=$(xenstore-read "$path")
61+
PVS_PROXY_INTERFACE=$($XSREAD "$path")
3662
if [ $? -ne 0 ] || [ -z "$PVS_PROXY_INTERFACE" ]; then
3763
handle_xs_error "$path"
3864
fi
3965

4066
path="${PRIVATE_PATH}/mac"
41-
PVS_VM_MAC=$(xenstore-read "$path")
67+
PVS_VM_MAC=$($XSREAD "$path")
4268
if [ $? -ne 0 ] || [ -z "$PVS_VM_MAC" ]; then
4369
handle_xs_error "$path"
4470
fi
4571

4672
path="${PRIVATE_PATH}/pvs-server-num"
47-
PVS_SERVER_NUM=$(xenstore-read "$path")
73+
PVS_SERVER_NUM=$($XSREAD "$path")
4874
if [ $? -ne 0 ] || [ -z "$PVS_SERVER_NUM" ]; then
4975
handle_xs_error "$path"
5076
fi
5177

5278
path="${PRIVATE_PATH}/bridge"
53-
bridge=$(xenstore-read "$path")
79+
bridge=$($XSREAD "$path")
5480
if [ $? -ne 0 ] || [ -z "$bridge" ]; then
5581
handle_xs_error "$path"
5682
fi
57-
PVS_BRIDGE=$(ovs-vsctl br-to-parent "$bridge")
83+
PVS_BRIDGE=$($VSCTL br-to-parent "$bridge")
5884

5985
PVS_RULE_PRIO=1000
6086

6187
case $ACTION in
6288
add)
63-
PVS_PROXY_MAC=$(ovs-vsctl get interface "$PVS_PROXY_INTERFACE" mac_in_use | tr -d '"')
64-
PVS_PROXY_OFPORT=$(ovs-vsctl get interface "$PVS_PROXY_INTERFACE" ofport)
65-
PVS_VM_OFPORT=$(ovs-vsctl get interface "$PVS_VM_INTERFACE" ofport)
89+
# Create a port/interface for the proxy daemon
90+
$VSCTL --may-exist add-port "$PVS_BRIDGE" "$PVS_PROXY_INTERFACE" -- set interface "$PVS_PROXY_INTERFACE" type=internal
91+
$IP link set "$PVS_PROXY_INTERFACE" up
92+
logger -t "$LOG_TAG" "Created proxy port $PVS_PROXY_INTERFACE"
93+
94+
PVS_PROXY_MAC=$($VSCTL get interface "$PVS_PROXY_INTERFACE" mac_in_use | tr -d '"')
95+
PVS_PROXY_OFPORT=$($VSCTL get interface "$PVS_PROXY_INTERFACE" ofport)
96+
PVS_VM_OFPORT=$($VSCTL get interface "$PVS_VM_INTERFACE" ofport)
6697
if [ $? -ne 0 ] || [ -z "$PVS_VM_OFPORT" ]; then
6798
handle_error "The $PVS_VM_INTERFACE interface was not found on a bridge"
6899
fi
@@ -71,13 +102,13 @@ case $ACTION in
71102

72103
for ((j=0; j<PVS_SERVER_NUM; j++)) do
73104
path="${PRIVATE_PATH}/pvs-server-$j-addresses"
74-
PVS_SERVER_IPS=$(xenstore-read "$path")
105+
PVS_SERVER_IPS=$($XSREAD "$path")
75106
if [ $? -ne 0 ] || [ -z "$PVS_SERVER_IPS" ]; then
76107
handle_xs_error "$path"
77108
fi
78109

79110
path="${PRIVATE_PATH}/pvs-server-$j-ports"
80-
PVS_PORTS=$(xenstore-read "$path")
111+
PVS_PORTS=$($XSREAD "$path")
81112
if [ $? -ne 0 ] || [ -z "$PVS_PORTS" ]; then
82113
handle_xs_error "$path"
83114
fi
@@ -91,7 +122,7 @@ case $ACTION in
91122
# Packets from proxied clients that have a PVS-server IP must
92123
# be dropped. This is done separately for vif and tap interfaces
93124
# by matching on the in_port.
94-
ovs-ofctl --strict add-flow "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO)),in_port="$PVS_VM_OFPORT",ip,nw_src="$PVS_SERVER_IP",actions=drop
125+
$OFCTL --strict add-flow "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO)),in_port="$PVS_VM_OFPORT",ip,nw_src="$PVS_SERVER_IP",actions=drop
95126

96127
# The following rules are independent of the in_port, so we'll
97128
# need just one copy per VIF. We'll only apply them if the
@@ -100,22 +131,23 @@ case $ACTION in
100131
if [ "${TYPE}" = "vif" ]; then
101132
for ((i=PVS_STARTPORT; i<=PVS_ENDPORT; i++)) do
102133
# Packets from client->server that need to be proxied.
103-
ovs-ofctl --strict add-flow "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO-1)),udp,dl_src="$PVS_VM_MAC",nw_dst="$PVS_SERVER_IP",tp_dst=$i,actions="$PVS_PROXY_OFPORT"
134+
$OFCTL --strict add-flow "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO-1)),udp,dl_src="$PVS_VM_MAC",nw_dst="$PVS_SERVER_IP",tp_dst=$i,actions="$PVS_PROXY_OFPORT"
104135
# Packets from proxy->client.
105-
ovs-ofctl --strict add-flow "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO)),udp,dl_src="$PVS_PROXY_MAC",dl_dst="$PVS_VM_MAC",nw_src="$PVS_SERVER_IP",tp_dst=$i,actions=NORMAL
136+
$OFCTL --strict add-flow "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO)),udp,dl_src="$PVS_PROXY_MAC",dl_dst="$PVS_VM_MAC",nw_src="$PVS_SERVER_IP",tp_dst=$i,actions=NORMAL
106137
# Packets from server->client that need to be proxied.
107-
ovs-ofctl --strict add-flow "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO-1)),udp,dl_dst="$PVS_VM_MAC",nw_src="$PVS_SERVER_IP",tp_dst=$i,actions="$PVS_PROXY_OFPORT"
138+
$OFCTL --strict add-flow "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO-1)),udp,dl_dst="$PVS_VM_MAC",nw_src="$PVS_SERVER_IP",tp_dst=$i,actions="$PVS_PROXY_OFPORT"
108139
done
109-
# Announce that on the OVS we have set up the rules for this VIF's pvs-proxy.
110-
xenstore-write "${PRIVATE_PATH}/pvs-rules-active" ''
111140
fi
112141
done
113142
unset IFS
143+
144+
# Announce that on the OVS we have set up the rules for this VIF's pvs-proxy
145+
$XSWRITE "${PRIVATE_PATH}/pvs-rules-active" ''
114146
done
115147
;;
116148
remove)
117-
PVS_PROXY_MAC=$(ovs-vsctl get interface "$PVS_PROXY_INTERFACE" mac_in_use | tr -d '"')
118-
PVS_VM_OFPORT=$(ovs-vsctl get interface "$PVS_VM_INTERFACE" ofport)
149+
PVS_PROXY_MAC=$($VSCTL get interface "$PVS_PROXY_INTERFACE" mac_in_use | tr -d '"')
150+
PVS_VM_OFPORT=$($VSCTL get interface "$PVS_VM_INTERFACE" ofport)
119151
if [ $? -ne 0 ] || [ -z "$PVS_VM_OFPORT" ]; then
120152
handle_error "The $PVS_VM_INTERFACE interface was not found on a bridge"
121153
fi
@@ -124,13 +156,13 @@ case $ACTION in
124156

125157
for ((j=0; j<PVS_SERVER_NUM; j++)) do
126158
path="${PRIVATE_PATH}/pvs-server-$j-addresses"
127-
PVS_SERVER_IPS=$(xenstore-read "$path")
159+
PVS_SERVER_IPS=$($XSREAD "$path")
128160
if [ $? -ne 0 ] || [ -z "$PVS_SERVER_IPS" ]; then
129161
handle_xs_error "$path"
130162
fi
131163

132164
path="${PRIVATE_PATH}/pvs-server-$j-ports"
133-
PVS_PORTS=$(xenstore-read "$path")
165+
PVS_PORTS=$($XSREAD "$path")
134166
if [ $? -ne 0 ] || [ -z "$PVS_PORTS" ]; then
135167
handle_xs_error "$path"
136168
fi
@@ -144,7 +176,7 @@ case $ACTION in
144176
# Packets from proxied clients that have a PVS-server IP must
145177
# be dropped. This is done separately for vif and tap interfaces
146178
# by matching on the in_port.
147-
ovs-ofctl --strict del-flows "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO)),in_port="$PVS_VM_OFPORT",ip,nw_src="$PVS_SERVER_IP"
179+
$OFCTL --strict del-flows "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO)),in_port="$PVS_VM_OFPORT",ip,nw_src="$PVS_SERVER_IP"
148180

149181
# The following rules are independent of the in_port, so we'll
150182
# need just one copy per VIF. We'll only apply them if the
@@ -153,22 +185,33 @@ case $ACTION in
153185
if [ "${TYPE}" = "vif" ]; then
154186
for ((i=PVS_STARTPORT; i<=PVS_ENDPORT; i++)) do
155187
# Packets from client->server that need to be proxied.
156-
ovs-ofctl --strict del-flows "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO-1)),udp,dl_src="$PVS_VM_MAC",nw_dst="$PVS_SERVER_IP",tp_dst=$i
188+
$OFCTL --strict del-flows "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO-1)),udp,dl_src="$PVS_VM_MAC",nw_dst="$PVS_SERVER_IP",tp_dst=$i
157189
# Packets from proxy->client.
158-
ovs-ofctl --strict del-flows "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO)),udp,dl_src="$PVS_PROXY_MAC",dl_dst="$PVS_VM_MAC",nw_src="$PVS_SERVER_IP",tp_dst=$i
190+
$OFCTL --strict del-flows "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO)),udp,dl_src="$PVS_PROXY_MAC",dl_dst="$PVS_VM_MAC",nw_src="$PVS_SERVER_IP",tp_dst=$i
159191
# Packets from server->client that need to be proxied.
160-
ovs-ofctl --strict del-flows "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO-1)),udp,dl_dst="$PVS_VM_MAC",nw_src="$PVS_SERVER_IP",tp_dst=$i
192+
$OFCTL --strict del-flows "$PVS_BRIDGE" priority=$((PVS_RULE_PRIO-1)),udp,dl_dst="$PVS_VM_MAC",nw_src="$PVS_SERVER_IP",tp_dst=$i
161193
done
162-
# Announce that on the OVS we have removed the rules for this VIF's pvs-proxy.
163-
xenstore-rm "${PRIVATE_PATH}/pvs-rules-active"
164194
fi
165195
done
166196
unset IFS
197+
198+
if [ "${TYPE}" = "vif" ]; then
199+
# Again, don't do the following when a tap goes away, because
200+
# vif may still be there.
201+
202+
# Remove the port/interface for the proxy daemon
203+
$IP link set "$PVS_PROXY_INTERFACE" down
204+
$VSCTL del-port "$PVS_BRIDGE" "$PVS_PROXY_INTERFACE"
205+
logger -t "$LOG_TAG" "Removed proxy port $PVS_PROXY_INTERFACE"
206+
207+
# Announce that on the OVS we have removed the rules for this VIF's pvs-proxy.
208+
XSRM "${PRIVATE_PATH}/pvs-rules-active"
209+
fi
167210
done
168211
;;
169212
reset)
170-
ovs-ofctl del-flows "$PVS_BRIDGE"
171-
ovs-ofctl --strict add-flow "$PVS_BRIDGE" priority=0,actions=NORMAL
213+
$OFCTL del-flows "$PVS_BRIDGE"
214+
$OFCTL --strict add-flow "$PVS_BRIDGE" priority=0,actions=NORMAL
172215
;;
173216
*)
174217
handle_error "Unknown command '$1'"

‎xc/xenops_server_xen.ml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2340,7 +2340,7 @@ module VIF = struct
23402340
let xenstore_of_pvs_proxy proxy =
23412341
match proxy with
23422342
| None -> []
2343-
| Some (servers, interface) ->
2343+
| Some (site, servers, interface) ->
23442344
let open Vif.PVS_proxy in
23452345
let server_keys =
23462346
List.mapi (fun i server ->
@@ -2352,6 +2352,7 @@ module VIF = struct
23522352
) servers
23532353
|> List.flatten
23542354
in
2355+
("pvs-site", site) ::
23552356
("pvs-interface", interface) ::
23562357
("pvs-server-num", string_of_int (List.length servers)) ::
23572358
server_keys

0 commit comments

Comments
 (0)