Skip to content

Commit da720e4

Browse files
EikykunEikykun
authored
enhance: enable iptables modify nat output chain to redirect outbound traffic by init container (#36)
1 parent 3bc3457 commit da720e4

File tree

4 files changed

+109
-60
lines changed

4 files changed

+109
-60
lines changed

‎artifacts/scripts/proxy-init.sh‎

Lines changed: 42 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -14,27 +14,27 @@ if [ ! -f "${SA_DIR}/token" ]; then
1414
fi
1515

1616
# Remove the old chains, to generate new configs.
17-
iptables -t nat -D PREROUTING -p tcp -j ctrlmesh_PROXY_INBOUND 2>/dev/null
18-
iptables -t mangle -D PREROUTING -p tcp -j ctrlmesh_PROXY_INBOUND 2>/dev/null
19-
iptables -t nat -D OUTPUT -p tcp -j ctrlmesh_PROXY_OUTPUT 2>/dev/null
17+
iptables -t nat -D PREROUTING -p tcp -j CTRLMESH_INBOUND 2>/dev/null
18+
iptables -t mangle -D PREROUTING -p tcp -j CTRLMESH_INBOUND 2>/dev/null
19+
iptables -t nat -D OUTPUT -p tcp -j CTRLMESH_OUTPUT 2>/dev/null
2020

2121
# Flush and delete the ctrlmesh chains.
22-
iptables -t nat -F ctrlmesh_PROXY_OUTPUT 2>/dev/null
23-
iptables -t nat -X ctrlmesh_PROXY_OUTPUT 2>/dev/null
24-
iptables -t nat -F ctrlmesh_PROXY_INBOUND 2>/dev/null
25-
iptables -t nat -X ctrlmesh_PROXY_INBOUND 2>/dev/null
26-
iptables -t mangle -F ctrlmesh_PROXY_INBOUND 2>/dev/null
27-
iptables -t mangle -X ctrlmesh_PROXY_INBOUND 2>/dev/null
28-
iptables -t mangle -F ctrlmesh_PROXY_DIVERT 2>/dev/null
29-
iptables -t mangle -X ctrlmesh_PROXY_DIVERT 2>/dev/null
30-
iptables -t mangle -F ctrlmesh_PROXY_TPROXY 2>/dev/null
31-
iptables -t mangle -X ctrlmesh_PROXY_TPROXY 2>/dev/null
22+
iptables -t nat -F CTRLMESH_OUTPUT 2>/dev/null
23+
iptables -t nat -X CTRLMESH_OUTPUT 2>/dev/null
24+
iptables -t nat -F CTRLMESH_INBOUND 2>/dev/null
25+
iptables -t nat -X CTRLMESH_INBOUND 2>/dev/null
26+
iptables -t mangle -F CTRLMESH_INBOUND 2>/dev/null
27+
iptables -t mangle -X CTRLMESH_INBOUND 2>/dev/null
28+
iptables -t mangle -F CTRLMESH_DIVERT 2>/dev/null
29+
iptables -t mangle -X CTRLMESH_DIVERT 2>/dev/null
30+
iptables -t mangle -F CTRLMESH_TPROXY 2>/dev/null
31+
iptables -t mangle -X CTRLMESH_TPROXY 2>/dev/null
3232

3333
# Must be last, the others refer to it
34-
iptables -t nat -F ctrlmesh_PROXY_REDIRECT 2>/dev/null
35-
iptables -t nat -X ctrlmesh_PROXY_REDIRECT 2>/dev/null
36-
iptables -t nat -F ctrlmesh_PROXY_IN_REDIRECT 2>/dev/null
37-
iptables -t nat -X ctrlmesh_PROXY_IN_REDIRECT 2>/dev/null
34+
iptables -t nat -F CTRLMESH_REDIRECT 2>/dev/null
35+
iptables -t nat -X CTRLMESH_REDIRECT 2>/dev/null
36+
iptables -t nat -F CTRLMESH_IN_REDIRECT 2>/dev/null
37+
iptables -t nat -X CTRLMESH_IN_REDIRECT 2>/dev/null
3838

3939
if [ "${1:-}" = "clean" ]; then
4040
echo "Only cleaning, no new rules added"
@@ -70,13 +70,13 @@ set -o pipefail
7070
set -x # echo on
7171

7272
# Create a new chain for redirecting outbound traffic to the apiserver port.
73-
# In both chains, '-j RETURN' bypasses Proxy and '-j ctrlmesh_PROXY_REDIRECT' redirects to Proxy.
74-
iptables -t nat -N ctrlmesh_PROXY_REDIRECT
75-
iptables -t nat -A ctrlmesh_PROXY_REDIRECT -p tcp -j REDIRECT --to-port "${PROXY_APISERVER_PORT}"
73+
# In both chains, '-j RETURN' bypasses Proxy and '-j CTRLMESH_REDIRECT' redirects to Proxy.
74+
iptables -t nat -N CTRLMESH_REDIRECT
75+
iptables -t nat -A CTRLMESH_REDIRECT -p tcp -j REDIRECT --to-port "${PROXY_APISERVER_PORT}"
7676

7777
# Use this chain also for redirecting inbound traffic to the webhook port when not using TPROXY.
78-
iptables -t nat -N ctrlmesh_PROXY_IN_REDIRECT
79-
iptables -t nat -A ctrlmesh_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port "${PROXY_WEBHOOK_PORT}"
78+
iptables -t nat -N CTRLMESH_IN_REDIRECT
79+
iptables -t nat -A CTRLMESH_IN_REDIRECT -p tcp -j REDIRECT --to-port "${PROXY_WEBHOOK_PORT}"
8080

8181
# Handling of inbound ports. Traffic will be redirected to Proxy, which will process and forward
8282
# to the local webhook. If not set, no inbound port will be intercepted by the iptables.
@@ -85,56 +85,56 @@ if [ -n "${INBOUND_WEBHOOK_PORT}" ]; then
8585
# When using TPROXY, create a new chain for routing all inbound traffic to
8686
# Proxy. Any packet entering this chain gets marked with the ${INBOUND_TPROXY_MARK} mark,
8787
# so that they get routed to the loopback interface in order to get redirected to Proxy.
88-
# In the ctrlmesh_PROXY_INBOUND chain, '-j ctrlmesh_PROXY_DIVERT' reroutes to the loopback
88+
# In the CTRLMESH_INBOUND chain, '-j CTRLMESH_DIVERT' reroutes to the loopback
8989
# interface.
9090
# Mark all inbound packets.
91-
iptables -t mangle -N ctrlmesh_PROXY_DIVERT
92-
iptables -t mangle -A ctrlmesh_PROXY_DIVERT -j MARK --set-mark "${INBOUND_TPROXY_MARK}"
93-
iptables -t mangle -A ctrlmesh_PROXY_DIVERT -j ACCEPT
91+
iptables -t mangle -N CTRLMESH_DIVERT
92+
iptables -t mangle -A CTRLMESH_DIVERT -j MARK --set-mark "${INBOUND_TPROXY_MARK}"
93+
iptables -t mangle -A CTRLMESH_DIVERT -j ACCEPT
9494

95-
# Route all packets marked in chain ctrlmesh_PROXY_DIVERT using routing table ${INBOUND_TPROXY_ROUTE_TABLE}.
95+
# Route all packets marked in chain CTRLMESH_DIVERT using routing table ${INBOUND_TPROXY_ROUTE_TABLE}.
9696
ip -f inet rule add fwmark "${INBOUND_TPROXY_MARK}" lookup "${INBOUND_TPROXY_ROUTE_TABLE}"
9797
# In routing table ${INBOUND_TPROXY_ROUTE_TABLE}, create a single default rule to route all traffic to
9898
# the loopback interface.
9999
ip -f inet route add local default dev lo table "${INBOUND_TPROXY_ROUTE_TABLE}" || ip route show table all
100100

101101
# Create a new chain for redirecting inbound traffic to the common Envoy
102102
# port.
103-
# In the ctrlmesh_PROXY_INBOUND chain, '-j RETURN' bypasses Envoy and
104-
# '-j ctrlmesh_PROXY_TPROXY' redirects to Envoy.
105-
iptables -t mangle -N ctrlmesh_PROXY_TPROXY
106-
iptables -t mangle -A ctrlmesh_PROXY_TPROXY ! -d 127.0.0.1/32 -p tcp -j TPROXY --tproxy-mark "${INBOUND_TPROXY_MARK}"/0xffffffff --on-port "${PROXY_PORT}"
103+
# In the CTRLMESH_INBOUND chain, '-j RETURN' bypasses Envoy and
104+
# '-j CTRLMESH_TPROXY' redirects to Envoy.
105+
iptables -t mangle -N CTRLMESH_TPROXY
106+
iptables -t mangle -A CTRLMESH_TPROXY ! -d 127.0.0.1/32 -p tcp -j TPROXY --tproxy-mark "${INBOUND_TPROXY_MARK}"/0xffffffff --on-port "${PROXY_PORT}"
107107

108108
table=mangle
109109
else
110110
table=nat
111111
fi
112-
iptables -t "${table}" -N ctrlmesh_PROXY_INBOUND
113-
iptables -t "${table}" -A PREROUTING -p tcp -j ctrlmesh_PROXY_INBOUND
112+
iptables -t "${table}" -N CTRLMESH_INBOUND
113+
iptables -t "${table}" -A PREROUTING -p tcp -j CTRLMESH_INBOUND
114114

115115
if [ "${INBOUND_INTERCEPTION_MODE}" = "TPROXY" ]; then
116-
iptables -t mangle -A ctrlmesh_PROXY_INBOUND -p tcp --dport "${INBOUND_WEBHOOK_PORT}" -m socket -j ctrlmesh_PROXY_DIVERT || echo "No socket match support"
117-
iptables -t mangle -A ctrlmesh_PROXY_INBOUND -p tcp --dport "${INBOUND_WEBHOOK_PORT}" -m socket -j ctrlmesh_PROXY_DIVERT || echo "No socket match support"
118-
iptables -t mangle -A ctrlmesh_PROXY_INBOUND -p tcp --dport "${INBOUND_WEBHOOK_PORT}" -j ctrlmesh_PROXY_TPROXY
116+
iptables -t mangle -A CTRLMESH_INBOUND -p tcp --dport "${INBOUND_WEBHOOK_PORT}" -m socket -j CTRLMESH_DIVERT || echo "No socket match support"
117+
iptables -t mangle -A CTRLMESH_INBOUND -p tcp --dport "${INBOUND_WEBHOOK_PORT}" -m socket -j CTRLMESH_DIVERT || echo "No socket match support"
118+
iptables -t mangle -A CTRLMESH_INBOUND -p tcp --dport "${INBOUND_WEBHOOK_PORT}" -j CTRLMESH_TPROXY
119119
else
120-
iptables -t nat -A ctrlmesh_PROXY_INBOUND -p tcp --dport "${INBOUND_WEBHOOK_PORT}" -j ctrlmesh_PROXY_IN_REDIRECT
120+
iptables -t nat -A CTRLMESH_INBOUND -p tcp --dport "${INBOUND_WEBHOOK_PORT}" -j CTRLMESH_IN_REDIRECT
121121
fi
122122
fi
123123

124124
# Create a new chain for selectively redirecting outbound packets to Proxy.
125-
iptables -t nat -N ctrlmesh_PROXY_OUTPUT
125+
iptables -t nat -N CTRLMESH_OUTPUT
126126

127-
# Jump to the ctrlmesh_PROXY_OUTPUT chain from OUTPUT chain for all tcp traffic.
128-
iptables -t nat -A OUTPUT -p tcp -j ctrlmesh_PROXY_OUTPUT
127+
# Jump to the CTRLMESH_OUTPUT chain from OUTPUT chain for all tcp traffic.
128+
iptables -t nat -A OUTPUT -p tcp -j CTRLMESH_OUTPUT
129129

130130
for uid in ${PROXY_UID}; do
131131
# Avoid infinite loops. Don't redirect Proxy traffic directly back to
132132
# Proxy for non-loopback traffic.
133-
iptables -t nat -A ctrlmesh_PROXY_OUTPUT -m owner --uid-owner "${uid}" -j RETURN
133+
iptables -t nat -A CTRLMESH_OUTPUT -m owner --uid-owner "${uid}" -j RETURN
134134
done
135135

136136
# Redirect all apiserver outbound traffic to Proxy.
137-
iptables -t nat -A ctrlmesh_PROXY_OUTPUT -d "${KUBERNETES_SERVICE_HOST}" -j ctrlmesh_PROXY_REDIRECT
137+
iptables -t nat -A CTRLMESH_OUTPUT -d "${KUBERNETES_SERVICE_HOST}" -j CTRLMESH_REDIRECT
138138

139139
# Generate certs
140140
mount -o remount,rw "${SA_DIR}"

‎pkg/apis/ctrlmesh/types.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,13 @@ const (
3131

3232
// Labels
3333
const (
34-
CtrlmeshControlPrefix = "ctrlmesh.kusionstack.io/"
35-
CtrlmeshIgnoreWebhookLabel = "ctrlmesh.kusionstack.io/ignore-webhook"
36-
CtrlmeshIgnoreValidateLabel = "ctrlmesh.kusionstack.io/ignore-validate"
37-
CtrlmeshDefaultReplicasLabel = "ctrlmesh.kusionstack.io/default-replicas"
38-
CtrlmeshEnableProxyLabel = "ctrlmesh.kusionstack.io/enable-proxy"
34+
CtrlmeshControlPrefix = "ctrlmesh.kusionstack.io/"
35+
CtrlmeshIgnoreWebhookLabel = "ctrlmesh.kusionstack.io/ignore-webhook"
36+
CtrlmeshIgnoreValidateLabel = "ctrlmesh.kusionstack.io/ignore-validate"
37+
CtrlmeshDefaultReplicasLabel = "ctrlmesh.kusionstack.io/default-replicas"
38+
CtrlmeshEnableProxyLabel = "ctrlmesh.kusionstack.io/enable-proxy"
39+
CtrlmeshEnableIptableMode = "ctrlmesh.kusionstack.io/enable-iptables"
40+
3941
CtrlmeshAutoShardingRootLabel = "ctrlmesh.kusionstack.io/auto-sharding-root"
4042
CtrlmeshInRollingLabel = "ctrlmesh.kusionstack.io/rolling"
4143
CtrlmeshDisableFakeKubeconfigArgLabel = "ctrlmesh.kusionstack.io/disable-fake-kubeconfig-arg"

‎pkg/cmd/proxy/main.go‎

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ import (
3535

3636
"github.com/KusionStack/controller-mesh/pkg/apis/ctrlmesh/constants"
3737
"github.com/KusionStack/controller-mesh/pkg/client"
38-
3938
proxyapiserver "github.com/KusionStack/controller-mesh/pkg/proxy/apiserver"
4039
proxycache "github.com/KusionStack/controller-mesh/pkg/proxy/cache"
4140
"github.com/KusionStack/controller-mesh/pkg/proxy/circuitbreaker"
@@ -56,6 +55,8 @@ var (
5655
webhookCertDir = flag.String(constants.ProxyWebhookCertDirFlag, "", "The directory where the webhook certs generated or mounted.")
5756

5857
proxyIptablePort = flag.Int(constants.ProxyIptablesFlag, constants.ProxyIptablesPort, "port that http-tproxy listens on")
58+
59+
enableIpTable = os.Getenv(constants.EnvIPTable) == "true"
5960
)
6061

6162
func main() {
@@ -66,7 +67,17 @@ func main() {
6667
klog.Fatalf("Environment %s=%s %s=%s not exist.",
6768
constants.EnvPodNamespace, os.Getenv(constants.EnvPodNamespace), constants.EnvPodName, os.Getenv(constants.EnvPodName))
6869
}
69-
cfg := ctrl.GetConfigOrDie()
70+
var cfg *rest.Config
71+
72+
if enableIpTable {
73+
var err error
74+
cfg, err = getRestConfig()
75+
if err != nil {
76+
klog.Fatalf("Failed to get rest config: %v", err)
77+
}
78+
} else {
79+
cfg = ctrl.GetConfigOrDie()
80+
}
7081
cfg.UserAgent = "ctrlmesh"
7182
if err := client.NewRegistry(cfg); err != nil {
7283
klog.Fatalf("Failed to new client registry: %v", err)
@@ -165,3 +176,41 @@ func serveHTTP(ctx context.Context, readyHandler *healthz.Handler) {
165176
klog.Fatalf("Serve HTTP shutting down on :%d: %v", *metricsHealthPort, err)
166177
}
167178
}
179+
180+
func getRestConfig() (*rest.Config, error) {
181+
const (
182+
tokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
183+
//rootCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/..data/ca.crt"
184+
)
185+
host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")
186+
if len(host) == 0 || len(port) == 0 {
187+
return nil, rest.ErrNotInCluster
188+
}
189+
190+
token, err := os.ReadFile(tokenFile)
191+
if err != nil {
192+
return nil, err
193+
}
194+
195+
tlsClientConfig := rest.TLSClientConfig{Insecure: true}
196+
197+
//if _, err := certutil.NewPool(rootCAFile); err != nil {
198+
// klog.Errorf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err)
199+
//} else {
200+
// tlsClientConfig.CAFile = rootCAFile
201+
//}
202+
203+
cfg := &rest.Config{
204+
// TODO: switch to using cluster DNS.
205+
Host: "https://" + net.JoinHostPort(host, port),
206+
TLSClientConfig: tlsClientConfig,
207+
BearerToken: string(token),
208+
BearerTokenFile: tokenFile,
209+
210+
Burst: 3000,
211+
QPS: 2000.0,
212+
}
213+
klog.V(3).Infof("Starting with rest config: %v", utils.DumpJSON(cfg))
214+
215+
return cfg, nil
216+
}

‎pkg/webhook/pod/injector.go‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ func (h *MutatingHandler) injectByShardingConfig(ctx context.Context, pod *v1.Po
131131
if *proxyImage == "" {
132132
return fmt.Errorf("the images for ControllerMesh init or proxy container have not set in args")
133133
}
134-
134+
enableIpTable := pod.Labels[ctrlmesh.CtrlmeshEnableIptableMode] == "true"
135135
imagePullPolicy := v1.PullAlways
136136
if *proxyImagePullPolicy == string(v1.PullIfNotPresent) {
137137
imagePullPolicy = v1.PullIfNotPresent
@@ -177,6 +177,13 @@ func (h *MutatingHandler) injectByShardingConfig(ctx context.Context, pod *v1.Po
177177
},
178178
}
179179

180+
if enableIpTable {
181+
proxyContainer.Env = append(proxyContainer.Env, v1.EnvVar{
182+
Name: constants.EnvIPTable,
183+
Value: "true",
184+
})
185+
}
186+
180187
if val, ok := pod.Annotations[ctrlmesh.CtrlmeshProxyContainerResourceAnno]; ok {
181188
req := &v1.ResourceRequirements{}
182189
if err := json.Unmarshal([]byte(val), req); err != nil {
@@ -213,15 +220,6 @@ func (h *MutatingHandler) injectByShardingConfig(ctx context.Context, pod *v1.Po
213220
proxyContainer.Env = append(proxyContainer.Env, apiserverHostPortEnvs...)
214221
}
215222

216-
ipTableEnvs := getEnv(pod, constants.EnvIPTable)
217-
enableIpTable := false
218-
if len(ipTableEnvs) > 0 {
219-
initContainer.Env = append(initContainer.Env, ipTableEnvs...)
220-
//proxyContainer.Env = append(proxyContainer.Env, ipTableEnvs...)
221-
if ipTableEnvs[0].Value == "true" {
222-
enableIpTable = true
223-
}
224-
}
225223
if !enableIpTable {
226224
if err := h.applyFakeConfigMap(pod); err != nil {
227225
return err
@@ -271,7 +269,7 @@ func (h *MutatingHandler) injectByShardingConfig(ctx context.Context, pod *v1.Po
271269
proxyContainer.VolumeMounts = append(proxyContainer.VolumeMounts, certVolumeMounts[0])
272270
}
273271
}
274-
if *initImage != "" {
272+
if enableIpTable && *initImage != "" {
275273
pod.Spec.InitContainers = append([]v1.Container{*initContainer}, pod.Spec.InitContainers...)
276274
}
277275
if pod.Labels == nil {

0 commit comments

Comments
 (0)