veth虚拟网卡
2020-09-22 本文已影响0人
分享放大价值
veth是kernel提供的一种虚拟网卡，总是成对出现。在一端发送数据，就可以在另一端接收到，类似一根网线。那么它是如何实现的呢？今天就看一下它的实现。
veth创建
可以使用ip命令创建，如下。
[root@localhost ~]# ip link add vetha type veth peer name vethb
在kernel端需要提前加载veth module，如下，这个module做的事情很简单，就是注册一个 rtnl_link_ops
[root@localhost ~]# modprobe veth
[root@localhost ~]# lsmod | grep veth
veth 13410 0
//veth.ko 初始化
#define DRV_NAME "veth"
static struct rtnl_link_ops veth_link_ops = {
.kind = DRV_NAME,
.priv_size = sizeof(struct veth_priv),
.setup = veth_setup,
.validate = veth_validate,
.newlink = veth_newlink,
.dellink = veth_dellink,
.policy = veth_policy,
.maxtype = VETH_INFO_MAX,
};
/*
* init/fini
*/
static __init int veth_init(void)
{
return rtnl_link_register(&veth_link_ops);
}
int rtnl_link_register(struct rtnl_link_ops *ops)
{
int err;
rtnl_lock();
err = __rtnl_link_register(ops);
rtnl_unlock();
return err;
}
int __rtnl_link_register(struct rtnl_link_ops *ops)
{
if (rtnl_link_ops_get(ops->kind))
return -EEXIST;
/* The check for setup is here because if ops
* does not have that filled up, it is not possible
* to use the ops for creating device. So do not
* fill up dellink as well. That disables rtnl_dellink.
*/
if (ops->setup && !ops->dellink)
ops->dellink = unregister_netdevice_queue;
list_add_tail(&ops->list, &link_ops);
return 0;
}
通过命令 ip link add vetha type veth peer name vethb 创建veth时，在kernel中调用rtnl_newlink，会根据传入的type查找rtnl_link_ops，再调用rtnl_link_ops的newlink创建veth的peer，并将两端veth分别放在对方的私有数据中。
rtnl_newlink
if (linkinfo[IFLA_INFO_KIND]) {
nla_strlcpy(kind, linkinfo[IFLA_INFO_KIND], sizeof(kind));
ops = rtnl_link_ops_get(kind);
struct net_device *dev;
dev = rtnl_create_link(dest_net, ifname, name_assign_type, ops, tb);
if (ops->newlink) {
err = ops->newlink(net, dev, tb, data);
static int veth_newlink(struct net *src_net, struct net_device *dev,
struct nlattr *tb[], struct nlattr *data[])
struct net_device *peer;
peer = rtnl_create_link(net, ifname, name_assign_type,
&veth_link_ops, tbp);
register_netdevice(peer);
register_netdevice(dev);
/*
* tie the deviced together
*/
//这里是关键，将peer放在dev的priv中，将dev放在peer的priv中，
//这样将这两个虚拟设备绑定到一起。
priv = netdev_priv(dev);
rcu_assign_pointer(priv->peer, peer);
priv = netdev_priv(peer);
rcu_assign_pointer(priv->peer, dev);
static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
//取出peer设备
struct veth_priv *priv = netdev_priv(dev);
rcv = rcu_dereference(priv->peer);
//调用 dev_forward_skb 发送数据时，已经换成peer设备rcv了
//相当于 peer 设备的接收
if (likely(dev_forward_skb(rcv, skb) == NET_RX_SUCCESS)) {
struct pcpu_vstats *stats = this_cpu_ptr(dev->vstats);
u64_stats_update_begin(&stats->syncp);
stats->bytes += length;
stats->packets++;
u64_stats_update_end(&stats->syncp);
} else {
drop:
atomic64_inc(&priv->dropped);
}
rcu_read_unlock();
return NETDEV_TX_OK;
}
//如果报文可转发，则调用netif_rx_internal将报文送入主机协议栈
int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
{
return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
}
int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
{
if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
atomic_long_inc(&dev->rx_dropped);
kfree_skb(skb);
return NET_RX_DROP;
}
}
if (unlikely(!is_skb_forwardable(dev, skb))) {
atomic_long_inc(&dev->rx_dropped);
kfree_skb(skb);
return NET_RX_DROP;
}
skb_scrub_packet(skb, true);
skb->protocol = eth_type_trans(skb, dev);
skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
return 0;
}
//根据目的mac设备 pkt_type
__be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev)
{
unsigned short _service_access_point;
const unsigned short *sap;
const struct ethhdr *eth;
skb->dev = dev;
skb_reset_mac_header(skb);
skb_pull_inline(skb, ETH_HLEN);
eth = eth_hdr(skb);
if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
if (ether_addr_equal_64bits(eth->h_dest, dev->broadcast))
skb->pkt_type = PACKET_BROADCAST;
else
skb->pkt_type = PACKET_MULTICAST;
}
else if (unlikely(!ether_addr_equal_64bits(eth->h_dest,
dev->dev_addr)))
skb->pkt_type = PACKET_OTHERHOST;
....
}
veth使用
我见过的使用方法有两种，如下
a. 一端在 root namespace，另一端放在其他 namespace，连接两个namespace，比如在k8s中，calico, cilium和ovs这些cni都是如此实现的。
b.两端分别放在两个网桥上，连接网桥。
第二种情况比较简单，重点说一下第一种情况遇到的几个问题。
实验步骤如下，
创建一对veth口，vetha和vethb，
创建一个namespace test，
将vetha放入namespace test，
将vetha和vethb都up起来，
给vetha配置ip 1.1.1.2
[root@localhost ~]# ip link add vetha type veth peer name vethb
[root@localhost ~]# ip link set dev vethb up
[root@localhost ~]# ip netns add test
[root@localhost ~]# ip link set dev vetha netns test
[root@localhost ~]# ip netns exec test ip link set dev vetha up
[root@localhost ~]# ip netns exec test ip address add dev vetha 1.1.1.2/24
[root@localhost ~]# ip netns exec test ip a
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
30: vetha@if29: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP qlen 1000
link/ether 96:7f:a6:ea:93:23 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 1.1.1.2/24 scope global vetha
valid_lft forever preferred_lft forever
inet6 fe80::74ef:e3ff:fe5d:2db0/64 scope link
valid_lft forever preferred_lft forever
[root@localhost ~]# ip netns exec test ip r
1.1.1.0/24 dev vetha proto kernel scope link src 1.1.1.2
[root@localhost ~]# ip netns exec test arp -n
Address HWtype HWaddress Flags Mask Iface
1.1.1.4 (incomplete) vetha
[root@localhost ~]# ifconfig vethb
vethb: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet6 fe80::947f:a6ff:feea:9321 prefixlen 64 scopeid 0x20<link>
ether 96:7f:a6:ea:93:21 txqueuelen 1000 (Ethernet)
RX packets 208784 bytes 12836250 (12.2 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 1143 bytes 62469 (61.0 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
问题1：如果在test namespace中，设置一个静态arp，ping一个不存在的地址1.1.1.4
[root@localhost ~]# ip netns exec test arp -s 1.1.1.4 00:00:00:00:00:01
[root@localhost ~]# ip netns exec test arp -n
Address HWtype HWaddress Flags Mask Iface
1.1.1.4 ether 00:00:00:00:00:01 CM vetha
icmp报文到达vethb设备后，走host的协议栈，匹配到host的默认路由表，应该会从em1发出去，但是结果是vethb可以抓到icmp报文，em1抓不到。
[root@localhost ~]# ip r
default via 10.164.129.1 dev em1 proto static metric 100
10.10.10.0/24 dev gre10 proto kernel scope link src 10.10.10.1
10.164.129.0/24 dev em1 proto kernel scope link src 10.164.129.16 metric 100
[root@localhost ~]# tcpdump -vne -i vethb icmp
tcpdump: listening on vethb, link-type EN10MB (Ethernet), capture size 262144 bytes
08:04:49.386991 f6:d4:d2:de:20:be > 00:00:00:00:00:01, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 48297, offset 0, flags [DF], proto ICMP (1), length 84)
1.1.1.2 > 1.1.1.4: ICMP echo request, id 30782, seq 21, length 64
08:04:50.386985 f6:d4:d2:de:20:be > 00:00:00:00:00:01, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 48427, offset 0, flags [DF], proto ICMP (1), length 84)
1.1.1.2 > 1.1.1.4: ICMP echo request, id 30782, seq 22, length 64
^C
2 packets captured
2 packets received by filter
0 packets dropped by kernel
[root@localhost ~]# tcpdump -vne -i em1 icmp
tcpdump: listening on em1, link-type EN10MB (Ethernet), capture size 262144 bytes
^C
0 packets captured
0 packets received by filter
0 packets dropped by kernel
原因是icmp报文的目的mac为00:00:00:00:00:01，而在vethb收到此报文后，在函数eth_type_trans中会根据目的mac给skb->pkt_type赋值，因为目的mac不为vethb的mac，所以skb->pkt_type被设置成PACKET_OTHERHOST。
veth_xmit ->dev_forward_skb -> __dev_forward_skb -> eth_type_trans
if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
if (ether_addr_equal_64bits(eth->h_dest, dev->broadcast))
skb->pkt_type = PACKET_BROADCAST;
else
skb->pkt_type = PACKET_MULTICAST;
}
else if (unlikely(!ether_addr_equal_64bits(eth->h_dest,
dev->dev_addr)))
skb->pkt_type = PACKET_OTHERHOST;
随后将报文送入主机协议栈，在ip_rcv中有个判断如果skb->pkt_type == PACKET_OTHERHOST就直接drop报文，比较恶心的是，这个drop没有统计信息可看。
看来得在test namespace中将1.1.1.4对应的mac设置为vethb的mac地址。
netif_rx_internal -> enqueue_to_backlog -> process_backlog ->__netif_receive_skb -> __netif_receive_skb_core -> ip_rcv
int ip_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev)
{
const struct iphdr *iph;
u32 len;
/* When the interface is in promisc. mode, drop all the crap
* that it receives, do not try to analyse it.
*/
if (skb->pkt_type == PACKET_OTHERHOST)
goto drop;
drop:
kfree_skb(skb);
out:
return NET_RX_DROP;
}
问题2: 如下，将1.1.1.4对应的mac修改为vethb的mac了，但是仍然有问题，vethb可以收到，em1收不到。
[root@localhost ~]# ip link show dev vethb
33: vethb@if34: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT qlen 1000
link/ether 96:ec:6a:a8:67:ed brd ff:ff:ff:ff:ff:ff link-netnsid 0
[root@localhost ~]# ip netns exec test arp -d 1.1.1.4
[root@localhost ~]# ip netns exec test arp -s 1.1.1.4 96:ec:6a:a8:67:ed
[root@localhost ~]# ip netns exec test arp -n
Address HWtype HWaddress Flags Mask Iface
1.1.1.4 ether 96:ec:6a:a8:67:ed CM vetha
[root@localhost ~]# ip link show dev vethb
33: vethb@if34: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT qlen 1000
link/ether 96:ec:6a:a8:67:ed brd ff:ff:ff:ff:ff:ff link-netnsid 0
[root@localhost ~]# tcpdump -vne -i vethb icmp
tcpdump: listening on vethb, link-type EN10MB (Ethernet), capture size 262144 bytes
08:15:21.495979 f6:d4:d2:de:20:be > 96:ec:6a:a8:67:ed, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 23666, offset 0, flags [DF], proto ICMP (1), length 84)
1.1.1.2 > 1.1.1.4: ICMP echo request, id 31770, seq 32, length 64
^C
1 packet captured
1 packet received by filter
0 packets dropped by kernel
[root@localhost ~]# tcpdump -vne -i em1 icmp
tcpdump: listening on em1, link-type EN10MB (Ethernet), capture size 262144 bytes
^C
0 packets captured
0 packets received by filter
0 packets dropped by kernel
问题3: 这又是另一个问题了，在调用ip_route_input_noref查找路由时，虽然可以匹配到默认路由，但是因为 vethb 没有开启forward功能，所以仍然会失败。
ip_route_input_noref -> ip_route_input_slow
fl4.flowi4_oif = 0;
fl4.flowi4_iif = dev->ifindex;
fl4.flowi4_mark = skb->mark;
fl4.flowi4_tos = tos;
fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
fl4.daddr = daddr;
fl4.saddr = saddr;
err = fib_lookup(net, &fl4, &res);
if (!IN_DEV_FORWARD(in_dev)) {
err = -EHOSTUNREACH;
goto no_route;
}
no_route:
RT_CACHE_STAT_INC(in_no_route);
res.type = RTN_UNREACHABLE;
res.fi = NULL;
可以通过下面的命令查看丢包计数 in_no_route
cat /proc/net/stat/rt_cache | awk -F " " '{print $5}'
接下来使能vethb的forwarding试试看
[root@localhost ~]# echo 1 > /proc/sys/net/ipv4/conf/vethb/forwarding
[root@localhost ~]# cat /proc/sys/net/ipv4/conf/vethb/forwarding
1
注意：如果 /proc/sys/net/ipv4/conf/all/forwarding 使能了，则新创建的网卡的forwarding 功能都会默认被使能。
问题4: 再次ping还是不行，这是由于反向路径检查失败导致的。会调用fib_validate_source使用报文的源ip作为目的查找路由表，只能匹配到默认路由，因为默认路由出接口和报文入接口不是同一个，所以判断失败。收发同一个报文应该是同一个设备，这称为对称路由。
ip_route_input_slow -> __mkroute_input
/* Ignore rp_filter for packets protected by IPsec. */
err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
in_dev->dev, in_dev, &itag);
if (err < 0) {
ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
saddr);
goto cleanup;
}
int fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst,
u8 tos, int oif, struct net_device *dev,
struct in_device *idev, u32 *itag)
{
int r = secpath_exists(skb) ? 0 : IN_DEV_RPFILTER(idev);
//反向路径检查开关，为0就不做检查
if (!r && !fib_num_tclassid_users(dev_net(dev)) &&
IN_DEV_ACCEPT_LOCAL(idev) &&
(dev->ifindex != oif || !IN_DEV_TX_REDIRECTS(idev))) {
*itag = 0;
return 0;
}
return __fib_validate_source(skb, src, dst, tos, oif, dev, r, idev, itag);
}
static int __fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst,
u8 tos, int oif, struct net_device *dev,
int rpf, struct in_device *idev, u32 *itag)
{
int ret, no_addr;
struct fib_result res;
struct flowi4 fl4;
struct net *net;
bool dev_match;
fl4.flowi4_oif = 0;
fl4.flowi4_iif = oif ? : LOOPBACK_IFINDEX;
fl4.daddr = src;
fl4.saddr = dst;
fl4.flowi4_tos = tos;
fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
no_addr = idev->ifa_list == NULL;
fl4.flowi4_mark = IN_DEV_SRC_VMARK(idev) ? skb->mark : 0;
net = dev_net(dev);
if (fib_lookup(net, &fl4, &res))
goto last_resort;
if (res.type != RTN_UNICAST &&
(res.type != RTN_LOCAL || !IN_DEV_ACCEPT_LOCAL(idev)))
goto e_inval;
if (!rpf && !fib_num_tclassid_users(dev_net(dev)) &&
(dev->ifindex != oif || !IN_DEV_TX_REDIRECTS(idev)))
goto last_resort;
fib_combine_itag(itag, &res);
dev_match = false;
#ifdef CONFIG_IP_ROUTE_MULTIPATH
for (ret = 0; ret < res.fi->fib_nhs; ret++) {
struct fib_nh *nh = &res.fi->fib_nh[ret];
if (nh->nh_dev == dev) {
dev_match = true;
break;
}
}
#else
//如果路由中的出接口是入接口才会成功。否则就是反向路径检查失败。
if (FIB_RES_DEV(res) == dev)
dev_match = true;
#endif
if (dev_match) {
ret = FIB_RES_NH(res).nh_scope >= RT_SCOPE_HOST;
return ret;
}
if (no_addr)
goto last_resort;
if (rpf == 1)
goto e_rpf;
fl4.flowi4_oif = dev->ifindex;
ret = 0;
if (fib_lookup(net, &fl4, &res) == 0) {
if (res.type == RTN_UNICAST)
ret = FIB_RES_NH(res).nh_scope >= RT_SCOPE_HOST;
}
return ret;
last_resort:
if (rpf)
goto e_rpf;
*itag = 0;
return 0;
e_inval:
return -EINVAL;
e_rpf:
return -EXDEV;
}
可以通过下面的命令查看drop计数 in_martian_src
[root@localhost ~]# cat /proc/net/stat/rt_cache | awk -F " " '{print $8}'
这个问题的解决办法有两个
a. 添加对称路由
b. 关闭反向路径检查
a. 添加对称路由，如下添加到test namespace路由后，icmp报文可以从em1发出去了
[root@localhost ~]# ip route add 1.1.1.2 dev vethb
[root@localhost ~]# ip r
default via 10.164.129.1 dev em1 proto static metric 100
1.1.1.2 dev vethb scope link
10.10.10.0/24 dev gre10 proto kernel scope link src 10.10.10.1
10.164.129.0/24 dev em1 proto kernel scope link src 10.164.129.16 metric 100
169.254.0.0/16 dev provisioning_nw scope link metric 1016
169.254.0.0/16 dev idrac_nw scope link metric 1017
192.168.0.0/24 dev provisioning_nw proto kernel scope link src 192.168.0.253
192.168.10.0/24 dev idrac_nw proto kernel scope link src 192.168.10.13
192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1
[root@localhost ~]# tcpdump -vne -i em1 icmp or arp
tcpdump: listening on em1, link-type EN10MB (Ethernet), capture size 262144 bytes
08:56:10.509045 90:b1:1c:55:37:1e > 00:00:0c:07:ac:02, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 28970, offset 0, flags [DF], proto ICMP (1), length 84)
1.1.1.2 > 1.1.1.4: ICMP echo request, id 1009, seq 1664, length 64
08:56:11.509051 90:b1:1c:55:37:1e > 00:00:0c:07:ac:02, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 29200, offset 0, flags [DF], proto ICMP (1), length 84)
1.1.1.2 > 1.1.1.4: ICMP echo request, id 1009, seq 1665, length 64
b. 关闭反向路径检查
rp_filter取决于设备和all的最大值，所以必须把设备和all的rp_filter都关闭
#define IN_DEV_RPFILTER(in_dev) IN_DEV_MAXCONF((in_dev), RP_FILTER)
#define IN_DEV_MAXCONF(in_dev, attr) \
(max(IPV4_DEVCONF_ALL(dev_net(in_dev->dev), attr), \
IN_DEV_CONF_GET((in_dev), attr)))
[root@localhost ~]# ip route del 1.1.1.2 dev vethb
[root@localhost ~]# echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter
[root@localhost ~]# echo 0 > /proc/sys/net/ipv4/conf/vethb/rp_filter
[root@localhost ~]# tcpdump -vne -i em1 icmp or arp
tcpdump: listening on em1, link-type EN10MB (Ethernet), capture size 262144 bytes
09:01:22.555047 90:b1:1c:55:37:1e > 00:00:0c:07:ac:02, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 58344, offset 0, flags [DF], proto ICMP (1), length 84)
1.1.1.2 > 1.1.1.4: ICMP echo request, id 1009, seq 1976, length 64
09:01:23.555046 90:b1:1c:55:37:1e > 00:00:0c:07:ac:02, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 58481, offset 0, flags [DF], proto ICMP (1), length 84)
1.1.1.2 > 1.1.1.4: ICMP echo request, id 1009, seq 1977, length 64
proxy_arp
到这里icmp报文算是成功发出去了，但是test namespace中1.1.1.4的mac地址是手动设置的，不太灵活，可以使用设备的 proxy_arp 功能。
//使能 proxy_arp
[root@localhost ~]# echo 1 > /proc/sys/net/ipv4/conf/vethb/proxy_arp
[root@localhost ~]# taskset -c 3 ip netns exec test ping 1.1.1.4
PING 1.1.1.4 (1.1.1.4) 56(84) bytes of data.
^C
--- 1.1.1.4 ping statistics ---
1 packets transmitted, 0 received, 100% packet loss, time 0ms
//学到了vethb的mac地址
[root@localhost ~]# ip netns exec test arp -n
Address HWtype HWaddress Flags Mask Iface
1.1.1.4 ether 96:ec:6a:a8:67:ed C vetha
但是在 arp 处理过程中，也会查找路由，反向路径检查等流程，所以上面的问题也都会遇到，按照上面的设置一下就行。
arp_process
if (arp->ar_op == htons(ARPOP_REQUEST) &&
ip_route_input_noref(skb, tip, sip, 0, dev) == 0) {
rt = skb_rtable(skb);
addr_type = rt->rt_type;
if (addr_type == RTN_LOCAL) {
...
} else if (IN_DEV_FORWARD(in_dev)) {
if (addr_type == RTN_UNICAST &&
(arp_fwd_proxy(in_dev, dev, rt) ||
arp_fwd_pvlan(in_dev, dev, rt, sip, tip) ||
(rt->dst.dev != dev &&
pneigh_lookup(&arp_tbl, net, &tip, dev, 0)))) {
n = neigh_event_ns(&arp_tbl, sha, &sip, dev);
if (n)
neigh_release(n);
if (NEIGH_CB(skb)->flags & LOCALLY_ENQUEUED ||
skb->pkt_type == PACKET_HOST ||
NEIGH_VAR(in_dev->arp_parms, PROXY_DELAY) == 0) {
arp_send(ARPOP_REPLY, ETH_P_ARP, sip,
dev, tip, sha, dev->dev_addr,
sha);
} else {
pneigh_enqueue(&arp_tbl,
in_dev->arp_parms, skb);
return 0;
}
goto out;
}
总结
a. 如果veth一端在其他namespace，另一端在root namespace，并没有被加入到网桥，则test namespace中通过veth发送的报文的目的mac如果是单播的，则必须是veth的peer设备的mac。可以在test namespace中静态配置或者设置代理arp
b. 为了通过反向路径检查，可以关闭反向路径检查或者设置对称路由。
c. 必须使能设备的 forwarding 功能。
用到的命令如下，其实在k8s的calico cni网络中，基本上就是下面的几个设置。
echo 1 > /proc/sys/net/ipv4/conf/vethb/proxy_arp
echo 1 > /proc/sys/net/ipv4/conf/vethb/forwarding
echo 0 > /proc/sys/net/ipv4/conf/vethb/rp_filter
ip route add 1.1.1.2 dev vethb
