一台服务器通过多个网卡连接多个网络，需要确保不同网络发来的数据能够按照原路返回，在具有多个网络接口的CentOS服务器上，除非定义了策略路由，否则入站数据包将发送到默认接口上。

模拟一种场景：服务器上有三个网口并接入三个不同运营商的网络，通过原线路返回的策略路由实现多线多IP同时在线，以实现不同运营商用户访问其对应的网络线路，同时可让从同一运营商过来的请求由原运营商线路返回，比如：电信IP过来的请求按照电信路由返回，从联通IP过来的求从联通路由返回，来减少网络延时

拓扑为模拟环境

内网 172.16.2.254  网关 172.16.2.2  网卡eth0

电信 22.22.22.22   网关 22.22.22.1   网卡eth1

联通 33.33.33.33   网关 33.33.33.1   网卡eth2

移动 44.44.44.44   网关 44.44.44.1   网卡eth3

禁用网卡预测命名规则

mv /etc/sysconfig/network-scripts/ifcfg-ens33 /etc/sysconfig/network-scripts/ifcfg-eth0
mv /etc/sysconfig/network-scripts/ifcfg-ens34 /etc/sysconfig/network-scripts/ifcfg-eth1
mv /etc/sysconfig/network-scripts/ifcfg-ens35 /etc/sysconfig/network-scripts/ifcfg-eth2
mv /etc/sysconfig/network-scripts/ifcfg-ens36 /etc/sysconfig/network-scripts/ifcfg-eth3
sed -e 's/\<quiet\>/& net.ifnames=0 biosdevname=0 ipv6.disable=1/' -i /etc/default/grub
grub2-mkconfig -o /boot/grub2/grub.cfg
reboot

修改网卡名称

sed -i 's/^NAME.*$/NAME=LAN/g' /etc/sysconfig/network-scripts/ifcfg-eth0
sed -i 's/DEVICE.*$/DEVICE=eth0/g' /etc/sysconfig/network-scripts/ifcfg-eth0
sed -i 's/^NAME.*$/NAME=Telecom/g' /etc/sysconfig/network-scripts/ifcfg-eth1
sed -i 's/DEVICE.*$/DEVICE=eth1/g' /etc/sysconfig/network-scripts/ifcfg-eth1
sed -i 's/^NAME.*$/NAME=Unicom/g' /etc/sysconfig/network-scripts/ifcfg-eth2
sed -i 's/DEVICE.*$/DEVICE=eth2/g' /etc/sysconfig/network-scripts/ifcfg-eth2
sed -i 's/^NAME.*$/NAME=Mobile/g' /etc/sysconfig/network-scripts/ifcfg-eth3
sed -i 's/DEVICE.*$/DEVICE=eth3/g' /etc/sysconfig/network-scripts/ifcfg-eth3
systemctl restart NetworkManager
nmcli connection show

修改IP地址

nmcli connection modify LAN ipv4.method manual ipv4.addresses "172.16.2.254/24" ipv4.gateway 172.16.2.2 ipv4.dns "1.2.4.8" ipv6.method ignore connection.autoconnect yes
nmcli connection modify Telecom ipv4.method manual ipv4.addresses "22.22.22.22/24" ipv4.dns "144.144.144.144" ipv6.method ignore connection.autoconnect yes
nmcli connection modify Unicom ipv4.method manual ipv4.addresses "33.33.33.33/24" ipv4.dns "223.5.5.5" ipv6.method ignore connection.autoconnect yes
nmcli connection modify Mobile ipv4.method manual ipv4.addresses "44.44.44.44/24" ipv4.dns "180.76.76.76" ipv6.method ignore connection.autoconnect yes
nmcli connection reload 
reboot

安装组件

yum install network-scripts -y
yum install NetworkManager-config-routing-rules -y
systemctl enable NetworkManager-dispatcher.service
systemctl start NetworkManager-dispatcher.service

为接口创建新的路由表条目

#路由策略数据库控制了kernel搜素多个路由表之间的顺序，每一条rule规则都可以定义一个0到32767之间的优先级，数字越小优先级越高
cp -p /etc/iproute2/rt_tables /etc/iproute2/rt_tables.bkp
echo "#Dual Gateway" >> /etc/iproute2/rt_tables
echo '100     LAN' >> /etc/iproute2/rt_tables
echo '101     Telecom' >> /etc/iproute2/rt_tables
echo '102     Unicom' >> /etc/iproute2/rt_tables
echo '103     Mobile' >> /etc/iproute2/rt_tables

调整内核参数

echo "net.ipv4.conf.default.arp_filter = 2" >> /etc/sysctl.conf
echo "net.ipv4.conf.all.arp_announce = 2" >> /etc/sysctl.conf
echo "net.ipv4.conf.default.arp_announce = 2" >> /etc/sysctl.conf
echo "net.ipv4.conf.default.rp_filter = 2" >> /etc/sysctl.conf
echo "net.ipv4.conf.all.rp_filter = 2" >> /etc/sysctl.conf
echo "net.ipv4.conf.eth0.rp_filter = 2" >> /etc/sysctl.conf
echo "net.ipv4.conf.eth1.rp_filter = 2" >> /etc/sysctl.conf
echo "net.ipv4.conf.eth2.rp_filter = 2" >> /etc/sysctl.conf
echo "net.ipv4.conf.eth3.rp_filter = 2" >> /etc/sysctl.conf

echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf
echo "net.ipv4.tcp_syncookies = 1" >> /etc/sysctl.conf
echo "net.ipv4.tcp_window_scaling = 1" >> /etc/sysctl.conf
echo "net.core.rmem_max = 270532608" >> /etc/sysctl.conf
echo "net.core.wmem_max = 270532608" >> /etc/sysctl.conf
echo "net.ipv4.tcp_rmem = 4096        87380   270532608" >> /etc/sysctl.conf
echo "net.ipv4.tcp_wmem = 4096        16384   270532608" >> /etc/sysctl.conf
echo "net.core.somaxconn = 50000" >> /etc/sysctl.conf
echo "net.ipv4.tcp_max_tw_buckets = 1440000" >> /etc/sysctl.conf
echo "net.ipv4.tcp_max_syn_backlog = 3240000" >> /etc/sysctl.conf
echo "net.ipv4.tcp_keepalive_time = 600" >> /etc/sysctl.conf
echo "net.ipv4.tcp_keepalive_intvl = 10" >> /etc/sysctl.conf
echo "net.ipv4.tcp_keepalive_probes = 9" >> /etc/sysctl.conf
echo "net.ipv4.tcp_fin_timeout = 7" >> /etc/sysctl.conf

执行sysctl -p

指定每个接口所需的路由和规则

#内网
echo "default via 172.16.2.2 dev eth0 table LAN" >> /etc/sysconfig/network-scripts/route-eth0
echo "from 172.16.2.0/24 table LAN" >> /etc/sysconfig/network-scripts/route-eth0

#电信线路
echo "default via 22.22.22.1 src 22.22.22.22 dev eth1 table Telecom" > /etc/sysconfig/network-scripts/route-eth1
echo "from 22.22.22.22 table Telecom" > /etc/sysconfig/network-scripts/rule-eth1

#联通线路
echo "default via 33.33.33.1 src 33.33.33.33 dev eth2 table Unicom" > /etc/sysconfig/network-scripts/route-eth2
echo "from 33.33.33.33 table Unicom" > /etc/sysconfig/network-scripts/rule-eth2

#移动线路
echo "default via 44.44.44.1 src 44.44.44.44 dev eth3 table Mobile" > /etc/sysconfig/network-scripts/route-eth3
echo "from 44.44.44.44 table Mobile" > /etc/sysconfig/network-scripts/rule-eth3

启用修改的规则和路由文件

nmcli connection reload
service network restart

刷新路由表

ip route flush table LAN
ip route flush table Telecom 
ip route flush table Unicom
ip route flush table Mobile

查看路由表

ip rule
ip route list table Telecom

测试

ip route get 8.8.8.8 from 22.22.22.22
ip route get 8.8.8.8 from 33.33.33.33
ip route get 8.8.8.8 from 44.44.44.44