IP multipathing with RHEL 5

In this page you can find sample working configs, which do following:

- setup two bond interfaces (bond0, bond1) with 802.3ad support
- setup two aliases for bond0
- setup two VLANs on bond1
- setup custom routes for these VLANs
- setup default gw trought bond0

All this we make on RedHat Linux 5.4, server is SunFire x4150 with 4x onboard ethernet devices.

Basic config:

[root@dtgs1 network-scripts]# cat /etc/modprobe.conf 
alias eth0 e1000e
alias eth1 e1000e
alias eth2 e1000e
alias eth3 e1000e
alias scsi_hostadapter mptbase
alias scsi_hostadapter1 mptsas
alias scsi_hostadapter2 ata_piix
alias scsi_hostadapter3 ahci
alias scsi_hostadapter4 usb-storage
alias bond0 bonding
alias bond1 bonding
options bond0 mode=4 miimon=100
options bond1 mode=4 miimon=100

[root@dtgs1 network-scripts]# cat /etc/sysconfig/network
NETWORKING=yes
NETWORKING_IPV6=no
HOSTNAME=dtgs1
GATEWAYDEV=bond0

[root@dtgs1 network-scripts]# cat ifcfg-eth0
# Intel Corporation 80003ES2LAN Gigabit Ethernet Controller (Copper)
DEVICE=eth0
HWADDR=00:1e:68:2f:04:dc
ONBOOT=yes
BOOTPROTO=none
TYPE=Ethernet
MASTER=bond0
SLAVE=yes

[root@dtgs1 network-scripts]# cat ifcfg-eth1
# Intel Corporation 80003ES2LAN Gigabit Ethernet Controller (Copper)
DEVICE=eth1
HWADDR=00:1e:68:2f:04:dd
ONBOOT=yes
BOOTPROTO=none
TYPE=Ethernet
MASTER=bond0
SLAVE=yes

[root@dtgs1 network-scripts]# cat ifcfg-eth2
# Intel Corporation 82571EB Gigabit Ethernet Controller
DEVICE=eth2
HWADDR=00:1e:68:2f:04:de
ONBOOT=yes
BOOTPROTO=none
TYPE=Ethernet
MASTER=bond1
SLAVE=yes

[root@dtgs1 network-scripts]# cat ifcfg-eth3
# Intel Corporation 82571EB Gigabit Ethernet Controller
DEVICE=eth3
HWADDR=00:1e:68:2f:04:df
ONBOOT=yes
BOOTPROTO=none
TYPE=Ethernet
MASTER=bond1
SLAVE=yes

[root@dtgs1 network-scripts]# cat ifcfg-bond0
DEVICE=bond0
BOOTPROTO=none
ONBOOT=yes
IPADDR=10.128.50.11
NETMASK=255.255.255.0
GATEWAY=10.128.50.1
USERCTL=no

[root@dtgs1 network-scripts]# cat ifcfg-bond0\:0 
DEVICE=bond0:0
BOOTPROTO=none
ONBOOT=yes
IPADDR=10.128.50.12
NETMASK=255.255.255.0
GATEWAY=10.128.50.1
USERCTL=no

[root@dtgs1 network-scripts]# cat ifcfg-bond0\:1
DEVICE=bond0:1
BOOTPROTO=none
ONBOOT=yes
IPADDR=10.128.50.13
NETMASK=255.255.255.0
GATEWAY=10.128.50.1
USERCTL=no

[root@dtgs1 network-scripts]# cat ifcfg-bond1
DEVICE=bond1
BOOTPROTO=none
ONBOOT=yes
USERCTL=no

[root@dtgs1 network-scripts]# cat ifcfg-bond1.100 
DEVICE=bond1.100
BOOTPROTO=none
ONBOOT=yes
IPADDR=10.128.100.11
NETMASK=255.255.255.0
GATEWAY=10.128.100.1
USERCTL=no
VLAN=yes

[root@dtgs1 network-scripts]# cat ifcfg-bond1.106
DEVICE=bond1.106
BOOTPROTO=none
ONBOOT=yes
IPADDR=10.128.106.11
NETMASK=255.255.255.0
GATEWAY=10.128.106.1
USERCTL=no
VLAN=yes

[root@dtgs1 network-scripts]# cat route-bond1.100 
ADDRESS0=10.128.120.0
NETMASK0=255.255.255.0
GATEWAY0=10.128.100.1

ADDRESS1=10.128.180.0
NETMASK1=255.255.255.0
GATEWAY1=10.128.100.1

[root@dtgs1 network-scripts]# cat route-bond1.106
ADDRESS0=10.128.240.0
NETMASK0=255.255.240.0
GATEWAY0=10.128.106.1

A result of this configuration:

[root@dtgs1 network-scripts]# ip a
1: lo:  mtu 16436 qdisc noqueue 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0:  mtu 1500 qdisc pfifo_fast master bond0 qlen 1000
    link/ether 00:1e:68:2f:04:dc brd ff:ff:ff:ff:ff:ff
3: eth1:  mtu 1500 qdisc pfifo_fast master bond0 qlen 1000
    link/ether 00:1e:68:2f:04:dc brd ff:ff:ff:ff:ff:ff
4: eth2:  mtu 1500 qdisc pfifo_fast master bond1 qlen 1000
    link/ether 00:1e:68:2f:04:de brd ff:ff:ff:ff:ff:ff
5: eth3:  mtu 1500 qdisc pfifo_fast master bond1 qlen 1000
    link/ether 00:1e:68:2f:04:de brd ff:ff:ff:ff:ff:ff
6: sit0:  mtu 1480 qdisc noop 
    link/sit 0.0.0.0 brd 0.0.0.0
7: bond0:  mtu 1500 qdisc noqueue 
    link/ether 00:1e:68:2f:04:dc brd ff:ff:ff:ff:ff:ff
    inet 10.128.50.11/24 brd 10.128.50.255 scope global bond0
    inet 10.128.50.12/24 brd 10.128.50.255 scope global secondary bond0:0
    inet 10.128.50.13/24 brd 10.128.50.255 scope global secondary bond0:1
    inet6 fe80::21e:68ff:fe2f:4dc/64 scope link 
       valid_lft forever preferred_lft forever
8: bond1:  mtu 1500 qdisc noqueue 
    link/ether 00:1e:68:2f:04:de brd ff:ff:ff:ff:ff:ff
    inet6 fe80::21e:68ff:fe2f:4de/64 scope link 
       valid_lft forever preferred_lft forever
9: bond1.100@bond1:  mtu 1500 qdisc noqueue 
    link/ether 00:1e:68:2f:04:de brd ff:ff:ff:ff:ff:ff
    inet 10.128.100.11/24 brd 10.128.100.255 scope global bond1.100
    inet6 fe80::21e:68ff:fe2f:4de/64 scope link 
       valid_lft forever preferred_lft forever
10: bond1.106@bond1:  mtu 1500 qdisc noqueue 
    link/ether 00:1e:68:2f:04:de brd ff:ff:ff:ff:ff:ff
    inet 10.128.106.11/24 brd 10.128.106.255 scope global bond1.106
    inet6 fe80::21e:68ff:fe2f:4de/64 scope link 
       valid_lft forever preferred_lft forever

[root@dtgs1 network-scripts]# ip r
10.128.100.0/24 dev bond1.100  proto kernel  scope link  src 10.128.100.11 
10.128.180.0/24 via 10.128.100.1 dev bond1.100 
10.128.50.0/24 dev bond0  proto kernel  scope link  src 10.128.50.11 
10.128.106.0/24 dev bond1.106  proto kernel  scope link  src 10.128.106.11 
10.128.120.0/24 via 10.128.100.1 dev bond1.100 
10.128.240.0/20 via 10.128.106.1 dev bond1.106 
169.254.0.0/16 dev bond1.106  scope link 
default via 10.128.50.1 dev bond0