├── 11-cake-dual-ifb ├── cake-dual-ifb ├── README.md └── cake-dual-ifb.nft /11-cake-dual-ifb: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | [ -n "$DEVICE" ] || exit 0 4 | 5 | [ "$ACTION" = ifup ] && /etc/init.d/cake-dual-ifb stop && /etc/init.d/cake-dual-ifb start 6 | 7 | [ "$ACTION" = ifdown ] && /etc/init.d/cake-dual-ifb stop 8 | -------------------------------------------------------------------------------- /cake-dual-ifb: -------------------------------------------------------------------------------- 1 | #!/bin/sh /etc/rc.common 2 | 3 | exec &> /var/log/cake-dual-ifb.log 4 | 5 | START=91 6 | STOP=4 7 | 8 | # local interfaces to combine flows from 9 | ifaces="br-lan br-guest" 10 | 11 | start() 12 | { 13 | # Setup dual IFBs 14 | ip link add name ifb-ul type ifb 15 | ip link add name ifb-dl type ifb 16 | ip link set ifb-ul up 17 | ip link set ifb-dl up 18 | 19 | for iface in $ifaces; do 20 | 21 | tc qdisc add dev $iface handle ffff: ingress 22 | tc qdisc add dev $iface handle 1: root prio 23 | 24 | # capture $if (ingress) -> wan 25 | # filter on fwmark 1 26 | tc filter add dev $iface parent ffff: protocol all handle 1 fw flowid 1:1 action mirred egress redirect dev ifb-ul 27 | 28 | # capture wan -> $iface (egress) and restore DSCPs from conntracks 29 | # filter on fwmark 1 30 | tc filter add dev $iface parent 1: protocol all handle 1 fw flowid 1:1 action ctinfo dscp 63 128 action mirred egress redirect dev ifb-dl 31 | 32 | done 33 | 34 | tc qdisc add dev wan handle ffff: ingress 35 | tc qdisc add dev wan handle 1: root prio 36 | 37 | # capture OpenWrt -> wan (egress) 38 | # filter by fwmark 2 (use nftables to skip over WireGuard traffic with fwmark 3 and apply fwmark 2 to the remainder) 39 | tc filter add dev wan parent 1: protocol all handle 2 fw flowid 1:1 action mirred egress redirect dev ifb-ul 40 | 41 | # capture wan (ingress) -> OpenWrt and restore DSCPs from conntracks 42 | # filter on conntrack bit 128 set 43 | tc filter add dev wan parent ffff: prio 1 protocol all matchall action ctinfo cpmark continue 44 | tc filter add dev wan parent ffff: prio 2 protocol all handle 256/256 fw flowid 1:1 action ctinfo dscp 63 128 action mirred egress redirect dev ifb-dl 45 | 46 | # apply CAKE on the IFBs 47 | tc qdisc add dev ifb-ul root cake bandwidth 30Mbit diffserv4 dual-srchost nonat wash no-ack-filter noatm overhead 92 48 | tc qdisc add dev ifb-dl root cake bandwidth 25Mbit diffserv4 dual-dsthost nonat nowash ingress no-ack-filter noatm overhead 92 49 | } 50 | 51 | stop() 52 | { 53 | for iface in $ifaces; do 54 | 55 | tc qdisc del dev $iface ingress 56 | tc qdisc del dev $iface root 57 | 58 | done 59 | 60 | tc qdisc del dev wan ingress 61 | tc qdisc del dev wan root 62 | 63 | tc qdisc del dev ifb-ul root 64 | tc qdisc del dev ifb-dl root 65 | ip link set ifb-ul down 66 | ip link del ifb-ul 67 | ip link set ifb-dl down 68 | ip link del ifb-dl 69 | } 70 | 71 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cake-dual-ifb 2 | 3 | cake-dual-ifb sets up dual IFBs and applies CAKE on them using nftables and appropriate tc calls. DSCPs for use with diffserv4 can be set by the router and/or by LAN clients and conntrack restore is used to restore DSCPs on download based on upload DSCPs. 4 | 5 | ### CAKE interfaces 6 | 7 | Set up CAKE using dual upload and download IFBs: 8 | 9 | ![image](https://user-images.githubusercontent.com/10721999/188687637-630aa3c5-ee42-4139-be4d-283e1515b54d.png) 10 | 11 | That is, create 'ifb-ul' by mirroring: 12 | 13 | - a) appropriately selected ingress from br-lan and br-guest; and 14 | 15 | - b) appropriately selected egress from wan. 16 | 17 | And create 'ifb-dl' by mirroring: 18 | 19 | - a) appropriately selected egress from br-lan and br-guest; and 20 | - b) appropriately selected ingress from wan. 21 | 22 | Appropriate selection is achieved through fwmarks using nftables to isolate and avoid duplication of the relevant flows including unencrytyped WireGuard traffic and OpenWrt->WAN and WAN->OpenWrt traffic, and skip out the LAN-LAN traffic. 23 | 24 | Then apply CAKE on 'ifb-ul' and 'ifb-dl'. 25 | 26 | This permits CAKE to properly function despite complex setups like use of VPN pbr and a guest LAN. 27 | 28 | ### DSCPs 29 | 30 | cake-dual-ifb is designed to handle DSCPs as follows: 31 | 32 | - firstly, determine the DSCPs associated with upload packets; and 33 | - secondly, apply those DSCPs to corresponding download packets associated with the same connection. 34 | 35 | This is achieved by: 36 | 37 | - firstly, using nftables to set the DSCPs to the 'conntrack marks' on upload; and 38 | - secondly, using tc-ctinfo to restore those stored DSCPs from the 'conntrack marks' on download. 39 | 40 | This facilitates setting DSCPs not only by the router itself but also by LAN clients. As compared to relying upon port ranges and the like, setting DSCPs in LAN clients offers a robust way to set DSCPs at the application level. 41 | 42 | ### nftables 43 | 44 | nftables is leveraged to apply fwmarks, DSCPs, and save DSCPs to conntracks as appropriate. 45 | 46 | To understand the logic adopted in cake-dual-ifb.nft consider this helpful diagram: 47 | 48 | ![image](https://user-images.githubusercontent.com/10721999/188932157-881bd4ef-e1ab-46d7-bd1b-966e78f00429.png) 49 | 50 | Source: https://wiki.nftables.org/wiki-nftables/index.php/Netfilter_hooks 51 | 52 | 53 | ## Required packages 54 | 55 | This script requires at least the following packages: 56 | 57 | - **tc-tiny** 58 | - **kmod-ifb** 59 | - **kmod-sched** 60 | - **kmod-sched-core** 61 | - **kmod-sched-cake** 62 | - **kmod-sched-ctinfo** 63 | - **kmod-sched-prio** 64 | - **kmod-nft-bridge** 65 | 66 | ## Installation on OpenWrt 67 | 68 | To install: 69 | 70 | - place cake-dual-ifb in /etc/init.d 71 | - chmod +x cake-dual-ifb 72 | - place 11-cake-dual-ifb in /etc/hotplug.d/iface/ 73 | - chmod +x 11-cake-dual-ifb 74 | - place cake-dual-ifb.nft in /usr/share/nftables.d/ruleset-post/ 75 | - edit cake-dual-ifb.nft for your use case 76 | - verify interfaces (e.g. replace or delete br-lan / br-guest lines as required) 77 | 78 | ### To setup DSCP setting by the router ### 79 | 80 | - amend cake-dual-ifb.nft as appropriate 81 | 82 | This can optionally override anything set by the LAN clients. 83 | 84 | ### Setting DSCPs in Microsoft Windows Based LAN Clients ### 85 | 86 | If using Microsoft Windows, DSCPs can be set at the application level by creating the registry key 'QoS' (it not present) as in: 87 | 88 | HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\Tcpip\QoS\ 89 | 90 | And then creating the string "Do not use NLA" inside the QoS key with value "1" 91 | 92 | ![image](https://user-images.githubusercontent.com/10721999/187535155-d4fd286b-9f20-40ce-8ff9-98ed36591721.png) 93 | 94 | And then by creating appropriate QoS policies in the Local Group Policy Editor: 95 | 96 | ![image](https://user-images.githubusercontent.com/10721999/187747512-4c608e11-92a9-4484-b07f-3695baa98b85.png) 97 | 98 | ### Verifying Correct Operation and DSCP Handling ### 99 | 100 | Verify correct operation and DSCP handling using tcpdump: 101 | 102 | ```bash 103 | opkg update; opkg install tcpdump 104 | # First check correct flows and DSCPs correctly set by your LAN client on upload 105 | tcpdump -i ifb-ul -vv udp 106 | # Second check correct flows and corresponding DSCPs are getting set by router on download 107 | tcpdump -i ifb-dl -vv udp 108 | ``` 109 | -------------------------------------------------------------------------------- /cake-dual-ifb.nft: -------------------------------------------------------------------------------- 1 | table inet cake-dual-ifb-inet 2 | flush table inet cake-dual-ifb-inet 3 | 4 | table bridge cake-dual-ifb-bridge 5 | flush table bridge cake-dual-ifb-bridge 6 | 7 | # local interfaces to collect traffic from 8 | # ingress traffic through forward hook is sent to ifb-ul 9 | # egress traffic through forward hook is sent to ifb-dl 10 | define IFACE_NAMES = { 11 | br-lan, 12 | br-guest 13 | } 14 | 15 | # local interface IP addresses 16 | # traffic with these destination addresses is skipped 17 | define IFACE_IPS = { 18 | 192.168.1.1, 19 | 192.168.2.1 20 | } 21 | 22 | # local MAC addresses to set to bulk (e.g. IoT devices) 23 | define BULK_MACS = { 24 | XX, 25 | YY 26 | } 27 | 28 | table inet cake-dual-ifb-inet { 29 | 30 | chain hook-output { 31 | 32 | type filter hook output priority filter 33 | 34 | # OpenWrt->wan 35 | oifname wan mark !=3 mark set 2 goto process-openwrt-to-wan 36 | 37 | } 38 | 39 | chain hook-forward { 40 | 41 | type filter hook forward priority filter 42 | 43 | # lan->wan 44 | iifname $IFACE_NAMES goto process-lan-to-wan 45 | 46 | # wan->lan 47 | oifname $IFACE_NAMES goto process-wan-to-lan 48 | 49 | } 50 | 51 | chain hook-postrouting { 52 | 53 | type filter hook postrouting priority filter 54 | 55 | # fix ttl to help disguise use of router over mobile network 56 | # for bridge mode set ttl to 64 57 | # for USB tethering set ttl to 65 58 | 59 | oifname wan ip ttl set 64 60 | } 61 | 62 | chain process-openwrt-to-wan { 63 | 64 | jump classify-dscp 65 | jump store-dscp-in-conntrack 66 | # mark special bit '256' in conntrack for tc filtering on wan ingress 67 | ct mark set ct mark or 256 68 | } 69 | 70 | chain process-lan-to-wan { 71 | 72 | # Handle DNS hijacking (effectively resulting in openwrt-to-wan) 73 | tcp dport 53 mark set 2 goto process-openwrt-to-wan 74 | udp dport 53 mark set 2 goto process-openwrt-to-wan 75 | 76 | } 77 | 78 | chain process-wan-to-lan { 79 | 80 | oifname $IFACE_NAMES mark set 1 81 | 82 | # Handle DNS hijacking 83 | tcp sport 53 mark set 0 84 | udp sport 53 mark set 0 85 | 86 | } 87 | 88 | # OpenWrt->wan dscp classication by router 89 | chain classify-dscp { 90 | 91 | meta l4proto . th dport vmap @rules_proto_dport 92 | 93 | } 94 | 95 | map rules_proto_dport { 96 | type inet_proto . inet_service : verdict 97 | elements = { 98 | tcp . 53 : goto dscp_set_voice, # DNS 99 | udp . 53 : goto dscp_set_voice, # DNS 100 | tcp . 853 : goto dscp_set_voice, # DNS-over-TLS 101 | udp . 853 : goto dscp_set_voice, # DNS-over-TLS 102 | udp . 123 : goto dscp_set_voice # NTP 103 | } 104 | } 105 | 106 | # designate packet for cake tin: bulk 107 | chain dscp_set_bulk { 108 | ip dscp set cs1 109 | } 110 | 111 | # designate packet for cake tin: besteffort 112 | chain dscp_set_besteffort { 113 | ip dscp set cs0 114 | } 115 | 116 | # designate packet for cake tin: video 117 | chain dscp_set_video { 118 | ip dscp set cs2 119 | } 120 | 121 | # designate packet for cake tin: voice 122 | chain dscp_set_voice { 123 | ip dscp set cs4 124 | } 125 | 126 | chain store-dscp-in-conntrack { 127 | 128 | # store DSCPs in conntracks 129 | ip version 4 ct mark set (@nh,8,8 & 252) >> 2 130 | ip6 version 6 ct mark set (@nh,0,16 & 4032) >> 6 131 | ct mark set ct mark or 128 132 | } 133 | } 134 | 135 | table bridge cake-dual-ifb-bridge { 136 | 137 | chain hook-input { 138 | 139 | type filter hook input priority filter 140 | 141 | # lan->wan 142 | meta pkttype unicast ip daddr != $IFACE_IPS goto process-lan-to-wan 143 | } 144 | 145 | chain process-lan-to-wan { 146 | 147 | ibrname $IFACE_NAMES mark set 1 148 | 149 | # Handle DNS hijacking 150 | tcp dport 53 mark set 0 151 | udp dport 53 mark set 0 152 | 153 | jump classify-dscp 154 | goto store-dscp-in-conntrack 155 | 156 | } 157 | 158 | # lan->wan dscp classication by router 159 | # set/override DHCP classifications on lan-originating traffic 160 | chain classify-dscp { 161 | 162 | meta l4proto . th dport vmap @rules_proto_dport 163 | 164 | # IoT devices (put mac addresses here) 165 | ibrname $IFACE_NAMES ether saddr $BULK_MACS goto dscp_set_bulk 166 | } 167 | 168 | map rules_proto_dport { 169 | type inet_proto . inet_service : verdict 170 | elements = { 171 | udp . 123 : goto dscp_set_voice # NTP 172 | } 173 | } 174 | 175 | # designate packet for cake tin: bulk 176 | chain dscp_set_bulk { 177 | ip dscp set cs1 178 | } 179 | 180 | # designate packet for cake tin: besteffort 181 | chain dscp_set_besteffort { 182 | ip dscp set cs0 183 | } 184 | 185 | # designate packet for cake tin: video 186 | chain dscp_set_video { 187 | ip dscp set cs2 188 | } 189 | 190 | # designate packet for cake tin: voice 191 | chain dscp_set_voice { 192 | ip dscp set cs4 193 | } 194 | 195 | chain store-dscp-in-conntrack { 196 | 197 | # store DSCPs in conntracks 198 | ip version 4 ct mark set (@nh,8,8 & 252) >> 2 199 | ip6 version 6 ct mark set (@nh,0,16 & 4032) >> 6 200 | ct mark set ct mark or 128 201 | } 202 | 203 | } 204 | --------------------------------------------------------------------------------