#!/bin/bash set -e if [ "$1" != "--inner" ]; then if [ ! -d "/run/netns" ]; then mkdir /run/netns chmod 0755 /run/netns fi export tmpdir=$(mktemp -p /run -d netns-cgnat-demo-XXXXXXX) trap 'rm -rf "${tmpdir}"' EXIT export NAMESPACEDIR="${tmpdir}/netns" mkdir "${NAMESPACEDIR}" chmod 0755 "${NAMESPACEDIR}" # Run actuall demo in network+mount+UTS namespaces unshare -m -n -u -- "$0" --inner echo "Cleaning up" # cleanup afterwards exit 0 fi show_failed_command() { local rc=$? if [ "${rc}" -ne 0 ]; then printf 'Failed command: %s\n' "${BASH_COMMAND}" fi exit $rc } trap show_failed_command EXIT cd "$(dirname "$(readlink -f "$0")")" cp tmux_base.conf "${tmpdir}/tmux.conf" printf >>"${tmpdir}/tmux.conf" 'new-session -n main -s cgnat-demo "%s"\n' "${SHELL} -i" printf >>"${tmpdir}/tmux.conf" 'new-window -d -n trace "nft monitor trace"\n' printf >>"${tmpdir}/tmux.conf" 'new-window -d -n conntrack "conntrack -E -o timestamp"\n' # setup local ip-netns "namespace" (so ip-netns names don't conflict with other stuff) mount -o bind "${NAMESPACEDIR}" /run/netns mount --make-private /run/netns # gonna do routing sysctl -q net.ipv4.ip_forward=1 sysctl -q net.ipv6.conf.default.forwarding=1 sysctl -q net.ipv6.conf.all.forwarding=1 # basic setup of our main network namespace ip link set dev lo up ./fix-vrf-rules.sh netns() { local name="$1" shift ip netns exec "${name}" "$@" } create_netns() { local name="$1" ip netns add "${name}" # basic setup ip -n "${name}" link set dev lo up netns "${name}" ./fix-vrf-rules.sh } # build explicit VRF to uplink (and route others through) ip link add name "up" type vrf table "1" ip link set dev "up" up printf >>"${tmpdir}/tmux.conf" 'new-window -d -n up -e debian_chroot=up "%s"\n' "ip vrf exec up ${SHELL} -i" export UPLINK="100.127.255.254" # last usable ip in 100.64.0.0/10 export UPLINK6="2001:db8:a::ffff" export PUBLIC="192.0.2.1" # build "uplink": uplink has one client: the "main" netns create_netns "uplink" ip link add name muplink type veth peer client1 ip link set dev client1 netns "uplink" ip -n "uplink" address add "${PUBLIC}/32" dev lo ip -n "uplink" link set dev client1 up ip -n "uplink" address add "${UPLINK}/10" dev client1 ip -n "uplink" address add "${UPLINK6}/64" dev client1 ip -n "uplink" route add "2001:db8:b::/48" via 2001:db8:a::1 dev client1 ip link set dev muplink vrf "up" up ip address add 100.64.0.1/10 dev muplink ip address add 2001:db8:a::1/64 dev muplink ip route add default vrf "up" via "${UPLINK}" dev muplink ip -6 route add default vrf "up" via "${UPLINK6}" dev muplink printf >>"${tmpdir}/tmux.conf" 'new-window -d -n uplink -e debian_chroot=uplink "%s"\n' "ip netns exec uplink ${SHELL} -i" declare -A VRFIDS create_client_vrf() { local vrfname="$1" local vrfid=$2 VRFIDS[${vrfname}]=${vrfid} ip link add name "${vrfname}" type vrf table "${vrfid}" ip link add name "br-${vrfname}" type bridge ip link set dev "br-${vrfname}" master "${vrfname}" up ip link set dev "${vrfname}" up ip address add "${UPLINK}/10" dev "br-${vrfname}" ip address add "2001:db8:b:${vrfid}::ffff/64" dev "br-${vrfname}" ip route add "2001:db8:b:${vrfid}::/64" vrf "up" dev "${vrfname}" # route-leak IPv6 clients ip route add default vrf "${vrfname}" dev "up" # route-leak uplink ip -6 route add default vrf "${vrfname}" dev "up" # route-leak uplink printf >>"${tmpdir}/tmux.conf" 'new-window -d -n %s -e debian_chroot=%s "%s"\n' "${vrfname}" "${vrfname}" "ip vrf exec ${vrfname} ${SHELL} -i" } create_client() { local vrfname="$1" local name="$2" local id="$3" local vrfid=${VRFIDS[$vrfname]} local ip="100.64.0.${id}/10" local ipv6="2001:db8:b:${vrfid}::${id}/64" create_netns "${name}" ip link add name "${name}" type veth peer cuplink ip link set dev cuplink netns "${name}" ip -n "${name}" address add "${ip}" dev cuplink ip -n "${name}" address add "${ipv6}" dev cuplink ip -n "${name}" link set dev cuplink up ip -n "${name}" route add default via "${UPLINK}" dev cuplink ip -n "${name}" route add default via "2001:db8:b:${vrfid}::ffff" dev cuplink sysctl -q "net.ipv6.conf.${name}.disable_ipv6=1" # disable ipv6 on bridge slave ip link set dev "${name}" master "br-${vrfname}" up printf >>"${tmpdir}/tmux.conf" 'new-window -d -n %s -e debian_chroot=%s "%s"\n' "${name}" "${name}" "ip netns exec ${name} ${SHELL} -i" } # setup firewall / NAT /usr/sbin/nft -f nft.conf create_client_vrf "blue" 10 create_client "blue" "blue_c1" 1 create_client "blue" "blue_c2" 2 create_client_vrf "red" 20 create_client "red" "red_c1" 1 # without NAT ipv4 seems to be working: ip -n "blue_c2" address add "192.0.2.2/32" dev cuplink ip route add "192.0.2.2/32" vrf "blue" dev br-blue # on bridge in vrf blue ip route add "192.0.2.2/32" vrf "up" dev blue # leak to vrf up ip -n "uplink" route add "192.0.2.2/32" via "100.64.0.1" dev client1 # static route in uplink echo echo "--- Have fun checking it out yourself (exit the shell to close the experiment)." export debian_chroot=cgnat-demo exec tmux -L "cgnat-demo-$$" -f "${tmpdir}/tmux.conf" attach