WIP: Simple working client/server

stage3/README.md

@@ -0,0 +1,16 @@
+## Stage3:
+
+* Point-to-point Tunnel w/ no Encryption
+* Server gets client's addr from first packet
+* Add packet counter to detect skipped and late packets
+
+### Learnings
+
+* Directional packet loss is an issue.
+  * Sending to hetzner: ~380 Mbits/sec
+  * From hetzner: ~800 Mbits/sec
+* Runs of dropped packets are generally small < 30
+  * Saw a few cases of 100-200
+* Runs of correctly-sequenced packets are generally >> drops
+* Late packets aren't so common
+* Dropping late packets causes large slow-down.

stage3/client.go

@@ -0,0 +1,35 @@
+package stage3
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+	"runtime/debug"
+)
+
+func RunClient(serverAddrStr string) {
+	defer func() {
+		if r := recover(); r != nil {
+			fmt.Printf("%v", r)
+			debug.PrintStack()
+		}
+	}()
+
+	iface, err := openInterface(network, clientIP, netName)
+	must(err)
+
+	myAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf(":%d", port))
+	must(err)
+
+	conn, err := net.ListenUDP("udp", myAddr)
+	must(err)
+
+	serverAddr, err := netip.ParseAddrPort(fmt.Sprintf("%s:%d", serverAddrStr, port))
+	must(err)
+
+	_, err = conn.WriteToUDPAddrPort([]byte{1, 2, 3, 4, 5, 6, 7, 8}, serverAddr)
+	must(err)
+
+	go readFromIFace(iface, conn, clientIP, serverIP, serverAddr)
+	readFromConn(iface, conn, serverIP)
+}

stage3/cmd/client/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+go build
+sudo setcap cap_net_admin+iep ./client
+./client 144.76.78.93

stage3/cmd/client/main.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"log"
+	"os"
+	"vppn/stage3"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		log.Fatalf("Usage: %s <addr:port>", os.Args[0])
+	}
+	stage3.RunClient(os.Args[1])
+}

stage3/cmd/server/build.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+go build
+ssh kevin "killall server"
+scp server kevin:/home/jdl/tmp/
+ssh root@kevin "sudo setcap cap_net_admin+iep /home/jdl/tmp/server"
+ssh kevin "/home/jdl/tmp/server"

stage3/cmd/server/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "vppn/stage3"
+
+func main() {
+	stage3.RunServer()
+}

stage3/interface.go

@@ -0,0 +1,142 @@
+package stage3
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+const (
+	if_mtu       = 1200
+	if_queue_len = 1000
+)
+
+func openInterface(network []byte, localIP byte, name string) (io.ReadWriteCloser, error) {
+	if len(network) != 4 {
+		return nil, fmt.Errorf("expected network to be 4 bytes, got %d", len(network))
+	}
+	ip := net.IPv4(network[0], network[1], network[2], localIP)
+
+	//////////////////////////
+	// Create TUN Interface //
+	//////////////////////////
+
+	tunFD, err := syscall.Open("/dev/net/tun", syscall.O_RDWR|unix.O_CLOEXEC, 0600)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open TUN device: %w", err)
+	}
+
+	// New interface request.
+	req, err := unix.NewIfreq(name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create new TUN interface request: %w", err)
+	}
+
+	// Flags:
+	//
+	// IFF_NO_PI => don't add packet info data to packets sent to the interface.
+	// IFF_TUN   => create a TUN device handling IP packets.
+	req.SetUint16(unix.IFF_NO_PI | unix.IFF_TUN)
+
+	err = unix.IoctlIfreq(tunFD, unix.TUNSETIFF, req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to set TUN device settings: %w", err)
+	}
+
+	// Name may not be exactly the same?
+	name = req.Name()
+
+	/////////////
+	// Set MTU //
+	/////////////
+
+	// We need a socket file descriptor to set other options for some reason.
+	sockFD, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open socket: %w", err)
+	}
+	defer unix.Close(sockFD)
+
+	req, err = unix.NewIfreq(name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create MTU interface request: %w", err)
+	}
+
+	req.SetUint32(if_mtu)
+	if err = unix.IoctlIfreq(sockFD, unix.SIOCSIFMTU, req); err != nil {
+		return nil, fmt.Errorf("failed to set interface MTU: %w", err)
+	}
+
+	//////////////////////
+	// Set Queue Length //
+	//////////////////////
+
+	req, err = unix.NewIfreq(name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create IP interface request: %w", err)
+	}
+
+	req.SetUint16(if_queue_len)
+	if err = unix.IoctlIfreq(sockFD, unix.SIOCSIFTXQLEN, req); err != nil {
+		return nil, fmt.Errorf("failed to set interface queue length: %w", err)
+	}
+
+	/////////////////////
+	// Set IP and Mask //
+	/////////////////////
+
+	req, err = unix.NewIfreq(name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create IP interface request: %w", err)
+	}
+
+	if err := req.SetInet4Addr(ip.To4()); err != nil {
+		return nil, fmt.Errorf("failed to set interface request IP: %w", err)
+	}
+
+	if err = unix.IoctlIfreq(sockFD, unix.SIOCSIFADDR, req); err != nil {
+		return nil, fmt.Errorf("failed to set interface IP: %w", err)
+	}
+
+	// SET MASK - must happen after setting address.
+	req, err = unix.NewIfreq(name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create mask interface request: %w", err)
+	}
+
+	if err := req.SetInet4Addr(net.IPv4(255, 255, 255, 0).To4()); err != nil {
+		return nil, fmt.Errorf("failed to set interface request mask: %w", err)
+	}
+
+	if err := unix.IoctlIfreq(sockFD, unix.SIOCSIFNETMASK, req); err != nil {
+		return nil, fmt.Errorf("failed to set interface mask: %w", err)
+	}
+
+	////////////////////////
+	// Bring Interface Up //
+	////////////////////////
+
+	req, err = unix.NewIfreq(name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create up interface request: %w", err)
+	}
+
+	// Get current flags.
+	if err = unix.IoctlIfreq(sockFD, unix.SIOCGIFFLAGS, req); err != nil {
+		return nil, fmt.Errorf("failed to get interface flags: %w", err)
+	}
+
+	flags := req.Uint16() | unix.IFF_UP | unix.IFF_RUNNING
+
+	// Set UP flag / broadcast flags.
+	req.SetUint16(flags)
+	if err = unix.IoctlIfreq(sockFD, unix.SIOCSIFFLAGS, req); err != nil {
+		return nil, fmt.Errorf("failed to set interface up: %w", err)
+	}
+
+	return os.NewFile(uintptr(tunFD), "tun"), nil
+}

stage3/packet.go

@@ -0,0 +1,23 @@
+package stage3
+
+import "unsafe"
+
+const headerSize = 9
+
+type packetHeader struct {
+	SrcIP   byte
+	Counter uint64
+}
+
+func (h packetHeader) Marshal(buf []byte) int {
+	buf = buf[:9]
+	buf[0] = h.SrcIP
+	*(*uint64)(unsafe.Pointer(&buf[1])) = h.Counter
+	return headerSize
+}
+
+func (h *packetHeader) Parse(buf []byte) int {
+	h.SrcIP = buf[0]
+	h.Counter = *(*uint64)(unsafe.Pointer(&buf[1]))
+	return headerSize
+}

stage3/packet_test.go

@@ -0,0 +1,22 @@
+package stage3
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestPacketHeader(t *testing.T) {
+	b := make([]byte, 1024)
+
+	h := packetHeader{
+		SrcIP:   8,
+		Counter: 2354,
+	}
+	n := h.Marshal(b)
+	h2 := packetHeader{}
+	h2.Parse(b[:n])
+
+	if !reflect.DeepEqual(h, h2) {
+		t.Fatal(h, h2)
+	}
+}

stage3/server.go

@@ -0,0 +1,147 @@
+package stage3
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/netip"
+	"runtime/debug"
+)
+
+var (
+	network    = []byte{10, 1, 1, 0}
+	serverIP   = byte(1)
+	clientIP   = byte(2)
+	port       = uint16(5151)
+	netName    = "testnet"
+	bufferSize = if_mtu * 2
+)
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func RunServer() {
+	defer func() {
+		if r := recover(); r != nil {
+			fmt.Printf("%v", r)
+			debug.PrintStack()
+		}
+	}()
+
+	iface, err := openInterface(network, serverIP, netName)
+	must(err)
+
+	myAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf(":%d", port))
+	must(err)
+
+	conn, err := net.ListenUDP("udp", myAddr)
+	must(err)
+
+	// Get remoteAddr from a packet.
+	buf := make([]byte, 8)
+	_, remoteAddr, err := conn.ReadFromUDPAddrPort(buf)
+	log.Printf("Got remote addr: %v", remoteAddr)
+	must(err)
+
+	go readFromIFace(iface, conn, serverIP, clientIP, remoteAddr)
+	readFromConn(iface, conn, clientIP)
+}
+
+func readFromIFace(iface io.ReadWriteCloser, conn *net.UDPConn, localIP, remoteIP byte, remoteAddr netip.AddrPort) {
+	var (
+		n       int
+		packet  = make([]byte, bufferSize)
+		version byte
+		ip      byte
+		err     error
+		counter uint64
+		buf     = make([]byte, bufferSize)
+	)
+
+	for {
+		n, err = iface.Read(packet[:bufferSize])
+		must(err)
+		packet = packet[:n]
+
+		if len(packet) < 20 {
+			log.Printf("Dropping small packet: %d", n)
+			continue
+		}
+
+		packet = packet[:n]
+		version = packet[0] >> 4
+
+		switch version {
+		case 4:
+			ip = packet[19]
+		case 6:
+			ip = packet[39]
+		default:
+			log.Printf("Dropping packet with IP version: %d", version)
+			continue
+		}
+
+		if ip != remoteIP {
+			log.Printf("Dropping packet for incorrect IP: %d", ip)
+			continue
+		}
+
+		h := packetHeader{SrcIP: localIP, Counter: counter}
+		counter++
+		buf = buf[:headerSize+len(packet)]
+		h.Marshal(buf)
+		copy(buf[headerSize:], packet)
+
+		_, err = conn.WriteToUDPAddrPort(buf, remoteAddr)
+		must(err)
+	}
+}
+
+func readFromConn(iface io.ReadWriteCloser, conn *net.UDPConn, remoteIP byte) {
+	var (
+		n       int
+		packet  = make([]byte, bufferSize)
+		err     error
+		counter uint64
+		run     uint64
+		h       packetHeader
+	)
+
+	for {
+		// We assume that we're only receiving packets from one source.
+		n, err = conn.Read(packet[:bufferSize])
+		must(err)
+
+		packet = packet[:n]
+		if len(packet) < headerSize {
+			fmt.Print("_")
+			continue
+		}
+
+		h.Parse(packet)
+		if h.SrcIP != remoteIP {
+			fmt.Print("?")
+			continue
+		}
+
+		if h.Counter == counter+1 {
+			run++
+			counter = h.Counter
+		} else if h.Counter > counter+1 {
+			fmt.Printf("x(%d/%d)", h.Counter-counter+1, run)
+			run = 0
+			counter = h.Counter
+		} else if h.Counter <= counter {
+			//log.Printf("Skipped late packet: -%d", counter-h.Counter)
+			//continue
+			fmt.Print("<")
+		}
+
+		_, err = iface.Write(packet[headerSize:])
+		must(err)
+	}
+}

stage3/startup.go

@@ -0,0 +1 @@
+package stage3