update route handling

10-mycelium.conflist

@@ -2,6 +2,9 @@
   "cniVersion": "1.0.0",
   "name": "mycelium-network",
   "plugins": [
+    {
+      "type": "loopback"
+    },
     {
       "type": "mycelium-cni",
       "myceliumInterface": "mycelium"

README.md

@@ -10,15 +10,22 @@ This CNI plugin integrates with the Mycelium overlay network to provide IPv6 con
 
 - Mycelium daemon running on the host
 - Go 1.21+
+- Standard CNI plugins (for loopback and other basic functionality)
 - Root privileges for installation
 
 ## Installation
 
 ```bash
-# Download dependencies and build the plugin
+# 1. Install standard CNI plugins (required for loopback)
+CNI_VERSION="v1.3.0"
+wget https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz
+sudo mkdir -p /opt/cni/bin
+sudo tar -xzf cni-plugins-linux-amd64-${CNI_VERSION}.tgz -C /opt/cni/bin/
+
+# 2. Download dependencies and build the plugin
 make build
 
-# Install plugin and configuration
+# 3. Install plugin and configuration
 make install
 ```
 
@@ -26,13 +33,16 @@ make install
 
 ## Configuration
 
-The plugin uses a CNI configuration file (`10-mycelium.conflist`) that specifies the Mycelium interface name:
+The plugin uses a CNI configuration file (`10-mycelium.conflist`) that includes the loopback plugin and specifies the Mycelium interface name:
 
 ```json
 {
   "cniVersion": "1.0.0",
   "name": "mycelium-network",
   "plugins": [
+    {
+      "type": "loopback"
+    },
     {
       "type": "mycelium-cni",
       "myceliumInterface": "mycelium"
@@ -49,6 +59,32 @@ The plugin uses a CNI configuration file (`10-mycelium.conflist`) that specifies
 
 ## Usage with Kubernetes
 
+### For k3s
+
+k3s requires special setup since it uses Flannel CNI by default:
+
+```bash
+# 1. Install k3s without default CNI
+curl -sfL https://get.k3s.io | sh -s - --flannel-backend=none --disable-network-policy
+
+# OR modify existing k3s installation
+sudo systemctl edit k3s
+# Add these lines:
+# [Service]
+# ExecStart=
+# ExecStart=/usr/local/bin/k3s server --flannel-backend=none --disable-network-policy
+
+# 2. Install CNI plugins and Mycelium CNI plugin (follow installation steps above)
+
+# 3. Copy CNI config to k3s location
+sudo cp /etc/cni/net.d/10-mycelium.conflist /var/lib/rancher/k3s/agent/etc/cni/net.d/
+
+# 4. Restart k3s
+sudo systemctl restart k3s
+```
+
+### For standard Kubernetes
+
 ### 1. Setup Mycelium on all nodes
 
 First, install Mycelium on all Kubernetes nodes:
@@ -88,12 +124,19 @@ sudo systemctl enable --now mycelium
 On each Kubernetes node:
 
 ```bash
+# Install standard CNI plugins first (if not already done)
+CNI_VERSION="v1.3.0"
+wget https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz
+sudo mkdir -p /opt/cni/bin
+sudo tar -xzf cni-plugins-linux-amd64-${CNI_VERSION}.tgz -C /opt/cni/bin/
+
 # Download dependencies, build and install the plugin
 make build
 sudo make install
 
 # Verify installation
 ls -la /opt/cni/bin/mycelium-cni
+ls -la /opt/cni/bin/loopback
 ls -la /etc/cni/net.d/10-mycelium.conflist
 ```
 
@@ -202,8 +245,19 @@ Check common issues:
 sudo systemctl status mycelium
 ip -6 addr show mycelium
 
-# Check CNI logs
+# Check CNI logs (kubelet for standard k8s, k3s for k3s)
 journalctl -u kubelet | grep -i cni
+# OR for k3s:
+journalctl -u k3s | grep -i cni
+
+# Verify CNI plugins are installed
+ls -la /opt/cni/bin/loopback
+ls -la /opt/cni/bin/mycelium-cni
+
+# Check CNI configuration location
+ls -la /etc/cni/net.d/10-mycelium.conflist
+# OR for k3s:
+ls -la /var/lib/rancher/k3s/agent/etc/cni/net.d/10-mycelium.conflist
 
 # Verify network namespaces
 sudo ip netns list
@@ -213,6 +267,11 @@ kubectl exec -it <pod> -- ip link show
 kubectl exec -it <pod> -- ip -6 route show
 ```
 
+**Common errors and solutions:**
+- `failed to find plugin "loopback"`: Install standard CNI plugins (see installation section)
+- `failed to find interface mycelium`: Mycelium daemon not running or interface not created
+- `no global IPv6 address found`: Mycelium not connected to network peers
+
 ## Architecture
 
 Based on the docker-demo.sh script, this plugin:

main.go

@@ -62,7 +62,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 	// Configure container interface
 	containerIP := generateContainerIP(myceliumIP, args.ContainerID)
-	if err := configureContainerInterface(containerNS, containerVethName, containerIP, hostVethName); err != nil {
+	if err := configureContainerInterface(containerNS, containerVethName, containerIP, hostVeth); err != nil {
 		return fmt.Errorf("failed to configure container interface: %v", err)
 	}
 
@@ -182,7 +182,7 @@ func createVethPair(hostName, containerName string) (netlink.Link, netlink.Link,
 	return hostVeth, containerVeth, nil
 }
 
-func configureContainerInterface(containerNS netns.NsHandle, ifName string, containerIP net.IP, hostVethName string) error {
+func configureContainerInterface(containerNS netns.NsHandle, ifName string, containerIP net.IP, hostVeth netlink.Link) error {
 	// Switch to container namespace
 	originalNS, err := netns.Get()
 	if err != nil {
@@ -216,27 +216,23 @@ func configureContainerInterface(containerNS netns.NsHandle, ifName string, cont
 			return err
 		}
 
-		// Get host veth link-local address for routing
+		// Get host veth link-local address (it should be available now)
-		hostVeth, err := netlink.LinkByName(hostVethName)
+		hostLinkLocal, err := getHostVethLinkLocal(hostVeth)
-		if err == nil {
+		if err != nil {
-			hostAddrs, err := netlink.AddrList(hostVeth, netlink.FAMILY_V6)
+			return fmt.Errorf("failed to get host veth link-local address: %v", err)
-			if err == nil {
+		}
-				for _, addr := range hostAddrs {
+
-					if addr.IP.IsLinkLocalUnicast() {
+		// Add route to Mycelium network via host veth link-local address
-						// Add route to Mycelium network via host veth
+		route := &netlink.Route{
-						route := &netlink.Route{
+			Dst: &net.IPNet{
-							Dst: &net.IPNet{
+				IP:   net.ParseIP("400::"),
-								IP:   net.ParseIP("400::"),
+				Mask: net.CIDRMask(7, 128),
-								Mask: net.CIDRMask(7, 128),
+			},
-							},
+			Gw:        hostLinkLocal,
-							Gw:        addr.IP,
+			LinkIndex: link.Attrs().Index,
-							LinkIndex: link.Attrs().Index,
+		}
-						}
+		if err := netlink.RouteAdd(route); err != nil {
-						netlink.RouteAdd(route)
+			return fmt.Errorf("failed to add route to 400::/7: %v", err)
-						break
-					}
-				}
-			}
 		}
 
 		return nil
@@ -254,3 +250,18 @@ func configureHostInterface(hostVeth netlink.Link, containerIP net.IP) error {
 
 	return netlink.RouteAdd(route)
 }
+
+func getHostVethLinkLocal(hostVeth netlink.Link) (net.IP, error) {
+	addrs, err := netlink.AddrList(hostVeth, netlink.FAMILY_V6)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get addresses for host veth: %v", err)
+	}
+
+	for _, addr := range addrs {
+		if addr.IP.IsLinkLocalUnicast() {
+			return addr.IP, nil
+		}
+	}
+
+	return nil, fmt.Errorf("no link-local address found on host veth")
+}