diff --git a/internal/server/fence.go b/internal/server/fence.go
index 67c041a0..c1d65460 100644
--- a/internal/server/fence.go
+++ b/internal/server/fence.go
@@ -170,7 +170,7 @@ func fenceMatch(
 	}
 	sw.mu.Lock()
 	var distance float64
-	if fence.distance {
+	if fence.distance && fence.obj != nil {
 		distance = details.obj.Distance(fence.obj)
 	}
 	sw.fmap = details.fmap