Minor fixes after introducing special BVH traversal for self-collision.

- Add parentheses to suppress an assertion on some compilers.

- It turns out cloth self-collision math is not precisely symmetric,
  so sort the triangle index pair to restore behavior exactly identical
  to the version before the new traversal.

Follow up to rB0796210c8df32

source/blender/blenkernel/intern/collision.c

@@ -1115,8 +1115,15 @@ static void cloth_selfcollision(void *__restrict userdata,
   float epsilon = clmd->coll_parms->selfepsilon;
   float pa[3], pb[3], vect[3];
 
-  tri_a = &clmd->clothObject->tri[data->overlap[index].indexA];
-  tri_b = &clmd->clothObject->tri[data->overlap[index].indexB];
+  /* Collision math is currently not symmetric, so ensure a stable order for each pair. */
+  int indexA = data->overlap[index].indexA, indexB = data->overlap[index].indexB;
+
+  if (indexA > indexB) {
+    SWAP(int, indexA, indexB);
+  }
+
+  tri_a = &clmd->clothObject->tri[indexA];
+  tri_b = &clmd->clothObject->tri[indexB];
 
   BLI_assert(cloth_bvh_selfcollision_is_active(clmd, clmd->clothObject, tri_a, tri_b));
 

source/blender/blenlib/intern/BLI_kdopbvh.c

@@ -1335,7 +1335,7 @@ BVHTreeOverlap *BLI_bvhtree_overlap_ex(
   /* 'RETURN_PAIRS' was not implemented without 'max_interactions'. */
   BLI_assert(overlap_pairs || max_interactions);
   /* Self-overlap does not support max interactions (it's not symmetrical). */
-  BLI_assert(!use_self || tree1 == tree2 && !max_interactions);
+  BLI_assert(!use_self || (tree1 == tree2 && !max_interactions));
 
   const int root_node_len = BLI_bvhtree_overlap_thread_num(tree1);
   const int thread_num = use_threading ? root_node_len : 1;