Edit Mesh: Do not use barycentric coord in EDGE_FIX shader and ...

... display vertex even when occluded.

Add back the vertex display because occluded vertex are not visible when
the triangle is almost parallel to the view.

The problem with the barycentric coord is that they are hard to work with
and their derivatives not enough precise to compute the vertex positions.

So we need to pass the vertices scree positions down to the fragment shader.

source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl

@@ -12,12 +12,17 @@ flat in ivec3 flag;
 #ifdef VERTEX_SELECTION
 in vec3 vertexColor;
 #endif
+
+#ifdef EDGE_FIX
+flat in vec2 ssPos[3];
+#else
+in vec3 barycentric;
+#endif
+
 #ifdef VERTEX_FACING
 in float facing;
 #endif
 
-in vec3 barycentric;
-
 out vec4 FragColor;
 
 /* Vertex flag is shifted and combined with the edge flag */
@@ -27,13 +32,41 @@ out vec4 FragColor;
 
 /* Style Parameters in pixel */
 
-void distToEdgeAndPoint(vec2 dir, vec2 ori, out float edge, out float point)
+void distToEdgesAndPoints(out vec3 edges, out vec3 points)
 {
-	dir = normalize(dir.xy);
-	dir = vec2(-dir.y, dir.x);
-	vec2 of = gl_FragCoord.xy - ori;
-	point = sqrt(dot(of, of));
-	edge = abs(dot(dir, of));
+#ifdef EDGE_FIX
+	vec2 e0 = normalize(ssPos[1] - ssPos[0] + 1e-8);
+	vec2 e1 = normalize(ssPos[2] - ssPos[1] + 1e-8);
+	vec2 e2 = normalize(ssPos[0] - ssPos[2] + 1e-8);
+	e0 = vec2(-e0.y, e0.x);
+	e1 = vec2(-e1.y, e1.x);
+	e2 = vec2(-e2.y, e2.x);
+	vec2 p0 = gl_FragCoord.xy - ssPos[0];
+	vec2 p1 = gl_FragCoord.xy - ssPos[1];
+	vec2 p2 = gl_FragCoord.xy - ssPos[2];
+	edges.z = abs(dot(e0, p0));
+	edges.x = abs(dot(e1, p1));
+	edges.y = abs(dot(e2, p2));
+#else
+	vec3 dx = dFdx(barycentric);
+	vec3 dy = dFdy(barycentric);
+	/* per component derivative */
+	vec2 d0 = vec2(dx.x, dy.x);
+	vec2 d1 = vec2(dx.y, dy.y);
+	vec2 d2 = vec2(dx.z, dy.z);
+	vec3 d = vec3(length(d0), length(d1), length(d2));
+
+	edges = abs(vec3(barycentric / d));
+#endif
+
+#if defined(VERTEX_SELECTION) && defined(EDGE_FIX)
+	points.x = dot(p0, p0);
+	points.y = dot(p1, p1);
+	points.z = dot(p2, p2);
+	points = sqrt(points);
+#else
+	points = vec3(1e10);
+#endif
 }
 
 void colorDist(vec4 color, float dist)
@@ -54,17 +87,8 @@ void colorDistEdge(vec4 color, float dist)
 
 void main()
 {
-	/* Step 1 : Computing Distances */
-	vec3 dx = dFdx(barycentric);
-	vec3 dy = dFdy(barycentric);
-	vec3 d = vec3(
-		length(vec2(dx.x, dy.x)),
-		length(vec2(dx.y, dy.y)),
-		length(vec2(dx.z, dy.z))
-	);
-	vec3 e = abs(vec3(barycentric / d));
-
-	/* Step 2 : coloring (order dependent) */
+	vec3 e, p;
+	distToEdgesAndPoints(e, p);
 
 	/* Face */
 	FragColor = faceColor;
@@ -99,9 +123,8 @@ void main()
 		}
 	}
 
-#if 0
+#if defined(VERTEX_SELECTION) && defined(EDGE_FIX)
 	/* Points */
-#ifdef VERTEX_SELECTION
 	for (int v = 0; v < 3; ++v) {
 		if ((flag[v] & EDGE_VERTEX_EXISTS) == 0) {
 			/* Leave as-is, no vertex. */
@@ -117,7 +140,6 @@ void main()
 		}
 	}
 #endif
-#endif
 
 #ifdef VERTEX_FACING
 	FragColor.a *= 1.0 - abs(facing) * 0.4;

source/blender/draw/modes/shaders/edit_mesh_overlay_geom_edge.glsl

@@ -21,7 +21,6 @@ flat out vec3 edgesCrease;
 flat out vec3 edgesBweight;
 flat out vec4 faceColor;
 flat out ivec3 flag;
-out vec3 barycentric;
 #ifdef VERTEX_SELECTION
 out vec3 vertexColor;
 #endif
@@ -89,20 +88,14 @@ void main()
 	ssPos[1] = pos[1];
 	flag[0] = flag[2] = (vData[0].x << 8);
 	flag[1] = (vData[1].x << 8);
-	barycentric = vec3(1.0);
 	doVertex(0, pPos[0] + vec4( dirs1.zw, 0.0, 0.0));
-
-	barycentric[2] = -1.0;
 	doVertex(0, pPos[0] + vec4(-dirs1.zw, 0.0, 0.0));
 
 	flag[2] |= vData[0].y;
 	edgesCrease[2] = vData[0].z / 255.0;
 	edgesBweight[2] = vData[0].w / 255.0;
 
-	barycentric = vec3(1.0);
 	doVertex(1, pPos[1] + vec4( dirs2.zw, 0.0, 0.0));
-
-	barycentric[2] = -1.0;
 	doVertex(1, pPos[1] + vec4(-dirs2.zw, 0.0, 0.0));
 
 	EndPrimitive();

source/blender/draw/modes/shaders/edit_mesh_overlay_geom_tri.glsl

@@ -28,7 +28,7 @@ flat out vec3 edgesBweight;
 flat out vec4 faceColor;
 flat out ivec3 flag;
 
-out vec3 barycentric;
+flat out vec2 ssPos[3];
 #ifdef VERTEX_SELECTION
 out vec3 vertexColor;
 #endif
@@ -52,8 +52,6 @@ void doVertex(int v)
 	facing = vFacing[v];
 #endif
 	gl_Position = pPos[v];
-	barycentric = vec3(0.0);
-	barycentric[v % 3] = 1.0;
 
 	EmitVertex();
 }
@@ -63,7 +61,6 @@ void doLoopStrip(int v, vec3 offset)
 	doVertex(v);
 
 	gl_Position.xyz += offset;
-	barycentric = vec3(1.0);
 
 	EmitVertex();
 }
@@ -95,7 +92,6 @@ void main()
 		faceColor = colorFace;
 
 	/* Vertex */
-	vec2 ssPos[3];
 	ssPos[0] = proj(pPos[0]);
 	ssPos[1] = proj(pPos[1]);
 	ssPos[2] = proj(pPos[2]);
@@ -106,6 +102,10 @@ void main()
 
 	EndPrimitive();
 
+	for (int v = 0; v < 3; ++v) {
+		flag[v] &= ~EDGE_VERTEX_EXISTS;
+	}
+
 	vec2 fixvec[6];
 	vec2 fixvecaf[6];