Fix T39781: Autosmooth feature: erroneously showing hidden vertices' normals too when in "vertex per face" display

So, turned out after all we need a foreachmapped helper for loops as well... :/

CDDM and EBDM were reasonably simple, but I fought hours with CCGDM (subsurf) to find only a
bad working solution (see D478). So I fallback to a code similar to CDDM one.
Probably not nice for performances, but loops and subsurf are not matching well...

Reviewers: campbellbarton

CC: brecht

Differential Revision: https://developer.blender.org/D478

source/blender/blenkernel/BKE_DerivedMesh.h

@@ -156,7 +156,7 @@ typedef enum DMDrawFlag {
 
 typedef enum DMForeachFlag {
 	DM_FOREACH_NOP = 0,
-	DM_FOREACH_USE_NORMAL = (1 << 0),  /* foreachMappedVert, foreachMappedFaceCenter */
+	DM_FOREACH_USE_NORMAL = (1 << 0),  /* foreachMappedVert, foreachMappedLoop, foreachMappedFaceCenter */
 } DMForeachFlag;
 
 typedef enum DMDirtyFlag {
@@ -309,6 +309,15 @@ struct DerivedMesh {
 	                                       const float v0co[3], const float v1co[3]),
 	                          void *userData);
 
+	/** Iterate over each mapped loop in the derived mesh, calling the given function
+	 * with the original loop index and the mapped loops's new coordinate and normal.
+	 */
+	void (*foreachMappedLoop)(DerivedMesh *dm,
+	                          void (*func)(void *userData, int vertex_index, int face_index,
+	                                       const float co[3], const float no[3]),
+	                          void *userData,
+	                          DMForeachFlag flag);
+
 	/** Iterate over each mapped face in the derived mesh, calling the
 	 * given function with the original face and the mapped face's (or
 	 * faces') center and normal.

source/blender/blenkernel/intern/cdderivedmesh.c

@@ -1691,6 +1691,35 @@ static void cdDM_foreachMappedEdge(
 	}
 }
 
+static void cdDM_foreachMappedLoop(
+        DerivedMesh *dm,
+        void (*func)(void *userData, int vertex_index, int face_index, const float co[3], const float no[3]),
+        void *userData,
+        DMForeachFlag flag)
+{
+	/* We can't use dm->getLoopDataLayout(dm) here, we want to always access dm->loopData, EditDerivedBMesh would
+	 * return loop data from bmesh itself. */
+	const float (*lnors)[3] = (flag & DM_FOREACH_USE_NORMAL) ? DM_get_loop_data_layer(dm, CD_NORMAL) : NULL;
+
+	const MVert *mv = CDDM_get_verts(dm);
+	const MLoop *ml = CDDM_get_loops(dm);
+	const MPoly *mp = CDDM_get_polys(dm);
+	const int *v_index = DM_get_vert_data_layer(dm, CD_ORIGINDEX);
+	const int *f_index = DM_get_poly_data_layer(dm, CD_ORIGINDEX);
+	int p_idx, i;
+
+	for (p_idx = 0; p_idx < dm->numPolyData; ++p_idx, ++mp) {
+		for (i = 0; i < mp->totloop; ++i, ++ml) {
+			const int v_idx = v_index ? v_index[ml->v] : ml->v;
+			const int f_idx = f_index ? f_index[p_idx] : p_idx;
+			const float *no = lnors ? *lnors++ : NULL;
+			if (!ELEM(ORIGINDEX_NONE, v_idx, f_idx)) {
+				func(userData, v_idx, f_idx, mv[ml->v].co, no);
+			}
+		}
+	}
+}
+
 static void cdDM_foreachMappedFaceCenter(
         DerivedMesh *dm,
         void (*func)(void *userData, int index, const float cent[3], const float no[3]),
@@ -1831,6 +1860,7 @@ static CDDerivedMesh *cdDM_create(const char *desc)
 
 	dm->foreachMappedVert = cdDM_foreachMappedVert;
 	dm->foreachMappedEdge = cdDM_foreachMappedEdge;
+	dm->foreachMappedLoop = cdDM_foreachMappedLoop;
 	dm->foreachMappedFaceCenter = cdDM_foreachMappedFaceCenter;
 
 	dm->release = cdDM_release;

source/blender/blenkernel/intern/editderivedmesh.c

@@ -378,6 +378,39 @@ static void emDM_drawUVEdges(DerivedMesh *dm)
 	glEnd();
 }
 
+static void emDM_foreachMappedLoop(
+        DerivedMesh *dm,
+        void (*func)(void *userData, int vertex_index, int face_index, const float co[3], const float no[3]),
+        void *userData,
+        DMForeachFlag flag)
+{
+	/* We can't use dm->getLoopDataLayout(dm) here, we want to always access dm->loopData, EditDerivedBMesh would
+	 * return loop data from bmesh itself. */
+	const float (*lnors)[3] = (flag & DM_FOREACH_USE_NORMAL) ? DM_get_loop_data_layer(dm, CD_NORMAL) : NULL;
+
+	EditDerivedBMesh *bmdm = (EditDerivedBMesh *)dm;
+	BMesh *bm = bmdm->em->bm;
+	BMFace *efa;
+	BMIter iter;
+
+	const float (*vertexCos)[3] = bmdm->vertexCos;
+	int f_idx;
+
+	BM_mesh_elem_index_ensure(bm, BM_VERT);
+
+	BM_ITER_MESH_INDEX (efa, &iter, bm, BM_FACES_OF_MESH, f_idx) {
+		BMLoop *l_iter, *l_first;
+
+		l_iter = l_first = BM_FACE_FIRST_LOOP(efa);
+		do {
+			const BMVert *eve = l_iter->v;
+			const int v_idx = BM_elem_index_get(eve);
+			const float *no = lnors ? *lnors++ : NULL;
+			func(userData, v_idx, f_idx, vertexCos ? vertexCos[v_idx] : eve->co, no);
+		} while ((l_iter = l_iter->next) != l_first);
+	}
+}
+
 static void emDM_foreachMappedFaceCenter(
         DerivedMesh *dm,
         void (*func)(void *userData, int index, const float co[3], const float no[3]),
@@ -1728,6 +1761,7 @@ DerivedMesh *getEditDerivedBMesh(BMEditMesh *em,
 	bmdm->dm.recalcTessellation = emDM_recalcTessellation;
 
 	bmdm->dm.foreachMappedVert = emDM_foreachMappedVert;
+	bmdm->dm.foreachMappedLoop = emDM_foreachMappedLoop;
 	bmdm->dm.foreachMappedEdge = emDM_foreachMappedEdge;
 	bmdm->dm.foreachMappedFaceCenter = emDM_foreachMappedFaceCenter;
 

source/blender/blenkernel/intern/subsurf_ccg.c

@@ -1554,6 +1554,35 @@ static void ccgDM_foreachMappedEdge(
 	ccgEdgeIterator_free(ei);
 }
 
+static void ccgDM_foreachMappedLoop(
+        DerivedMesh *dm,
+        void (*func)(void *userData, int vertex_index, int face_index, const float co[3], const float no[3]),
+        void *userData,
+        DMForeachFlag flag)
+{
+	/* We can't use dm->getLoopDataLayout(dm) here, we want to always access dm->loopData, EditDerivedBMesh would
+	 * return loop data from bmesh itself. */
+	const float (*lnors)[3] = (flag & DM_FOREACH_USE_NORMAL) ? DM_get_loop_data_layer(dm, CD_NORMAL) : NULL;
+
+	MVert *mv = dm->getVertArray(dm);
+	MLoop *ml = dm->getLoopArray(dm);
+	MPoly *mp = dm->getPolyArray(dm);
+	const int *v_index = dm->getVertDataArray(dm, CD_ORIGINDEX);
+	const int *f_index = dm->getPolyDataArray(dm, CD_ORIGINDEX);
+	int p_idx, i;
+
+	for (p_idx = 0; p_idx < dm->numPolyData; ++p_idx, ++mp) {
+		for (i = 0; i < mp->totloop; ++i, ++ml) {
+			const int v_idx = v_index ? v_index[ml->v] : ml->v;
+			const int f_idx = f_index ? f_index[p_idx] : p_idx;
+			const float *no = lnors ? *lnors++ : NULL;
+			if (!ELEM(ORIGINDEX_NONE, v_idx, f_idx)) {
+				func(userData, v_idx, f_idx, mv[ml->v].co, no);
+			}
+		}
+	}
+}
+
 static void ccgDM_drawVerts(DerivedMesh *dm)
 {
 	CCGDerivedMesh *ccgdm = (CCGDerivedMesh *) dm;
@@ -3461,6 +3490,7 @@ static CCGDerivedMesh *getCCGDerivedMesh(CCGSubSurf *ss,
 	ccgdm->dm.getVertCos = ccgdm_getVertCos;
 	ccgdm->dm.foreachMappedVert = ccgDM_foreachMappedVert;
 	ccgdm->dm.foreachMappedEdge = ccgDM_foreachMappedEdge;
+	ccgdm->dm.foreachMappedLoop = ccgDM_foreachMappedLoop;
 	ccgdm->dm.foreachMappedFaceCenter = ccgDM_foreachMappedFaceCenter;
 	
 	ccgdm->dm.drawVerts = ccgDM_drawVerts;

source/blender/editors/space_view3d/drawobject.c

@@ -2433,47 +2433,46 @@ static int draw_dm_test_freestyle_face_mark(BMesh *bm, BMFace *efa)
 #endif
 
 /* Draw loop normals. */
-static void draw_dm_loop_normals(BMEditMesh *em, Scene *scene, Object *ob, DerivedMesh *dm)
+static void draw_dm_loop_normals__mapFunc(void *userData, int vertex_index, int face_index,
+                                          const float co[3], const float no[3])
 {
-	/* XXX Would it be worth adding a dm->foreachMappedLoop func just for this? I doubt it... */
+	if (no) {
+		const drawDMNormal_userData *data = userData;
+		const BMVert *eve = BM_vert_at_index(data->bm, vertex_index);
+		const BMFace *efa = BM_face_at_index(data->bm, face_index);
+		float vec[3];
 
-	/* We can't use dm->getLoopDataLayout(dm) here, we want to always access dm->loopData, EditDerivedBMesh would
-	 * return loop data from bmesh itself. */
-	float (*lnors)[3] = DM_get_loop_data_layer(dm, CD_NORMAL);
-
-	if (lnors) {
-		drawDMNormal_userData data;
-		const MLoop *mloops = dm->getLoopArray(dm);
-		const MVert *mverts = dm->getVertArray(dm);
-		int i, totloops = dm->getNumLoops(dm);
-
-		data.bm = em->bm;
-		data.normalsize = scene->toolsettings->normalsize;
-
-		calcDrawDMNormalScale(ob, &data);
-
-		glBegin(GL_LINES);
-		for (i = 0; i < totloops; i++, mloops++, lnors++) {
-			float no[3];
-			const float *co = mverts[mloops->v].co;
-
-			if (!data.uniform_scale) {
-				mul_v3_m3v3(no, data.tmat, (float *)lnors);
-				normalize_v3(no);
-				mul_m3_v3(data.imat, no);
+		if (!(BM_elem_flag_test(eve, BM_ELEM_HIDDEN) || BM_elem_flag_test(efa, BM_ELEM_HIDDEN))) {
+			if (!data->uniform_scale) {
+				mul_v3_m3v3(vec, data->tmat, no);
+				normalize_v3(vec);
+				mul_m3_v3(data->imat, vec);
 			}
 			else {
-				copy_v3_v3(no, (float *)lnors);
+				copy_v3_v3(vec, no);
 			}
-			mul_v3_fl(no, data.normalsize);
-			add_v3_v3(no, co);
+			mul_v3_fl(vec, data->normalsize);
+			add_v3_v3(vec, co);
 			glVertex3fv(co);
-			glVertex3fv(no);
+			glVertex3fv(vec);
 		}
-		glEnd();
 	}
 }
 
+static void draw_dm_loop_normals(BMEditMesh *em, Scene *scene, Object *ob, DerivedMesh *dm)
+{
+	drawDMNormal_userData data;
+
+	data.bm = em->bm;
+	data.normalsize = scene->toolsettings->normalsize;
+
+	calcDrawDMNormalScale(ob, &data);
+
+	glBegin(GL_LINES);
+	dm->foreachMappedLoop(dm, draw_dm_loop_normals__mapFunc, &data, DM_FOREACH_USE_NORMAL);
+	glEnd();
+}
+
 /* Draw faces with color set based on selection
  * return 2 for the active face so it renders with stipple enabled */
 static DMDrawOption draw_dm_faces_sel__setDrawOptions(void *userData, int index)