Displace Modifier: add an option to displace along (averaged) custom normals, instead of vertex normals.

User suggestion/request from 'boby'.

source/blender/blenkernel/BKE_mesh.h

@@ -243,6 +243,10 @@ void BKE_mesh_normals_loop_custom_from_vertices_set(
         struct MPoly *mpolys, const float (*polynors)[3], const int numPolys,
         short (*r_clnors_data)[2]);
 
+void BKE_mesh_normals_loop_to_vertex(
+        const int numVerts, const struct MLoop *mloops, const int numLoops,
+        const float (*clnors)[3], float (*r_vert_clnors)[3]);
+
 void BKE_mesh_calc_poly_normal(
         const struct MPoly *mpoly, const struct MLoop *loopstart,
         const struct MVert *mvarray, float r_no[3]);

source/blender/blenkernel/intern/mesh_evaluate.c

@@ -1527,6 +1527,38 @@ void BKE_mesh_normals_loop_custom_from_vertices_set(
 	                             mpolys, polynors, numPolys, r_clnors_data, true);
 }
 
+/**
+ * Computes average per-vertex normals from given custom loop normals.
+ *
+ * @param clnors The computed custom loop normals.
+ * @param r_vert_clnors The (already allocated) array wher to store averaged per-vertex normals.
+ */
+void BKE_mesh_normals_loop_to_vertex(
+        const int numVerts, const MLoop *mloops, const int numLoops,
+        const float (*clnors)[3], float (*r_vert_clnors)[3])
+{
+	const MLoop *ml;
+	int i;
+
+	int *vert_loops_nbr = MEM_callocN(sizeof(*vert_loops_nbr) * (size_t)numVerts, __func__);
+
+	copy_vn_fl((float *)r_vert_clnors, 3 * numVerts, 0.0f);
+
+	for (i = 0, ml = mloops; i < numLoops; i++, ml++) {
+		const unsigned int v = ml->v;
+
+		add_v3_v3(r_vert_clnors[v], clnors[i]);
+		vert_loops_nbr[v]++;
+	}
+
+	for (i = 0; i < numVerts; i++) {
+		mul_v3_fl(r_vert_clnors[i], 1.0f / (float)vert_loops_nbr[i]);
+	}
+
+	MEM_freeN(vert_loops_nbr);
+}
+
+
 #undef LNOR_SPACE_TRIGO_THRESHOLD
 
 /** \} */

source/blender/makesdna/DNA_modifier_types.h

@@ -392,6 +392,7 @@ enum {
 	MOD_DISP_DIR_Z       = 2,
 	MOD_DISP_DIR_NOR     = 3,
 	MOD_DISP_DIR_RGB_XYZ = 4,
+	MOD_DISP_DIR_CLNOR   = 5,
 };
 
 /* DisplaceModifierData->texmapping */

source/blender/makesrna/intern/rna_modifier.c

@@ -1989,9 +1989,11 @@ static void rna_def_modifier_displace(BlenderRNA *brna)
 		{MOD_DISP_DIR_Y, "Y", 0, "Y", "Use the texture's intensity value to displace in the Y direction"},
 		{MOD_DISP_DIR_Z, "Z", 0, "Z", "Use the texture's intensity value to displace in the Z direction"},
 		{MOD_DISP_DIR_NOR, "NORMAL", 0, "Normal",
-		                   "Use the texture's intensity value to displace in the normal direction"},
+		 "Use the texture's intensity value to displace along the vertex normal"},
+		{MOD_DISP_DIR_CLNOR, "CUSTOM_NORMAL", 0, "Custom Normal",
+		 "Use the texture's intensity value to displace along the (averaged) custom normal (falls back to vertex)"},
 		{MOD_DISP_DIR_RGB_XYZ, "RGB_TO_XYZ", 0, "RGB to XYZ",
-		                       "Use the texture's RGB values to displace the mesh in the XYZ direction"},
+		 "Use the texture's RGB values to displace the mesh in the XYZ direction"},
 		{0, NULL, 0, NULL, NULL}
 	};
 

source/blender/modifiers/intern/MOD_displace.c

@@ -37,10 +37,11 @@
 #include "DNA_object_types.h"
 
 #include "BLI_utildefines.h"
-
+#include "BLI_math.h"
 
 #include "BKE_cdderivedmesh.h"
 #include "BKE_library.h"
+#include "BKE_mesh.h"
 #include "BKE_modifier.h"
 #include "BKE_texture.h"
 #include "BKE_deform.h"
@@ -98,6 +99,10 @@ static CustomDataMask requiredDataMask(Object *UNUSED(ob), ModifierData *md)
 	/* ask for UV coordinates if we need them */
 	if (dmd->texmapping == MOD_DISP_MAP_UV) dataMask |= CD_MASK_MTFACE;
 
+	if (dmd->direction == MOD_DISP_DIR_CLNOR) {
+		dataMask |= CD_MASK_CUSTOMLOOPNORMAL;
+	}
+
 	return dataMask;
 }
 
@@ -116,7 +121,7 @@ static bool dependsOnTime(ModifierData *md)
 static bool dependsOnNormals(ModifierData *md)
 {
 	DisplaceModifierData *dmd = (DisplaceModifierData *)md;
-	return (dmd->direction == MOD_DISP_DIR_NOR);
+	return ELEM(dmd->direction, MOD_DISP_DIR_NOR, MOD_DISP_DIR_CLNOR);
 }
 
 static void foreachObjectLink(ModifierData *md, Object *ob,
@@ -194,10 +199,12 @@ static void displaceModifier_do(
 	int i;
 	MVert *mvert;
 	MDeformVert *dvert;
+	int direction = dmd->direction;
 	int defgrp_index;
 	float (*tex_co)[3];
 	float weight = 1.0f; /* init value unused but some compilers may complain */
 	const float delta_fixed = 1.0f - dmd->midlevel;  /* when no texture is used, we fallback to white */
+	float (*vert_clnors)[3] = NULL;
 
 	if (!dmd->texture && dmd->direction == MOD_DISP_DIR_RGB_XYZ) return;
 	if (dmd->strength == 0.0f) return;
@@ -216,6 +223,25 @@ static void displaceModifier_do(
 		tex_co = NULL;
 	}
 
+	if (direction == MOD_DISP_DIR_CLNOR) {
+		CustomData *ldata = dm->getLoopDataLayout(dm);
+
+		if (CustomData_has_layer(ldata, CD_CUSTOMLOOPNORMAL)) {
+			float (*clnors)[3] = NULL;
+
+			if ((dm->dirty & DM_DIRTY_NORMALS) || !CustomData_has_layer(ldata, CD_NORMAL)) {
+				dm->calcLoopNormals(dm, true, (float)M_PI);
+			}
+
+			clnors = CustomData_get_layer(ldata, CD_NORMAL);
+			vert_clnors = MEM_mallocN(sizeof(*vert_clnors) * (size_t)numVerts, __func__);
+			BKE_mesh_normals_loop_to_vertex(numVerts, dm->getLoopArray(dm), dm->getNumLoops(dm), clnors, vert_clnors);
+		}
+		else {
+			direction = MOD_DISP_DIR_NOR;
+		}
+	}
+
 	for (i = 0; i < numVerts; i++) {
 		TexResult texres;
 		float strength = dmd->strength;
@@ -240,7 +266,7 @@ static void displaceModifier_do(
 		delta *= strength;
 		CLAMP(delta, -10000, 10000);
 
-		switch (dmd->direction) {
+		switch (direction) {
 			case MOD_DISP_DIR_X:
 				vertexCos[i][0] += delta;
 				break;
@@ -260,12 +286,19 @@ static void displaceModifier_do(
 				vertexCos[i][1] += delta * (mvert[i].no[1] / 32767.0f);
 				vertexCos[i][2] += delta * (mvert[i].no[2] / 32767.0f);
 				break;
+			case MOD_DISP_DIR_CLNOR:
+				madd_v3_v3fl(vertexCos[i], vert_clnors[i], delta);
+				break;
 		}
 	}
 
 	if (tex_co) {
 		MEM_freeN(tex_co);
 	}
+
+	if (vert_clnors) {
+		MEM_freeN(vert_clnors);
+	}
 }
 
 static void deformVerts(ModifierData *md, Object *ob,