Fix T42984 detail flood fill not respecting mask values for smooth/sharp

curves.

Issue here is that brush curve could return negative values. This would
result in overflow of mask values. Those were not visible during real
time preview because result would be clamped.

We had two functions in the code, one of which allowed negatives but I
don't think that we really want that, users have no control over the
negative values at all anyway.

Thanks to the reporter, Leon Cheung for figuring out the issue :)

source/blender/blenkernel/BKE_brush.h

@@ -66,8 +66,7 @@ void BKE_brush_randomize_texture_coordinates(struct UnifiedPaintSettings *ups, b
 
 /* brush curve */
 void BKE_brush_curve_preset(struct Brush *b, int preset);
-float BKE_brush_curve_strength_clamp(struct Brush *br, float p, const float len);
-float BKE_brush_curve_strength(struct Brush *br, float p, const float len); /* used for sculpt */
+float BKE_brush_curve_strength(struct Brush *br, float p, const float len);
 
 /* sampling */
 float BKE_brush_sample_tex_3D(const Scene *scene, struct Brush *br, const float point[3],

source/blender/blenkernel/intern/brush.c

@@ -964,7 +964,7 @@ void BKE_brush_randomize_texture_coordinates(UnifiedPaintSettings *ups, bool mas
 }
 
 /* Uses the brush curve control to find a strength value between 0 and 1 */
-float BKE_brush_curve_strength_clamp(Brush *br, float p, const float len)
+float BKE_brush_curve_strength(Brush *br, float p, const float len)
 {
 	float strength;
 
@@ -977,17 +977,6 @@ float BKE_brush_curve_strength_clamp(Brush *br, float p, const float len)
 
 	return strength;
 }
-/* same as above but can return negative values if the curve enables
- * used for sculpt only */
-float BKE_brush_curve_strength(Brush *br, float p, const float len)
-{
-	if (p >= len)
-		p = 1.0f;
-	else
-		p = p / len;
-
-	return curvemapping_evaluateF(br->curve, 0, p);
-}
 
 /* TODO: should probably be unified with BrushPainter stuff? */
 unsigned int *BKE_brush_gen_texture_cache(Brush *br, int half_side, bool use_secondary)
@@ -1045,7 +1034,7 @@ struct ImBuf *BKE_brush_gen_radial_control_imbuf(Brush *br, bool secondary)
 	for (i = 0; i < side; ++i) {
 		for (j = 0; j < side; ++j) {
 			float magn = sqrtf(pow2f(i - half) + pow2f(j - half));
-			im->rect_float[i * side + j] = BKE_brush_curve_strength_clamp(br, magn, half);
+			im->rect_float[i * side + j] = BKE_brush_curve_strength(br, magn, half);
 		}
 	}
 

source/blender/editors/sculpt_paint/paint_cursor.c

@@ -426,7 +426,7 @@ static int load_tex_cursor(Brush *br, ViewContext *vc, float zoom)
 				len = sqrtf(x * x + y * y);
 
 				if (len <= 1) {
-					float avg = BKE_brush_curve_strength_clamp(br, len, 1.0f);  /* Falloff curve */
+					float avg = BKE_brush_curve_strength(br, len, 1.0f);  /* Falloff curve */
 
 					buffer[index] = 255 - (GLubyte)(255 * avg);
 

source/blender/editors/sculpt_paint/paint_image_2d.c

@@ -359,7 +359,7 @@ static unsigned short *brush_painter_curve_mask_new(BrushPainter *painter, int d
 			float xy[2] = {x + xoff, y + yoff};
 			float len = len_v2(xy);
 
-			*m = (unsigned short)(65535.0f * BKE_brush_curve_strength_clamp(brush, len, radius));
+			*m = (unsigned short)(65535.0f * BKE_brush_curve_strength(brush, len, radius));
 		}
 	}
 

source/blender/editors/sculpt_paint/paint_image_proj.c

@@ -4286,7 +4286,7 @@ static void *do_projectpaint_thread(void *ph_v)
 				if (dist_sq <= brush_radius_sq) {
 					dist = sqrtf(dist_sq);
 
-					falloff = BKE_brush_curve_strength_clamp(ps->brush, dist, brush_radius);
+					falloff = BKE_brush_curve_strength(ps->brush, dist, brush_radius);
 
 					if (falloff > 0.0f) {
 						float texrgb[3];

source/blender/editors/sculpt_paint/paint_vertex.c

@@ -935,7 +935,7 @@ static float calc_vp_strength_col_dl(VPaint *vp, ViewContext *vc, const float co
 			else {
 				factor = 1.0f;
 			}
-			return factor * BKE_brush_curve_strength_clamp(brush, dist, brush_size_pressure);
+			return factor * BKE_brush_curve_strength(brush, dist, brush_size_pressure);
 		}
 	}
 	if (rgba)
@@ -3215,7 +3215,7 @@ static void gradientVert_update(DMGradient_userData *grad_data, int index)
 	/* no need to clamp 'alpha' yet */
 
 	/* adjust weight */
-	alpha = BKE_brush_curve_strength_clamp(grad_data->brush, alpha, 1.0f);
+	alpha = BKE_brush_curve_strength(grad_data->brush, alpha, 1.0f);
 
 	if (alpha != 0.0f) {
 		MDeformVert *dv = &me->dvert[index];