Sculpt: Fix T104090: Automask topology not constrained by brush radius

Referenced by issue #104090, Regression: Topology Auto-Masking no longer recognizing disconnected geometry
Referenced by issue #104618, Regression: Topology Auto-Masking breaks on Projected strokes

release/datafiles/locale

@@ -1 +1 @@
-Subproject commit f1425d8a7fc38e8111c2a9e125f0e7877dcd0fdf
+Subproject commit 08b372721b9b33a16f380cab23b2e5ded738ea96

release/scripts/addons

@@ -1 +1 @@
-Subproject commit c0a678d3686a591eb3041cc72b60aec2857d389a
+Subproject commit d887a4ea6b2a9d64b926034d4e78ecf7a48ca979

source/blender/editors/sculpt_paint/sculpt_automasking.cc

@@ -177,11 +177,29 @@ static float sculpt_automasking_normal_calc(SculptSession *ss,
   return 1.0f;
 }
 
+static bool sculpt_automasking_is_constrained_by_radius(const Brush *br)
+{
+  /* 2D falloff is not constrained by radius. */
+  if (br->falloff_shape == PAINT_FALLOFF_SHAPE_TUBE) {
+    return false;
+  }
+
+  if (ELEM(br->sculpt_tool, SCULPT_TOOL_GRAB, SCULPT_TOOL_THUMB, SCULPT_TOOL_ROTATE)) {
+    return true;
+  }
+  return false;
+}
+
 static bool SCULPT_automasking_needs_factors_cache(const Sculpt *sd, const Brush *brush)
 {
 
   const int automasking_flags = sculpt_automasking_mode_effective_bits(sd, brush);
 
+  if (automasking_flags & BRUSH_AUTOMASKING_TOPOLOGY &&
+      sculpt_automasking_is_constrained_by_radius(brush)) {
+    return true;
+  }
+
   if (automasking_flags & (BRUSH_AUTOMASKING_BOUNDARY_EDGES |
                            BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS | BRUSH_AUTOMASKING_VIEW_NORMAL)) {
     return brush && brush->automasking_boundary_edges_propagation_steps != 1;
@@ -537,7 +555,8 @@ float SCULPT_automasking_factor_get(AutomaskingCache *automasking,
     return automasking_factor_end(ss, automasking, vert, 0.0f);
   }
 
-  if (automasking->settings.flags & BRUSH_AUTOMASKING_TOPOLOGY &&
+  if (!automasking->settings.topology_use_brush_limit &&
+      automasking->settings.flags & BRUSH_AUTOMASKING_TOPOLOGY &&
       SCULPT_vertex_island_get(ss, vert) != automasking->settings.initial_island_nr) {
     return 0.0f;
   }
@@ -592,6 +611,53 @@ struct AutomaskFloodFillData {
   char symm;
 };
 
+static bool automask_floodfill_cb(
+    SculptSession *ss, PBVHVertRef from_v, PBVHVertRef to_v, bool /*is_duplicate*/, void *userdata)
+{
+  AutomaskFloodFillData *data = (AutomaskFloodFillData *)userdata;
+
+  *(float *)SCULPT_vertex_attr_get(to_v, ss->attrs.automasking_factor) = 1.0f;
+  *(float *)SCULPT_vertex_attr_get(from_v, ss->attrs.automasking_factor) = 1.0f;
+  return (!data->use_radius ||
+          SCULPT_is_vertex_inside_brush_radius_symm(
+              SCULPT_vertex_co_get(ss, to_v), data->location, data->radius, data->symm));
+}
+
+static void SCULPT_topology_automasking_init(Sculpt *sd, Object *ob)
+{
+  SculptSession *ss = ob->sculpt;
+  Brush *brush = BKE_paint_brush(&sd->paint);
+
+  if (BKE_pbvh_type(ss->pbvh) == PBVH_FACES && !ss->pmap) {
+    BLI_assert_unreachable();
+    return;
+  }
+
+  const int totvert = SCULPT_vertex_count_get(ss);
+  for (int i : IndexRange(totvert)) {
+    PBVHVertRef vertex = BKE_pbvh_index_to_vertex(ss->pbvh, i);
+
+    (*(float *)SCULPT_vertex_attr_get(vertex, ss->attrs.automasking_factor)) = 0.0f;
+  }
+
+  /* Flood fill automask to connected vertices. Limited to vertices inside
+   * the brush radius if the tool requires it. */
+  SculptFloodFill flood;
+  SCULPT_floodfill_init(ss, &flood);
+  const float radius = ss->cache ? ss->cache->radius : FLT_MAX;
+  SCULPT_floodfill_add_active(sd, ob, ss, &flood, radius);
+
+  AutomaskFloodFillData fdata = {0};
+
+  fdata.radius = radius;
+  fdata.use_radius = ss->cache && sculpt_automasking_is_constrained_by_radius(brush);
+  fdata.symm = SCULPT_mesh_symmetry_xyz_get(ob);
+
+  copy_v3_v3(fdata.location, SCULPT_active_vertex_co_get(ss));
+  SCULPT_floodfill_execute(ss, &flood, automask_floodfill_cb, &fdata);
+  SCULPT_floodfill_free(&flood);
+}
+
 static void sculpt_face_sets_automasking_init(Sculpt *sd, Object *ob)
 {
   SculptSession *ss = ob->sculpt;
@@ -866,7 +932,13 @@ AutomaskingCache *SCULPT_automasking_cache_init(Sculpt *sd, Brush *brush, Object
   /* Additive modes. */
   if (SCULPT_is_automasking_mode_enabled(sd, brush, BRUSH_AUTOMASKING_TOPOLOGY)) {
     SCULPT_vertex_random_access_ensure(ss);
+
+    if (sculpt_automasking_is_constrained_by_radius(brush)) {
+      automasking->settings.topology_use_brush_limit = true;
+      SCULPT_topology_automasking_init(sd, ob);
+    }
   }
+
   if (SCULPT_is_automasking_mode_enabled(sd, brush, BRUSH_AUTOMASKING_FACE_SETS)) {
     SCULPT_vertex_random_access_ensure(ss);
     sculpt_face_sets_automasking_init(sd, ob);

source/blender/editors/sculpt_paint/sculpt_intern.h

@@ -418,6 +418,8 @@ typedef struct AutomaskingSettings {
 
   float start_normal_limit, start_normal_falloff;
   float view_normal_limit, view_normal_falloff;
+
+  bool topology_use_brush_limit;
 } AutomaskingSettings;
 
 typedef struct AutomaskingCache {