Fix dyntopo undo crash

source/blender/blenkernel/BKE_brush_engine.h

@@ -143,7 +143,7 @@ typedef struct BrushTex {
   char idname[64], name[64];
 
   BrushChannelSet *channels;
-  MTex *__mtex;  // do not access directly. except for the actual evaluation code.
+  MTex __mtex;  // do not access directly. except for the actual evaluation code.
 } BrushTex;
 
 BrushTex *BKE_brush_tex_create();

source/blender/blenkernel/intern/brush_engine_presets.c

@@ -947,8 +947,21 @@ void BKE_brush_channelset_ui_init(Brush *brush, int tool)
   SHOWWRK(radius_unit);
   SHOWWRK(use_frontface);
 
-  SHOWWRK(autosmooth);
-  SHOWWRK(topology_rake);
+  if (!ELEM(tool, SCULPT_TOOL_PAINT, SCULPT_TOOL_SMEAR)) {
+    SHOWWRK(autosmooth);
+    SHOWWRK(topology_rake);
+  }
+
+  if (ELEM(tool, SCULPT_TOOL_PAINT, SCULPT_TOOL_SMEAR)) {
+    SHOWWRK(vcol_boundary_factor);
+    SHOWWRK(vcol_boundary_exponent);
+    SHOWWRK(vcol_boundary_radius_scale);
+    SHOWWRK(vcol_boundary_spacing);
+  }
+  else if (tool == SCULPT_TOOL_VCOL_BOUNDARY) {
+    SHOWWRK(vcol_boundary_exponent);
+  }
+
   SHOWWRK(normal_radius_factor);
   SHOWWRK(hardness);
 
@@ -1092,6 +1105,14 @@ void BKE_brush_builtin_create(Brush *brush, int tool)
       GETCH(strength)->fvalue = 1.0f;
       GETCH(dyntopo_disabled)->ivalue = 1;
       break;
+    case SCULPT_TOOL_SMEAR:
+      BRUSHSET_SET_FLOAT(chset, spacing, 5.0f);
+      BRUSHSET_SET_FLOAT(chset, strength, 1.0f);
+      BRUSHSET_LOOKUP(chset, strength)->mappings[BRUSH_MAPPING_PRESSURE].flag &=
+          ~BRUSH_MAPPING_ENABLED;
+      BRUSHSET_SET_BOOL(chset, dyntopo_disabled, true);
+      break;
+
     case SCULPT_TOOL_SLIDE_RELAX:
       GETCH(spacing)->fvalue = 10;
       GETCH(strength)->fvalue = 1.0f;

source/blender/bmesh/intern/bmesh_log.c

@@ -57,6 +57,34 @@
 
 #define CUSTOMDATA
 
+#ifdef DEBUG_LOG_REFCOUNTNG
+static struct {
+  char tag[4192];
+} namestack[256] = {0};
+int namestack_i = 1;
+
+void _namestack_push(const char *name)
+{
+  namestack_i++;
+
+  strcpy(namestack[namestack_i].tag, namestack[namestack_i - 1].tag);
+  strcat(namestack[namestack_i].tag, ".");
+  strcat(namestack[namestack_i].tag, name);
+}
+
+#  define namestack_push() _namestack_push(__func__)
+
+void namestack_pop()
+{
+  namestack_i--;
+}
+
+#  define namestack_head_name namestack[namestack_i].tag
+#else
+#  define namestack_push()
+#  define namestack_pop()
+#  define namestack_head_name ""
+#endif
 //#define DO_LOG_PRINT
 
 #ifdef DO_LOG_PRINT
@@ -187,6 +215,30 @@ struct BMLog {
   bool dead;
 };
 
+//#define PRINT_LOG_REF_COUNTING
+
+static void _bm_log_addref(BMLog *log, const char *func)
+{
+  log->refcount++;
+
+#ifdef PRINT_LOG_REF_COUNTING
+  printf("%d %s: bm_log_addref: %p\n", log->refcount, namestack_head_name, log);
+  fflush(stdout);
+#endif
+}
+
+static void _bm_log_decref(BMLog *log, const char *func)
+{
+  log->refcount--;
+#ifdef PRINT_LOG_REF_COUNTING
+  printf("%d %s: bm_log_decref: %p\n", log->refcount, namestack_head_name, log);
+  fflush(stdout);
+#endif
+}
+
+#define bm_log_addref(log) _bm_log_addref(log, __func__)
+#define bm_log_decref(log) _bm_log_decref(log, __func__)
+
 typedef struct BMLogVert {
 #ifdef DO_LOG_PRINT
   char msg[64];
@@ -246,7 +298,7 @@ static void full_copy_load(BMesh *bm, BMLog *log, BMLogEntry *entry);
 
 BMLogEntry *bm_log_entry_add_ex(
     BMesh *bm, BMLog *log, bool combine_with_last, BMLogEntryType type, BMLogEntry *last_entry);
-static void bm_log_entry_free(BMLogEntry *entry);
+static bool bm_log_entry_free(BMLogEntry *entry);
 static bool bm_log_free_direct(BMLog *log, bool safe_mode);
 
 static void *log_ghash_lookup(BMLog *log, GHash *gh, const void *key)
@@ -1172,13 +1224,15 @@ static void bm_log_entry_free_direct(BMLogEntry *entry)
 /* Free the data in a log entry
  * and handles bmlog ref counting
  * NOTE: does not free the log entry itself. */
-static void bm_log_entry_free(BMLogEntry *entry)
+static bool bm_log_entry_free(BMLogEntry *entry)
 {
   BMLog *log = entry->log;
   bool kill_log = false;
 
   if (log) {
-    log->refcount--;
+    namestack_push();
+    bm_log_decref(log);
+    namestack_pop();
 
     if (log->refcount < 0) {
       fprintf(stderr, "BMLog refcount error\n");
@@ -1193,6 +1247,8 @@ static void bm_log_entry_free(BMLogEntry *entry)
   if (kill_log) {
     bm_log_free_direct(log, true);
   }
+
+  return kill_log;
 }
 
 static int uint_compare(const void *a_v, const void *b_v)
@@ -1268,6 +1324,8 @@ BMLog *bm_log_from_existing_entries_create(BMesh *bm, BMLog *log, BMLogEntry *en
     log->entries.last = entry;
   }
 
+  namestack_push();
+
   for (entry = log->entries.first; entry; entry = entry->next) {
     BMLogEntry *entry2 = entry->combined_prev;
 
@@ -1275,13 +1333,15 @@ BMLog *bm_log_from_existing_entries_create(BMesh *bm, BMLog *log, BMLogEntry *en
       entry2->log = log;
       entry2 = entry2->combined_prev;
 
-      log->refcount++;
+      bm_log_addref(log);
     }
 
     entry->log = log;
-    log->refcount++;
+    bm_log_addref(log);
   }
 
+  namestack_pop();
+
   return log;
 }
 
@@ -1367,6 +1427,12 @@ static bool bm_log_free_direct(BMLog *log, bool safe_mode)
   return true;
 }
 
+// if true, make sure to call BM_log_free on the log
+bool BM_log_is_dead(BMLog *log)
+{
+  return log->dead;
+}
+
 bool BM_log_free(BMLog *log, bool safe_mode)
 {
   if (log->dead) {
@@ -1570,7 +1636,11 @@ BMLogEntry *bm_log_entry_add_ex(
 
   entry->log = log;
 
-  log->refcount++;
+  namestack_push();
+
+  bm_log_addref(log);
+
+  namestack_pop();
 
   if (combine_with_last) {
     if (!last_entry || last_entry == log->current_entry) {
@@ -1618,10 +1688,12 @@ BMLogEntry *BM_log_entry_add_ex(BMesh *bm, BMLog *log, bool combine_with_last)
  * This operation is only valid on the first and last entries in the
  * log. Deleting from the middle will assert.
  */
-void BM_log_entry_drop(BMLogEntry *entry)
+bool BM_log_entry_drop(BMLogEntry *entry)
 {
   BMLog *log = entry->log;
 
+  namestack_push();
+
   // go to head of entry subgroup
   while (entry->combined_next) {
     entry = entry->combined_next;
@@ -1647,9 +1719,11 @@ void BM_log_entry_drop(BMLogEntry *entry)
       entry2 = prev;
     }
 
+    namestack_pop();
     bm_log_entry_free(entry);
     MEM_freeN(entry);
-    return;
+
+    return false;
   }
 
   if (log && log->current_entry == entry) {
@@ -1670,8 +1744,12 @@ void BM_log_entry_drop(BMLogEntry *entry)
     entry2 = prev;
   }
 
-  bm_log_entry_free(entry);
+  bool ret = bm_log_entry_free(entry);
+
   MEM_freeN(entry);
+  namestack_pop();
+
+  return ret;
 }
 
 static void full_copy_load(BMesh *bm, BMLog *log, BMLogEntry *entry)

source/blender/bmesh/intern/bmesh_log.h

@@ -76,8 +76,11 @@ BMLogEntry *BM_log_entry_check_customdata(BMesh *bm, BMLog *log);
 /* Mark all used ids as unused for this node */
 void BM_log_cleanup_entry(BMLogEntry *entry);
 
-/* Remove an entry from the log */
-void BM_log_entry_drop(BMLogEntry *entry);
+/* Remove an entry from the log.
+   returns true if the log's refcount
+   reached zero and was freed*/
+bool BM_log_entry_drop(BMLogEntry *entry);
+bool BM_log_is_dead(BMLog *log);
 
 /* Undo one BMLogEntry.  node_layer_id is necassary to preserve node idxs with customdata, whose
  * layout might have changed */

source/blender/editors/sculpt_paint/sculpt_undo.c

@@ -1728,10 +1728,16 @@ static SculptUndoNode *sculpt_undo_face_sets_push(Object *ob, SculptUndoType typ
 
 void SCULPT_undo_ensure_bmlog(Object *ob)
 {
-  if (!ob->sculpt) {
+  SculptSession *ss = ob->sculpt;
+  Mesh *me = BKE_object_get_original_mesh(ob);
+
+  /*log exists or not in sculpt mode? good then*/
+  if (ss->bm_log || !ob->sculpt) {
     return;
   }
 
+  /*try to find log from entries in the undo stack*/
+
   UndoStack *ustack = ED_undo_stack_get();
 
   if (!ustack) {
@@ -1754,9 +1760,6 @@ void SCULPT_undo_ensure_bmlog(Object *ob)
 
   UndoSculpt *usculpt = sculpt_undosys_step_get_nodes(us);
 
-  SculptSession *ss = ob->sculpt;
-  Mesh *me = BKE_object_get_original_mesh(ob);
-
   if (!ss->bm && !(me->flag & ME_SCULPT_DYNAMIC_TOPOLOGY)) {
     return;
   }
@@ -1768,8 +1771,8 @@ void SCULPT_undo_ensure_bmlog(Object *ob)
 
   SculptUndoNode *unode = usculpt->nodes.first;
 
-  // this can happen in certain cases when going to/from other undo types
-  // I think.
+  /*when transition between undo step types the log might simply
+  have been freed, look for entries to rebuild it with*/
   if (!ss->bm_log) {
     if (unode && unode->bm_entry) {
       ss->bm_log = BM_log_from_existing_entries_create(ss->bm, unode->bm_entry);
@@ -2105,10 +2108,10 @@ SculptUndoNode *SCULPT_undo_push_node(Object *ob, PBVHNode *node, SculptUndoType
 
 void sculpt_undo_push_begin_ex(Object *ob, const char *name, bool no_first_entry_check)
 {
-  SCULPT_undo_ensure_bmlog(ob);
-
   UndoStack *ustack = ED_undo_stack_get();
 
+  SCULPT_undo_ensure_bmlog(ob);
+
   if (ob != NULL) {
     if (!no_first_entry_check && ob->sculpt && ob->sculpt->bm) {
       check_first_undo_entry_dyntopo(ob);
@@ -2124,6 +2127,23 @@ void sculpt_undo_push_begin_ex(Object *ob, const char *name, bool no_first_entry
   bContext *C = NULL;
 
   BKE_undosys_step_push_init_with_type(ustack, C, name, BKE_UNDOSYS_TYPE_SCULPT);
+
+  SculptSession *ss = ob->sculpt;
+
+  /*when pusing an undo node after
+    undoing to the start of the stack
+    the log ref count hits zero, we have to check it,
+    do cleanup and recreate it*/
+
+  if (ss && ss->bm && ss->bm_log && BM_log_is_dead(ss->bm_log)) {
+    // forcibly destroy all entries? the 'true' parameter
+    BM_log_free(ss->bm_log, true);
+    ss->bm_log = BM_log_create(ss->bm, ss->cd_dyn_vert);
+
+    if (ss->pbvh) {
+      BKE_pbvh_set_bm_log(ss->pbvh, ss->bm_log);
+    }
+  }
 }
 
 void SCULPT_undo_push_begin(Object *ob, const char *name)