Sculpt: more nasty customdata bugs

* Fixed mesh_sculpt_vertex_color_add_exec,
  it's post-exec undo push was overwriting
  the mesh cd layout.

source/blender/blenkernel/BKE_customdata.h

@@ -631,6 +631,7 @@ void CustomData_unmark_temporary_nocopy(struct CustomData *data);
 void CustomData_mark_temporary_nocopy(struct CustomData *data);
 
 int CustomData_get_elem_size(CustomDataLayer *layer);
+void CustomData_regen_active_refs(CustomData *data);
 
 #ifdef __cplusplus
 }

source/blender/blenkernel/intern/customdata.c

@@ -2240,10 +2240,21 @@ void CustomData_update_typemap(CustomData *data)
   }
 }
 
-void customdata_regen_active_refs(CustomData *data)
+ATTR_NO_OPT void CustomData_regen_active_refs(CustomData *data)
 {
   int i, j;
-  bool changed = false;
+
+  for (int i = 0; i < CD_NUMTYPES; i++) {
+    data->typemap[i] = -1;
+  }
+
+  for (i = 0, j = 0; i < data->totlayer; i++) {
+    CustomDataLayer *layer = &data->layers[i];
+
+    if (data->typemap[layer->type] == -1) {
+      data->typemap[layer->type] = i;
+    }
+  }
 
   /* explicitly flag active layers */
   for (i = 0, j = 0; i < data->totlayer; i++) {
@@ -2283,6 +2294,15 @@ void customdata_regen_active_refs(CustomData *data)
 
     int n = layer - base;
 
+    if (n < 0) {
+      printf("error!\n");
+      for (int j = 0; j < data->totlayer; j++) {
+        printf("%s", i == j ? "->" : "  ");
+        printf("%d : \"%s\"\n",
+               data->layers[i].type,
+               data->layers[i].name ? data->layers[i].name : "");
+      }
+    }
     if (layer->active) {
       base->active = n;
     }
@@ -2338,7 +2358,7 @@ void CustomData_copy_all_layout(const struct CustomData *source, struct CustomDa
     }
   }
 
-  customdata_regen_active_refs(dest);
+  CustomData_regen_active_refs(dest);
 }
 
 bool CustomData_merge(const struct CustomData *source,
@@ -2430,7 +2450,7 @@ bool CustomData_merge(const struct CustomData *source,
   }
 
   CustomData_update_typemap(dest);
-  customdata_regen_active_refs(dest);
+  CustomData_regen_active_refs(dest);
 
   return changed;
 }
@@ -3191,18 +3211,6 @@ void CustomData_free_temporary(CustomData *data, int totelem)
   int i, j;
   bool changed = false;
 
-  /* explicitly flag active layers */
-  for (i = 0, j = 0; i < data->totlayer; i++) {
-    CustomDataLayer *layer = &data->layers[i];
-    CustomDataLayer *base = data->layers + data->typemap[layer->type];
-    int n = layer - base;
-
-    layer->active = n == base->active;
-    layer->active_clone = n == base->active_clone;
-    layer->active_mask = n == base->active_mask;
-    layer->active_rnd = n == base->active_rnd;
-  }
-
   /* free temp layers */
   for (i = 0, j = 0; i < data->totlayer; i++) {
     CustomDataLayer *layer = &data->layers[i];
@@ -3222,48 +3230,6 @@ void CustomData_free_temporary(CustomData *data, int totelem)
     }
   }
 
-  CustomData_update_typemap(data);
-
-  /* regenerate active refs */
-  for (int i = 0; i < CD_NUMTYPES; i++) {
-    if (data->typemap[i] != -1) {
-      CustomDataLayer *base = data->layers + data->typemap[i];
-      base->active = base->active_clone = base->active_mask = base->active_rnd = 0;
-    }
-  }
-
-  /* set active n in base layer for all types */
-  for (i = 0; i < data->totlayer; i++) {
-    CustomDataLayer *layer = &data->layers[i];
-    CustomDataLayer *base = data->layers + data->typemap[layer->type];
-
-    int n = layer - base;
-
-    if (layer->active) {
-      base->active = n;
-    }
-    if (layer->active_mask) {
-      base->active_mask = n;
-    }
-    if (layer->active_clone) {
-      base->active_clone = n;
-    }
-    if (layer->active_rnd) {
-      base->active_rnd = n;
-    }
-  }
-
-  /* set active n in all layers */
-  for (i = 0; i < data->totlayer; i++) {
-    CustomDataLayer *layer = &data->layers[i];
-    CustomDataLayer *base = data->layers + data->typemap[layer->type];
-
-    layer->active = base->active;
-    layer->active_mask = base->active_mask;
-    layer->active_clone = base->active_clone;
-    layer->active_rnd = base->active_rnd;
-  }
-
   data->totlayer = j;
 
   if (data->totlayer <= data->maxlayer - CUSTOMDATA_GROW) {
@@ -3271,6 +3237,9 @@ void CustomData_free_temporary(CustomData *data, int totelem)
     changed = true;
   }
 
+  CustomData_update_typemap(data);
+  CustomData_regen_active_refs(data);
+
   if (changed) {
     customData_update_offsets(data);
   }
@@ -4734,6 +4703,9 @@ void CustomData_from_bmesh_block(const CustomData *source,
   /* copies a layer at a time */
   int dest_i = 0;
   for (int src_i = 0; src_i < source->totlayer; src_i++) {
+    if (source->layers[src_i].flag & CD_FLAG_NOCOPY) {
+      continue;
+    }
 
     /* find the first dest layer with type >= the source type
      * (this should work because layers are ordered by type)
@@ -5788,5 +5760,5 @@ void CustomData_blend_read(BlendDataReader *reader, CustomData *data, int count)
   }
 
   CustomData_update_typemap(data);
-  customdata_regen_active_refs(data);  // check for corrupted active layer refs
+  CustomData_regen_active_refs(data);  // check for corrupted active layer refs
 }

source/blender/blenkernel/intern/paint.c

@@ -1876,10 +1876,11 @@ void BKE_sculpt_update_object_after_eval(Depsgraph *depsgraph, Object *ob_eval)
    * other data when modifiers change the mesh. */
   Object *ob_orig = DEG_get_original_object(ob_eval);
   Mesh *me_eval = BKE_object_get_evaluated_mesh(ob_eval);
+  Mesh *me_orig = BKE_object_get_original_mesh(ob_orig);
 
   BLI_assert(me_eval != NULL);
   sculpt_update_object(depsgraph, ob_orig, me_eval, false, false, false);
-  SCULPT_dynamic_topology_sync_layers(ob_orig, me_eval);
+  SCULPT_dynamic_topology_sync_layers(ob_orig, me_orig);
 }
 
 void BKE_sculpt_color_layer_create_if_needed(struct Object *object)

source/blender/bmesh/intern/bmesh_log.c

@@ -1677,8 +1677,6 @@ BMLogEntry *BM_log_entry_check_customdata(BMesh *bm, BMLog *log)
   CustomData *cd1[4] = {&bm->vdata, &bm->edata, &bm->ldata, &bm->pdata};
   CustomData *cd2[4] = {&entry->vdata, &entry->edata, &entry->ldata, &entry->pdata};
 
-  void customdata_regen_active_refs(CustomData * data);
-
   for (int i = 0; i < 4; i++) {
     if (!CustomData_layout_is_same(cd1[i], cd2[i])) {
       printf("Customdata changed for undo\n");

source/blender/editors/mesh/mesh_data.c

@@ -37,6 +37,7 @@
 #include "BKE_customdata.h"
 #include "BKE_editmesh.h"
 #include "BKE_mesh.h"
+#include "BKE_object.h"
 #include "BKE_report.h"
 
 #include "DEG_depsgraph.h"
@@ -511,7 +512,10 @@ static bool sculpt_vertex_color_remove_poll(bContext *C)
 }
 
 /* NOTE: keep in sync with #ED_mesh_uv_texture_add. */
-int ED_mesh_sculpt_color_add(Mesh *me, const char *name, const bool active_set, const bool do_init)
+ATTR_NO_OPT int ED_mesh_sculpt_color_add(Mesh *me,
+                                         const char *name,
+                                         const bool active_set,
+                                         const bool do_init)
 {
   BMEditMesh *em;
   int layernum;
@@ -775,15 +779,21 @@ void MESH_OT_vertex_color_remove(wmOperatorType *ot)
 
 /*********************** Sculpt Vertex Color Operators ************************/
 
+void SCULPT_dynamic_topology_sync_layers(Object *ob, Mesh *me);
+
 static int mesh_sculpt_vertex_color_add_exec(bContext *C, wmOperator *UNUSED(op))
 {
   Object *ob = ED_object_context(C);
-  Mesh *me = ob->data;
+  Mesh *me = BKE_object_get_original_mesh(ob);
 
   if (ED_mesh_sculpt_color_add(me, NULL, true, true) == -1) {
     return OPERATOR_CANCELLED;
   }
 
+  /* have to call this to prevent the undo system from
+    overwriting the CD layout with ss->bm's layout*/
+  SCULPT_dynamic_topology_sync_layers(ob, me);
+
   return OPERATOR_FINISHED;
 }
 

source/blender/editors/sculpt_paint/sculpt_dyntopo.c

@@ -641,7 +641,7 @@ void SCULPT_dyntopo_node_layers_add(SculptSession *ss)
 /**
   Syncs customdata layers with internal bmesh, but ignores deleted layers.
 */
-void SCULPT_dynamic_topology_sync_layers(Object *ob, Mesh *me)
+ATTR_NO_OPT void SCULPT_dynamic_topology_sync_layers(Object *ob, Mesh *me)
 {
   SculptSession *ss = ob->sculpt;
 
@@ -736,26 +736,18 @@ void SCULPT_dynamic_topology_sync_layers(Object *ob, Mesh *me)
         modified |= idx - baseidx != cl2->active_clone;
         cl2->active_clone = idx - baseidx;
       }
-
-      for (int k = baseidx; k < data2->totlayer; k++) {
-        CustomDataLayer *cl3 = data2->layers + k;
-
-        if (cl3->type != cl2->type) {
-          break;
-        }
-
-        // based off of how CustomData_set_layer_XXXX_index works
-
-        cl3->active = (cl2->active + baseidx) - k;
-        cl3->active_rnd = (cl2->active_rnd + baseidx) - k;
-        cl3->active_mask = (cl2->active_mask + baseidx) - k;
-        cl3->active_clone = (cl2->active_clone + baseidx) - k;
-      }
     }
 
     BLI_array_free(newlayers);
   }
 
+  if (modified && ss->bm) {
+    CustomData_regen_active_refs(&ss->bm->vdata);
+    CustomData_regen_active_refs(&ss->bm->edata);
+    CustomData_regen_active_refs(&ss->bm->ldata);
+    CustomData_regen_active_refs(&ss->bm->pdata);
+  }
+
   if (modified) {
     SCULPT_dyntopo_node_layers_update_offsets(ss);
   }