Add a Self option to the Exact boolean modifier. Fixes T52425.

With this option, self-intersections in either or both operands
will be handled properly (if both sides are piecewise winding
number constant, and maybe some other cases too).
In the Boolean tool, this flag was there already but the code
forced a unary operation in that case; this commit corrects it
to make a binary operation. This flag makes the code slower, which
is why it is an option and not an always-on thing.

source/blender/blenloader/intern/versioning_290.c

@@ -513,6 +513,7 @@ void blo_do_versions_290(FileData *fd, Library *UNUSED(lib), Main *bmain)
           if (md->type == eModifierType_Boolean) {
             BooleanModifierData *bmd = (BooleanModifierData *)md;
             bmd->solver = eBooleanModifierSolver_Fast;
+            bmd->flag = 0;
           }
         }
       }

source/blender/bmesh/tools/bmesh_boolean.cc

@@ -340,8 +340,8 @@ static bool bmesh_boolean(BMesh *bm,
   IMesh m_in = mesh_from_bm(bm, looptris, looptris_tot, &m_triangulated, &arena);
   std::function<int(int)> shape_fn;
   int nshapes;
-  if (use_self) {
-    /* Unary boolean operation. Want every face where test_fn doesn't return -1. */
+  if (use_self && boolean_mode == BoolOpType::None) {
+    /* Unary knife operation. Want every face where test_fn doesn't return -1. */
     nshapes = 1;
     shape_fn = [bm, test_fn, user_data](int f) {
       BMFace *bmf = BM_face_at_index(bm, f);

source/blender/makesdna/DNA_modifier_types.h

@@ -863,7 +863,7 @@ typedef struct BooleanModifierData {
   struct Object *object;
   char operation;
   char solver;
-  char _pad[1];
+  char flag;
   char bm_flag;
   float double_threshold;
 } BooleanModifierData;
@@ -879,6 +879,10 @@ typedef enum {
   eBooleanModifierSolver_Exact = 1,
 } BooleanModifierSolver;
 
+enum {
+  eBooleanModifierFlag_Self = (1 << 0),
+};
+
 /* bm_flag only used when G_DEBUG. */
 enum {
   eBooleanModifierBMeshFlag_BMesh_Separate = (1 << 0),

source/blender/makesrna/intern/rna_modifier.c

@@ -2863,6 +2863,11 @@ static void rna_def_modifier_boolean(BlenderRNA *brna)
   RNA_def_property_ui_text(prop, "Solver", "Method for calculating booleans");
   RNA_def_property_update(prop, 0, "rna_Modifier_update");
 
+  prop = RNA_def_property(srna, "use_self", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(prop, NULL, "flag", eBooleanModifierFlag_Self);
+  RNA_def_property_ui_text(prop, "Self", "Allow self-intersection in operands");
+  RNA_def_property_update(prop, 0, "rna_Modifier_update");
+
   /* BMesh debugging options, only used when G_DEBUG is set */
 
   /* BMesh intersection options */

source/blender/modifiers/intern/MOD_boolean.c

@@ -77,6 +77,7 @@ static void initData(ModifierData *md)
   bmd->double_threshold = 1e-6f;
   bmd->operation = eBooleanModifierOp_Difference;
   bmd->solver = eBooleanModifierSolver_Exact;
+  bmd->flag = 0;
 }
 
 static bool isDisabled(const struct Scene *UNUSED(scene),
@@ -319,15 +320,18 @@ static Mesh *modifyMesh(ModifierData *md, const ModifierEvalContext *ctx, Mesh *
 
 #ifdef WITH_GMP
         const bool use_exact = bmd->solver == eBooleanModifierSolver_Exact;
+        const bool use_self = (bmd->flag & eBooleanModifierFlag_Self) != 0;
 #else
         if (bmd->solver == eBooleanModifierSolver_Exact) {
           BKE_modifier_set_error(md, "Compiled without GMP, using fast solver");
         }
         const bool use_exact = false;
+        const bool use_self = false;
 #endif
 
         if (use_exact) {
-          BM_mesh_boolean(bm, looptris, tottri, bm_face_isect_pair, NULL, false, bmd->operation);
+          BM_mesh_boolean(
+              bm, looptris, tottri, bm_face_isect_pair, NULL, use_self, bmd->operation);
         }
         else {
           BM_mesh_intersect(bm,
@@ -393,7 +397,10 @@ static void panel_draw(const bContext *UNUSED(C), Panel *panel)
   uiItemR(layout, ptr, "object", 0, NULL, ICON_NONE);
   uiItemR(layout, ptr, "solver", UI_ITEM_R_EXPAND, NULL, ICON_NONE);
 
-  if (!use_exact) {
+  if (use_exact) {
+    uiItemR(layout, ptr, "use_self", 0, NULL, ICON_NONE);
+  }
+  else {
     uiItemR(layout, ptr, "double_threshold", 0, NULL, ICON_NONE);
   }