Fix T74958: Infinite loop on using strip as modifier mask

Add recursion check before assigning strip as a mask for modifier.
Same check is used for recursion check when reassigning effect input, so it
should not be possible to create recursion at all.

source/blender/blenkernel/BKE_sequencer.h

@@ -624,6 +624,7 @@ void BKE_sequencer_color_balance_apply(struct StripColorBalance *cb,
 
 void BKE_sequencer_all_free_anim_ibufs(struct Scene *scene, int cfra);
 bool BKE_sequencer_check_scene_recursion(struct Scene *scene, struct ReportList *reports);
+bool BKE_sequencer_render_loop_check(struct Sequence *seq_main, struct Sequence *seq);
 
 #ifdef __cplusplus
 }

source/blender/blenkernel/intern/sequencer.c

@@ -6046,3 +6046,26 @@ bool BKE_sequencer_check_scene_recursion(Scene *scene, ReportList *reports)
 
   return false;
 }
+
+/* Check if "seq_main" (indirectly) uses strip "seq". */
+bool BKE_sequencer_render_loop_check(Sequence *seq_main, Sequence *seq)
+{
+  if (seq_main == seq) {
+    return true;
+  }
+
+  if (seq_main->seq1 && BKE_sequencer_render_loop_check(seq_main->seq1, seq) ||
+      seq_main->seq2 && BKE_sequencer_render_loop_check(seq_main->seq2, seq) ||
+      seq_main->seq3 && BKE_sequencer_render_loop_check(seq_main->seq3, seq)) {
+    return true;
+  }
+
+  SequenceModifierData *smd;
+  for (smd = seq_main->modifiers.first; smd; smd = smd->next) {
+    if (smd->mask_sequence && BKE_sequencer_render_loop_check(smd->mask_sequence, seq)) {
+      return true;
+    }
+  }
+
+  return false;
+}

source/blender/editors/space_sequencer/sequencer_edit.c

@@ -467,30 +467,6 @@ static bool seq_is_parent(Sequence *par, Sequence *seq)
   return ((par->seq1 == seq) || (par->seq2 == seq) || (par->seq3 == seq));
 }
 
-static bool seq_is_predecessor(Sequence *pred, Sequence *seq)
-{
-  if (!pred) {
-    return 0;
-  }
-  if (pred == seq) {
-    return 0;
-  }
-  if (seq_is_parent(pred, seq)) {
-    return 1;
-  }
-  if (pred->seq1 && seq_is_predecessor(pred->seq1, seq)) {
-    return 1;
-  }
-  if (pred->seq2 && seq_is_predecessor(pred->seq2, seq)) {
-    return 1;
-  }
-  if (pred->seq3 && seq_is_predecessor(pred->seq3, seq)) {
-    return 1;
-  }
-
-  return 0;
-}
-
 /** \} */
 
 /* -------------------------------------------------------------------- */
@@ -2233,14 +2209,16 @@ static int sequencer_reassign_inputs_exec(bContext *C, wmOperator *op)
   }
 
   if (!seq_effect_find_selected(
-          scene, last_seq, last_seq->type, &seq1, &seq2, &seq3, &error_msg)) {
+          scene, last_seq, last_seq->type, &seq1, &seq2, &seq3, &error_msg) ||
+      BKE_sequence_effect_get_num_inputs(last_seq->type) == 0) {
     BKE_report(op->reports, RPT_ERROR, error_msg);
     return OPERATOR_CANCELLED;
   }
   /* Check if reassigning would create recursivity. */
-  if (seq_is_predecessor(seq1, last_seq) || seq_is_predecessor(seq2, last_seq) ||
-      seq_is_predecessor(seq3, last_seq)) {
-    BKE_report(op->reports, RPT_ERROR, "Cannot reassign inputs: no cycles allowed");
+  if (BKE_sequencer_render_loop_check(seq1, last_seq) ||
+      BKE_sequencer_render_loop_check(seq2, last_seq) ||
+      BKE_sequencer_render_loop_check(seq3, last_seq)) {
+    BKE_report(op->reports, RPT_ERROR, "Cannot reassign inputs: recursion detected.");
     return OPERATOR_CANCELLED;
   }
 

source/blender/makesrna/intern/rna_sequencer.c

@@ -862,6 +862,43 @@ static int rna_Sequence_input_count_get(PointerRNA *ptr)
   return BKE_sequence_effect_get_num_inputs(seq->type);
 }
 
+static void rna_Sequence_input_set(PointerRNA *ptr,
+                                   PointerRNA ptr_value,
+                                   struct ReportList *reports,
+                                   int input_num)
+{
+
+  Sequence *seq = ptr->data;
+  Sequence *input = ptr_value.data;
+
+  if (BKE_sequencer_render_loop_check(input, seq)) {
+    BKE_report(reports, RPT_ERROR, "Cannot reassign inputs: recursion detected.");
+    return;
+  }
+
+  switch (input_num) {
+    case 1:
+      seq->seq1 = input;
+      break;
+    case 2:
+      seq->seq2 = input;
+      break;
+  }
+}
+
+static void rna_Sequence_input_1_set(PointerRNA *ptr,
+                                     PointerRNA ptr_value,
+                                     struct ReportList *reports)
+{
+  rna_Sequence_input_set(ptr, ptr_value, reports, 1);
+}
+
+static void rna_Sequence_input_2_set(PointerRNA *ptr,
+                                     PointerRNA ptr_value,
+                                     struct ReportList *reports)
+{
+  rna_Sequence_input_set(ptr, ptr_value, reports, 2);
+}
 #  if 0
 static void rna_SoundSequence_filename_set(PointerRNA *ptr, const char *value)
 {
@@ -1272,6 +1309,24 @@ static void rna_Sequence_modifier_clear(Sequence *seq, bContext *C)
   WM_main_add_notifier(NC_SCENE | ND_SEQUENCER, NULL);
 }
 
+static void rna_SequenceModifier_strip_set(PointerRNA *ptr,
+                                           PointerRNA value,
+                                           struct ReportList *reports)
+{
+  SequenceModifierData *smd = ptr->data;
+  Scene *scene = (Scene *)ptr->owner_id;
+  Editing *ed = BKE_sequencer_editing_get(scene, false);
+  Sequence *seq = sequence_get_by_modifier(ed, smd);
+  Sequence *target = (Sequence *)value.data;
+
+  if (target != NULL && BKE_sequencer_render_loop_check(target, seq)) {
+    BKE_report(reports, RPT_ERROR, "Recursion detected, can not use this strip");
+    return;
+  }
+
+  smd->mask_sequence = target;
+}
+
 static float rna_Sequence_fps_get(PointerRNA *ptr)
 {
   Scene *scene = (Scene *)ptr->owner_id;
@@ -2193,6 +2248,7 @@ static void rna_def_effect_inputs(StructRNA *srna, int count)
     prop = RNA_def_property(srna, "input_1", PROP_POINTER, PROP_NONE);
     RNA_def_property_pointer_sdna(prop, NULL, "seq1");
     RNA_def_property_flag(prop, PROP_EDITABLE | PROP_NEVER_NULL);
+    RNA_def_property_pointer_funcs(prop, NULL, "rna_Sequence_input_1_set", NULL, NULL);
     RNA_def_property_ui_text(prop, "Input 1", "First input for the effect strip");
   }
 
@@ -2200,6 +2256,7 @@ static void rna_def_effect_inputs(StructRNA *srna, int count)
     prop = RNA_def_property(srna, "input_2", PROP_POINTER, PROP_NONE);
     RNA_def_property_pointer_sdna(prop, NULL, "seq2");
     RNA_def_property_flag(prop, PROP_EDITABLE | PROP_NEVER_NULL);
+    RNA_def_property_pointer_funcs(prop, NULL, "rna_Sequence_input_2_set", NULL, NULL);
     RNA_def_property_ui_text(prop, "Input 2", "Second input for the effect strip");
   }
 
@@ -3053,8 +3110,11 @@ static void rna_def_modifier(BlenderRNA *brna)
 
   prop = RNA_def_property(srna, "input_mask_strip", PROP_POINTER, PROP_NONE);
   RNA_def_property_pointer_sdna(prop, NULL, "mask_sequence");
-  RNA_def_property_pointer_funcs(
-      prop, NULL, NULL, NULL, "rna_SequenceModifier_otherSequence_poll");
+  RNA_def_property_pointer_funcs(prop,
+                                 NULL,
+                                 "rna_SequenceModifier_strip_set",
+                                 NULL,
+                                 "rna_SequenceModifier_otherSequence_poll");
   RNA_def_property_flag(prop, PROP_EDITABLE);
   RNA_def_property_ui_text(prop, "Mask Strip", "Strip used as mask input for the modifier");
   RNA_def_property_update(prop, NC_SCENE | ND_SEQUENCER, "rna_SequenceModifier_update");