Stacked fullscreen area support (proper implemenation)

Adds support for stacked fullscreens. This basically means, if a user opens a
temporary fullscreen mode, such as the File Browser or the Image Editor render
view, from a different fullscreen, the "Back to Previous" function or the other
ways to escape those temporary fullscreens don't return to the split screen
layout but to the previous fullscreen he has been in.

I already committed something similar (f7e844570f) but that was only
supposed as a fix, it didn't work for the "Back to Previous" operator and the
implementation wasn't really reusable. This one looks a bit nicer + makes some
older hacks unnecessary :)

source/blender/editors/include/ED_screen.h

@@ -107,7 +107,7 @@ void    ED_screen_set_subwinactive(struct bContext *C, struct wmEvent *event);
 void    ED_screen_exit(struct bContext *C, struct wmWindow *window, struct bScreen *screen);
 void    ED_screen_animation_timer(struct bContext *C, int redraws, int refresh, int sync, int enable);
 void    ED_screen_animation_timer_update(struct bScreen *screen, int redraws, int refresh);
-void    ED_screen_restore_temp_type(struct bContext *C, ScrArea *sa, bool is_screen_change);
+void    ED_screen_restore_temp_type(struct bContext *C, ScrArea *sa);
 ScrArea *ED_screen_full_newspace(struct bContext *C, ScrArea *sa, int type);
 void    ED_screen_full_prevspace(struct bContext *C, ScrArea *sa);
 void    ED_screen_full_restore(struct bContext *C, ScrArea *sa);

source/blender/editors/render/render_view.c

@@ -194,6 +194,11 @@ ScrArea *render_view_open(bContext *C, int mx, int my)
 
 				/* makes ESC go back to prev space */
 				sima->flag |= SI_PREVSPACE;
+
+				/* we already had a fullscreen here -> mark new space as a stacked fullscreen */
+				if (sa->full) {
+					sa->flag |= AREA_FLAG_STACKED_FULLSCREEN;
+				}
 			}
 			else {
 				/* use any area of decent size */

source/blender/editors/screen/area.c

@@ -1645,6 +1645,8 @@ void ED_area_prevspace(bContext *C, ScrArea *sa)
 		/* no change */
 		return;
 	}
+	sa->flag &= ~AREA_FLAG_STACKED_FULLSCREEN;
+
 	ED_area_tag_redraw(sa);
 
 	/* send space change notifier */

source/blender/editors/screen/screen_edit.c

@@ -1762,18 +1762,16 @@ ScrArea *ED_screen_full_newspace(bContext *C, ScrArea *sa, int type)
 
 void ED_screen_full_prevspace(bContext *C, ScrArea *sa)
 {
-	wmWindow *win = CTX_wm_window(C);
-
-	ED_area_prevspace(C, sa);
-
-	if (sa->full) {
-		/* only toggle out of fullscreen if it wasn't set by the user (for stacked fullscreens) */
-		if (sa->flag & AREA_FLAG_TEMP_TYPE)
-			ED_screen_state_toggle(C, win, sa, SCREENMAXIMIZED);
+	if (sa->flag & AREA_FLAG_STACKED_FULLSCREEN) {
+		/* stacked fullscreen -> only go back to previous screen and don't toggle out of fullscreen */
+		ED_area_prevspace(C, sa);
+	}
+	else {
+		ED_screen_restore_temp_type(C, sa);
 	}
 }
 
-void ED_screen_restore_temp_type(bContext *C, ScrArea *sa, bool is_screen_change)
+void ED_screen_restore_temp_type(bContext *C, ScrArea *sa)
 {
 	/* incase nether functions below run */
 	ED_area_tag_redraw(sa);
@@ -1783,7 +1781,7 @@ void ED_screen_restore_temp_type(bContext *C, ScrArea *sa, bool is_screen_change
 		sa->flag &= ~AREA_FLAG_TEMP_TYPE;
 	}
 
-	if (is_screen_change && sa->full) {
+	if (sa->full) {
 		ED_screen_state_toggle(C, CTX_wm_window(C), sa, SCREENMAXIMIZED);
 	}
 }
@@ -1796,25 +1794,11 @@ void ED_screen_full_restore(bContext *C, ScrArea *sa)
 	bScreen *screen = CTX_wm_screen(C);
 	short state = (screen ? screen->state : SCREENMAXIMIZED);
 	
-	/* if fullscreen area has a secondary space (such as a file browser or fullscreen render 
-	 * overlaid on top of a existing setup) then return to the previous space */
+	/* if fullscreen area has a temporary space (such as a file browser or fullscreen render
+	 * overlaid on top of an existing setup) then return to the previous space */
 	
 	if (sl->next) {
-		/* specific checks for space types */
-
-		/* Special check added for non-render image window (back from fullscreen through "Back to Previous" button) */
-		if (sl->spacetype == SPACE_IMAGE) {
-			SpaceImage *sima = sa->spacedata.first;
-
-			if (sima->flag & (SI_PREVSPACE | SI_FULLWINDOW)) {
-				sima->flag &= ~SI_PREVSPACE;
-				sima->flag &= ~SI_FULLWINDOW;
-				ED_screen_full_prevspace(C, sa);
-			}
-			else
-				ED_screen_state_toggle(C, win, sa, state);
-		}
-		else if (sa->flag & AREA_FLAG_TEMP_TYPE) {
+		if (sa->flag & AREA_FLAG_TEMP_TYPE) {
 			ED_screen_full_prevspace(C, sa);
 		}
 		else {

source/blender/editors/screen/screen_ops.c

@@ -3697,9 +3697,9 @@ static int fullscreen_back_exec(bContext *C, wmOperator *op)
 		BKE_report(op->reports, RPT_ERROR, "No fullscreen areas were found");
 		return OPERATOR_CANCELLED;
 	}
-	
-	ED_screen_full_restore(C, sa);
-	
+
+	ED_screen_full_prevspace(C, sa);
+
 	return OPERATOR_FINISHED;
 }
 

source/blender/makesdna/DNA_screen_types.h

@@ -272,14 +272,16 @@ typedef struct ARegion {
 
 /* area->flag */
 enum {
-	HEADER_NO_PULLDOWN      = (1 << 0),
-	AREA_FLAG_DRAWJOINTO    = (1 << 1),
-	AREA_FLAG_DRAWJOINFROM  = (1 << 2),
-	AREA_TEMP_INFO          = (1 << 3),
-	AREA_FLAG_DRAWSPLIT_H   = (1 << 4),
-	AREA_FLAG_DRAWSPLIT_V   = (1 << 5),
+	HEADER_NO_PULLDOWN           = (1 << 0),
+	AREA_FLAG_DRAWJOINTO         = (1 << 1),
+	AREA_FLAG_DRAWJOINFROM       = (1 << 2),
+	AREA_TEMP_INFO               = (1 << 3),
+	AREA_FLAG_DRAWSPLIT_H        = (1 << 4),
+	AREA_FLAG_DRAWSPLIT_V        = (1 << 5),
 	/* used to check if we should switch back to prevspace (of a different type) */
-	AREA_FLAG_TEMP_TYPE	    = (1 << 6),
+	AREA_FLAG_TEMP_TYPE          = (1 << 6),
+	/* for temporary fullscreens (file browser, image editor render) that are opened above user set fullscreens */
+	AREA_FLAG_STACKED_FULLSCREEN = (1 << 7),
 };
 
 #define EDGEWIDTH	1

source/blender/windowmanager/intern/wm_event_system.c

@@ -1677,8 +1677,12 @@ static int wm_handler_fileselect_do(bContext *C, ListBase *handlers, wmEventHand
 				sa = handler->op_area;
 			}
 					
-			if (val == EVT_FILESELECT_OPEN) {
+			if (val == EVT_FILESELECT_OPEN || sa->full) {
 				ED_area_newspace(C, sa, SPACE_FILE);     /* 'sa' is modified in-place */
+				/* we already had a fullscreen here -> mark new space as a stacked fullscreen */
+				if (sa->full) {
+					sa->flag |= AREA_FLAG_STACKED_FULLSCREEN;
+				}
 			}
 			else {
 				sa = ED_screen_full_newspace(C, sa, SPACE_FILE);    /* sets context */
@@ -1702,15 +1706,11 @@ static int wm_handler_fileselect_do(bContext *C, ListBase *handlers, wmEventHand
 		case EVT_FILESELECT_CANCEL:
 		case EVT_FILESELECT_EXTERNAL_CANCEL:
 		{
-			/* XXX validate area and region? */
-			bScreen *screen = CTX_wm_screen(C);
-
 			/* remlink now, for load file case before removing*/
 			BLI_remlink(handlers, handler);
 
 			if (val != EVT_FILESELECT_EXTERNAL_CANCEL) {
-				ScrArea *sa = CTX_wm_area(C);
-				ED_screen_restore_temp_type(C, sa, screen != handler->filescreen);
+				ED_screen_full_prevspace(C, CTX_wm_area(C));
 			}
 
 			wm_handler_op_context(C, handler);
@@ -2494,7 +2494,6 @@ void WM_event_add_fileselect(bContext *C, wmOperator *op)
 	handler->op = op;
 	handler->op_area = CTX_wm_area(C);
 	handler->op_region = CTX_wm_region(C);
-	handler->filescreen = CTX_wm_screen(C);
 	
 	BLI_addhead(&win->modalhandlers, handler);
 	

source/blender/windowmanager/wm_event_system.h

@@ -65,9 +65,6 @@ typedef struct wmEventHandler {
 	struct ARegion *ui_region;          /* for derived/modal handlers */
 	struct ARegion *ui_menu;            /* for derived/modal handlers */
 
-	/* fileselect handler re-uses modal operator data */
-	struct bScreen *filescreen;         /* screen it started in, to validate exec */
-
 	/* drop box handler */
 	ListBase *dropboxes;