Fix T56799: Custom render passes missing when using Save Buffers

The problem here was that when a render result is allocated, the standard render passes are added according to the
pass bitfield. Then, when the render engine sets the result, it adds the additional passes which are then merged
into the main render result.

However, when using Save Buffers, the EXR is created before the actual render starts, so it's missing all those
additional passes.

To fix that, we need to query the render engine for a list of additional passes so they can be added before the EXR
is created. Luckily, there already is a function to do that for the node editor.

The same needs to be done when the EXR is loaded back.

Due to how that is implemented though (Render API calls into engine, engine calls back for each pass), if we have
multiple places that call this function there needs to be a way to tell which one the call came from in the pass
registration callback. Therefore, the original caller now provides a callback that is called for each pass.

source/blender/nodes/composite/nodes/node_composite_image.c

@@ -213,6 +213,21 @@ void node_cmp_rlayers_register_pass(bNodeTree *ntree, bNode *node, Scene *scene,
 	}
 }
 
+static void cmp_node_rlayer_create_outputs_cb(RenderEngine *UNUSED(engine), Scene *scene, SceneRenderLayer *srl,
+                                              const char *name, int UNUSED(channels), const char *UNUSED(chanid), int type)
+{
+	/* Register the pass in all scenes that have a render layer node for this layer.
+	 * Since multiple scenes can be used in the compositor, the code must loop over all scenes
+	 * and check whether their nodetree has a node that needs to be updated. */
+	/* NOTE: using G_MAIN seems valid here,
+	 * unless we want to register that for every other temp Main we could generate??? */
+	for (Scene *sce = G_MAIN->scene.first; sce; sce = sce->id.next) {
+		if (sce->nodetree) {
+			ntreeCompositRegisterPass(sce->nodetree, scene, srl, name, type);
+		}
+	}
+}
+
 static void cmp_node_rlayer_create_outputs(bNodeTree *ntree, bNode *node, LinkNodePair *available_sockets)
 {
 	Scene *scene = (Scene *)node->id;
@@ -228,7 +243,7 @@ static void cmp_node_rlayer_create_outputs(bNodeTree *ntree, bNode *node, LinkNo
 				node->storage = data;
 
 				RenderEngine *engine = RE_engine_create(engine_type);
-				engine_type->update_render_passes(engine, scene, srl);
+				RE_engine_update_render_passes(engine, scene, srl, cmp_node_rlayer_create_outputs_cb);
 				RE_engine_free(engine);
 
 				MEM_freeN(data);

source/blender/render/extern/include/RE_engine.h

@@ -37,6 +37,8 @@
 #include "RNA_types.h"
 #include "RE_bake.h"
 
+#include "BLI_threads.h"
+
 struct bNode;
 struct bNodeTree;
 struct Object;
@@ -102,6 +104,9 @@ typedef struct RenderEngineType {
 	ExtensionRNA ext;
 } RenderEngineType;
 
+typedef void (*update_render_passes_cb_t)(struct RenderEngine *engine, struct Scene *scene, struct SceneRenderLayer *srl,
+                                          const char *name, int channels, const char *chanid, int type);
+
 typedef struct RenderEngine {
 	RenderEngineType *type;
 	void *py_instance;
@@ -125,6 +130,10 @@ typedef struct RenderEngine {
 	int update_flag;
 	int job_update_flag;
 
+	/* callback for render pass query */
+	ThreadMutex update_render_passes_mutex;
+	update_render_passes_cb_t update_render_passes_cb;
+
 	rctf last_viewplane;
 	rcti last_disprect;
 	float last_viewmat[4][4];
@@ -163,6 +172,8 @@ bool RE_engine_is_external(struct Render *re);
 
 void RE_engine_frame_set(struct RenderEngine *engine, int frame, float subframe);
 
+void RE_engine_update_render_passes(struct RenderEngine *engine, struct Scene *scene, struct SceneRenderLayer *srl,
+                                    update_render_passes_cb_t callback);
 void RE_engine_register_pass(struct RenderEngine *engine, struct Scene *scene, struct SceneRenderLayer *srl,
                              const char *name, int channels, const char *chanid, int type);
 

source/blender/render/intern/include/render_result.h

@@ -44,6 +44,7 @@ struct ImBuf;
 struct ListBase;
 struct Render;
 struct RenderData;
+struct RenderEngine;
 struct RenderLayer;
 struct RenderResult;
 struct Scene;
@@ -85,8 +86,8 @@ void render_result_single_layer_end(struct Render *re);
 /* EXR Tile File Render */
 
 void render_result_save_empty_result_tiles(struct Render *re);
-void render_result_exr_file_begin(struct Render *re);
-void render_result_exr_file_end(struct Render *re);
+void render_result_exr_file_begin(struct Render *re, struct RenderEngine *engine);
+void render_result_exr_file_end(struct Render *re, struct RenderEngine *engine);
 
 /* render pass wrapper for gpencil */
 struct RenderPass *gp_add_pass(struct RenderResult *rr, struct RenderLayer *rl, int channels, const char *name, const char *viewname);
@@ -94,7 +95,7 @@ struct RenderPass *gp_add_pass(struct RenderResult *rr, struct RenderLayer *rl,
 void render_result_exr_file_merge(struct RenderResult *rr, struct RenderResult *rrpart, const char *viewname);
 
 void render_result_exr_file_path(struct Scene *scene, const char *layname, int sample, char *filepath);
-int render_result_exr_file_read_sample(struct Render *re, int sample);
+int render_result_exr_file_read_sample(struct Render *re, int sample, struct RenderEngine *engine);
 int render_result_exr_file_read_path(struct RenderResult *rr, struct RenderLayer *rl_single, const char *filepath);
 
 /* EXR cache */

source/blender/render/intern/source/envmap.c

@@ -500,7 +500,7 @@ static void render_envmap(Render *re, EnvMap *env)
 
 			if (envre->result->do_exr_tile) {
 				BLI_rw_mutex_lock(&envre->resultmutex, THREAD_LOCK_WRITE);
-				render_result_exr_file_end(envre);
+				render_result_exr_file_end(envre, NULL);
 				BLI_rw_mutex_unlock(&envre->resultmutex);
 			}
 

source/blender/render/intern/source/external_engine.c

@@ -146,6 +146,8 @@ RenderEngine *RE_engine_create_ex(RenderEngineType *type, bool use_for_viewport)
 		BLI_threaded_malloc_begin();
 	}
 
+	BLI_mutex_init(&engine->update_render_passes_mutex);
+
 	return engine;
 }
 
@@ -161,6 +163,8 @@ void RE_engine_free(RenderEngine *engine)
 		BLI_threaded_malloc_end();
 	}
 
+	BLI_mutex_end(&engine->update_render_passes_mutex);
+
 	MEM_freeN(engine);
 }
 
@@ -724,7 +728,7 @@ int RE_engine_render(Render *re, int do_all)
 	engine->tile_y = re->party;
 
 	if (re->result->do_exr_tile)
-		render_result_exr_file_begin(re);
+		render_result_exr_file_begin(re, engine);
 
 	if (type->update)
 		type->update(engine, re->main, re->scene);
@@ -745,19 +749,19 @@ int RE_engine_render(Render *re, int do_all)
 
 	BLI_rw_mutex_lock(&re->partsmutex, THREAD_LOCK_WRITE);
 
+	if (re->result->do_exr_tile) {
+		BLI_rw_mutex_lock(&re->resultmutex, THREAD_LOCK_WRITE);
+		render_result_save_empty_result_tiles(re);
+		render_result_exr_file_end(re, engine);
+		BLI_rw_mutex_unlock(&re->resultmutex);
+	}
+
 	/* re->engine becomes zero if user changed active render engine during render */
 	if (!persistent_data || !re->engine) {
 		RE_engine_free(engine);
 		re->engine = NULL;
 	}
 
-	if (re->result->do_exr_tile) {
-		BLI_rw_mutex_lock(&re->resultmutex, THREAD_LOCK_WRITE);
-		render_result_save_empty_result_tiles(re);
-		render_result_exr_file_end(re);
-		BLI_rw_mutex_unlock(&re->resultmutex);
-	}
-
 	if (re->r.scemode & R_EXR_CACHE_FILE) {
 		BLI_rw_mutex_lock(&re->resultmutex, THREAD_LOCK_WRITE);
 		render_result_exr_file_cache_write(re);
@@ -778,23 +782,27 @@ int RE_engine_render(Render *re, int do_all)
 	return 1;
 }
 
-void RE_engine_register_pass(struct RenderEngine *engine, struct Scene *scene, struct SceneRenderLayer *srl,
-                             const char *name, int UNUSED(channels), const char *UNUSED(chanid), int type)
+void RE_engine_update_render_passes(struct RenderEngine *engine, struct Scene *scene, struct SceneRenderLayer *srl,
+                                    update_render_passes_cb_t callback)
 {
-	/* The channel information is currently not used, but is part of the API in case it's needed in the future. */
-
-	if (!(scene && srl && engine)) {
+	if (!(scene && srl && engine && callback && engine->type->update_render_passes)) {
 		return;
 	}
 
-	/* Register the pass in all scenes that have a render layer node for this layer.
-	 * Since multiple scenes can be used in the compositor, the code must loop over all scenes
-	 * and check whether their nodetree has a node that needs to be updated. */
-	/* NOTE: using G_MAIN seems valid here,
-	 * unless we want to register that for every other temp Main we could generate??? */
-	for (Scene *sce = G_MAIN->scene.first; sce; sce = sce->id.next) {
-		if (sce->nodetree) {
-			ntreeCompositRegisterPass(sce->nodetree, scene, srl, name, type);
-		}
-	}
+	BLI_mutex_lock(&engine->update_render_passes_mutex);
+
+	engine->update_render_passes_cb = callback;
+	engine->type->update_render_passes(engine, scene, srl);
+
+	BLI_mutex_unlock(&engine->update_render_passes_mutex);
+}
+
+void RE_engine_register_pass(struct RenderEngine *engine, struct Scene *scene, struct SceneRenderLayer *srl,
+                             const char *name, int channels, const char *chanid, int type)
+{
+	if (!(scene && srl && engine && engine->update_render_passes_cb)) {
+		return;
+	}
+
+	engine->update_render_passes_cb(engine, scene, srl, name, channels, chanid, type);
 }

source/blender/render/intern/source/pipeline.c

@@ -1350,7 +1350,7 @@ static void main_render_result_end(Render *re)
 {
 	if (re->result->do_exr_tile) {
 		BLI_rw_mutex_lock(&re->resultmutex, THREAD_LOCK_WRITE);
-		render_result_exr_file_end(re);
+		render_result_exr_file_end(re, NULL);
 		BLI_rw_mutex_unlock(&re->resultmutex);
 	}
 
@@ -1382,7 +1382,7 @@ static void main_render_result_new(Render *re)
 
 	if (re->result) {
 		if (re->result->do_exr_tile) {
-			render_result_exr_file_begin(re);
+			render_result_exr_file_begin(re, NULL);
 		}
 	}
 }
@@ -2347,7 +2347,7 @@ static void composite_freestyle_renders(Render *re, int sample)
 
 				/* may be NULL in case of empty render layer */
 				if (freestyle_render) {
-					render_result_exr_file_read_sample(freestyle_render, sample);
+					render_result_exr_file_read_sample(freestyle_render, sample, NULL);
 					FRS_composite_result(re, srl, freestyle_render);
 					RE_FreeRenderResult(freestyle_render->result);
 					freestyle_render->result = NULL;
@@ -2446,7 +2446,7 @@ static void do_merge_fullsample(Render *re, bNodeTree *ntree)
 				if (re1 && (re1->r.scemode & R_FULL_SAMPLE)) {
 					if (sample) {
 						BLI_rw_mutex_lock(&re->resultmutex, THREAD_LOCK_WRITE);
-						render_result_exr_file_read_sample(re1, sample);
+						render_result_exr_file_read_sample(re1, sample, NULL);
 #ifdef WITH_FREESTYLE
 						if (re1->r.mode & R_EDGE_FRS)
 							composite_freestyle_renders(re1, sample);

source/blender/render/intern/source/render_result.c

@@ -58,6 +58,8 @@
 
 #include "intern/openexr/openexr_multi.h"
 
+#include "RE_engine.h"
+
 #include "render_result.h"
 #include "render_types.h"
 
@@ -1114,8 +1116,25 @@ void render_result_save_empty_result_tiles(Render *re)
 	}
 }
 
+static void render_result_register_pass_cb(RenderEngine *engine, Scene *UNUSED(scene), SceneRenderLayer *srl,
+                                           const char *name, int channels, const char *chanid, int UNUSED(type))
+{
+	RE_engine_add_pass(engine, name, channels, chanid, srl->name);
+}
+
+static void render_result_create_all_passes(RenderEngine *engine, Render *re, RenderLayer *rl)
+{
+	if (engine && engine->type->update_render_passes) {
+		SceneRenderLayer *srl;
+		srl = BLI_findstring(&re->r.layers, rl->name, offsetof(SceneRenderLayer, name));
+		if (srl) {
+			RE_engine_update_render_passes(engine, re->scene, srl, render_result_register_pass_cb);
+		}
+	}
+}
+
 /* begin write of exr tile file */
-void render_result_exr_file_begin(Render *re)
+void render_result_exr_file_begin(Render *re, RenderEngine *engine)
 {
 	RenderResult *rr;
 	RenderLayer *rl;
@@ -1123,6 +1142,8 @@ void render_result_exr_file_begin(Render *re)
 
 	for (rr = re->result; rr; rr = rr->next) {
 		for (rl = rr->layers.first; rl; rl = rl->next) {
+			render_result_create_all_passes(engine, re, rl);
+
 			render_result_exr_file_path(re->scene, rl->name, rr->sample_nr, str);
 			printf("write exr tmp file, %dx%d, %s\n", rr->rectx, rr->recty, str);
 			IMB_exrtile_begin_write(rl->exrhandle, str, 0, rr->rectx, rr->recty, re->partx, re->party);
@@ -1131,7 +1152,7 @@ void render_result_exr_file_begin(Render *re)
 }
 
 /* end write of exr tile file, read back first sample */
-void render_result_exr_file_end(Render *re)
+void render_result_exr_file_end(Render *re, RenderEngine *engine)
 {
 	RenderResult *rr;
 	RenderLayer *rl;
@@ -1148,7 +1169,7 @@ void render_result_exr_file_end(Render *re)
 	render_result_free_list(&re->fullresult, re->result);
 	re->result = NULL;
 
-	render_result_exr_file_read_sample(re, 0);
+	render_result_exr_file_read_sample(re, 0, engine);
 }
 
 /* save part into exr file */
@@ -1178,7 +1199,7 @@ void render_result_exr_file_path(Scene *scene, const char *layname, int sample,
 }
 
 /* only for temp buffer, makes exact copy of render result */
-int render_result_exr_file_read_sample(Render *re, int sample)
+int render_result_exr_file_read_sample(Render *re, int sample, RenderEngine *engine)
 {
 	RenderLayer *rl;
 	char str[FILE_MAXFILE + MAX_ID_NAME + MAX_ID_NAME + 100] = "";
@@ -1188,6 +1209,8 @@ int render_result_exr_file_read_sample(Render *re, int sample)
 	re->result = render_result_new(re, &re->disprect, 0, RR_USE_MEM, RR_ALL_LAYERS, RR_ALL_VIEWS);
 
 	for (rl = re->result->layers.first; rl; rl = rl->next) {
+		render_result_create_all_passes(engine, re, rl);
+
 		render_result_exr_file_path(re->scene, rl->name, sample, str);
 		printf("read exr tmp file: %s\n", str);