Fix T40986: crash on using the viewer node inside of group nodes.

Viewers were activated both inside the active group as well as the top
level tree (the latter being a quick fix for getting a fallback viewer).
This caused a race condition on the shared viewer image.

Now the active viewer is defined at node conversion time in the converter
so that only one can be active at a time without each node having to
follow complicated rules for exclusion.

source/blender/compositor/intern/COM_NodeConverter.cpp

@@ -156,3 +156,13 @@ void NodeConverter::addOutputVector(NodeOutput *output, const float value[3])
 	m_builder->addOperation(operation);
 	m_builder->mapOutputSocket(output, operation->getOutputSocket());
 }
+
+void NodeConverter::registerViewer(ViewerOperation *viewer)
+{
+	m_builder->registerViewer(viewer);
+}
+
+ViewerOperation *NodeConverter::active_viewer() const
+{
+	return m_builder->active_viewer();
+}

source/blender/compositor/intern/COM_NodeConverter.h

@@ -34,6 +34,8 @@ class NodeOperationInput;
 class NodeOperationOutput;
 class NodeOperationBuilder;
 
+class ViewerOperation;
+
 /** Interface type for converting a \a Node into \a NodeOperation.
  *  This is passed to \a Node::convertToOperation methods and allows them
  *  to register any number of operations, create links between them,
@@ -102,6 +104,11 @@ public:
 	 */
 	NodeOperation *setInvalidOutput(NodeOutput *output);
 	
+	/** Define a viewer operation as the active output, if possible */
+	void registerViewer(ViewerOperation *viewer);
+	/** The currently active viewer output operation */
+	ViewerOperation *active_viewer() const;
+	
 private:
 	/** The internal builder for storing the results of the graph construction. */
 	NodeOperationBuilder *m_builder;

source/blender/compositor/intern/COM_NodeGraph.cpp

@@ -101,8 +101,7 @@ void NodeGraph::add_bNodeTree(const CompositorContext &context, int nodes_start,
 	const bNodeTree *basetree = context.getbNodeTree();
 	
 	/* update viewers in the active edittree as well the base tree (for backdrop) */
-	bool is_active_group = ((parent_key.value == basetree->active_viewer_key.value) ||
-	                        (tree == basetree));
+	bool is_active_group = (parent_key.value == basetree->active_viewer_key.value);
 	
 	/* add all nodes of the tree to the node list */
 	for (bNode *node = (bNode *)tree->nodes.first; node; node = node->next) {

source/blender/compositor/intern/COM_NodeOperationBuilder.cpp

@@ -38,12 +38,14 @@ extern "C" {
 #include "COM_SocketProxyOperation.h"
 #include "COM_ReadBufferOperation.h"
 #include "COM_WriteBufferOperation.h"
+#include "COM_ViewerOperation.h"
 
 #include "COM_NodeOperationBuilder.h" /* own include */
 
 NodeOperationBuilder::NodeOperationBuilder(const CompositorContext *context, bNodeTree *b_nodetree) :
     m_context(context),
-    m_current_node(NULL)
+    m_current_node(NULL),
+    m_active_viewer(NULL)
 {
 	m_graph.from_bNodeTree(*context, b_nodetree);
 }
@@ -239,6 +241,25 @@ void NodeOperationBuilder::addNodeInputPreview(NodeInput *input)
 	}
 }
 
+void NodeOperationBuilder::registerViewer(ViewerOperation *viewer)
+{
+	if (m_active_viewer) {
+		if (m_current_node->isInActiveGroup()) {
+			/* deactivate previous viewer */
+			m_active_viewer->setActive(false);
+			
+			m_active_viewer = viewer;
+			viewer->setActive(true);
+		}
+	}
+	else {
+		if (m_current_node->getbNodeTree() == m_context->getbNodeTree()) {
+			m_active_viewer = viewer;
+			viewer->setActive(true);
+		}
+	}
+}
+
 /****************************
  **** Optimization Steps ****
  ****************************/

source/blender/compositor/intern/COM_NodeOperationBuilder.h

@@ -44,6 +44,7 @@ class NodeOperationOutput;
 
 class PreviewOperation;
 class WriteBufferOperation;
+class ViewerOperation;
 
 class NodeOperationBuilder {
 public:
@@ -87,6 +88,12 @@ private:
 	
 	Node *m_current_node;
 	
+	/** Operation that will be writing to the viewer image
+	 *  Only one operation can occupy this place at a time,
+	 *  to avoid race conditions
+	 */
+	ViewerOperation *m_active_viewer;
+	
 public:
 	NodeOperationBuilder(const CompositorContext *context, bNodeTree *b_nodetree);
 	~NodeOperationBuilder();
@@ -110,6 +117,11 @@ public:
 	/** Add a preview operation for a node input */
 	void addNodeInputPreview(NodeInput *input);
 	
+	/** Define a viewer operation as the active output, if possible */
+	void registerViewer(ViewerOperation *viewer);
+	/** The currently active viewer output operation */
+	ViewerOperation *active_viewer() const { return m_active_viewer; }
+	
 protected:
 	static NodeInput *find_node_input(const InputSocketMap &map, NodeOperationInput *op_input);
 	static const OpInputs &find_operation_inputs(const OpInputInverseMap &map, NodeInput *node_input);

source/blender/compositor/nodes/COM_SplitViewerNode.cpp

@@ -35,8 +35,7 @@ SplitViewerNode::SplitViewerNode(bNode *editorNode) : Node(editorNode)
 void SplitViewerNode::convertToOperations(NodeConverter &converter, const CompositorContext &context) const
 {
 	bNode *editorNode = this->getbNode();
-	bool is_active = ((editorNode->flag & NODE_DO_OUTPUT_RECALC || context.isRendering()) &&
-	                  (editorNode->flag & NODE_DO_OUTPUT) && this->isInActiveGroup());
+	bool do_output = (editorNode->flag & NODE_DO_OUTPUT_RECALC || context.isRendering()) && (editorNode->flag & NODE_DO_OUTPUT);
 
 	NodeInput *image1Socket = this->getInputSocket(0);
 	NodeInput *image2Socket = this->getInputSocket(1);
@@ -54,7 +53,6 @@ void SplitViewerNode::convertToOperations(NodeConverter &converter, const Compos
 	ViewerOperation *viewerOperation = new ViewerOperation();
 	viewerOperation->setImage(image);
 	viewerOperation->setImageUser(imageUser);
-	viewerOperation->setActive(is_active);
 	viewerOperation->setViewSettings(context.getViewSettings());
 	viewerOperation->setDisplaySettings(context.getDisplaySettings());
 
@@ -68,4 +66,7 @@ void SplitViewerNode::convertToOperations(NodeConverter &converter, const Compos
 	converter.addLink(splitViewerOperation->getOutputSocket(), viewerOperation->getInputSocket(0));
 
 	converter.addPreview(splitViewerOperation->getOutputSocket());
+
+	if (do_output)
+		converter.registerViewer(viewerOperation);
 }

source/blender/compositor/nodes/COM_ViewerNode.cpp

@@ -34,8 +34,7 @@ ViewerNode::ViewerNode(bNode *editorNode) : Node(editorNode)
 void ViewerNode::convertToOperations(NodeConverter &converter, const CompositorContext &context) const
 {
 	bNode *editorNode = this->getbNode();
-	bool is_active = (editorNode->flag & NODE_DO_OUTPUT_RECALC || context.isRendering()) &&
-	                 ((editorNode->flag & NODE_DO_OUTPUT) && this->isInActiveGroup());
+	bool do_output = (editorNode->flag & NODE_DO_OUTPUT_RECALC || context.isRendering()) && (editorNode->flag & NODE_DO_OUTPUT);
 	bool ignore_alpha = editorNode->custom2 & CMP_NODE_OUTPUT_IGNORE_ALPHA;
 
 	NodeInput *imageSocket = this->getInputSocket(0);
@@ -47,7 +46,6 @@ void ViewerNode::convertToOperations(NodeConverter &converter, const CompositorC
 	viewerOperation->setbNodeTree(context.getbNodeTree());
 	viewerOperation->setImage(image);
 	viewerOperation->setImageUser(imageUser);
-	viewerOperation->setActive(is_active);
 	viewerOperation->setChunkOrder((OrderOfChunks)editorNode->custom1);
 	viewerOperation->setCenterX(editorNode->custom3);
 	viewerOperation->setCenterY(editorNode->custom4);
@@ -74,4 +72,7 @@ void ViewerNode::convertToOperations(NodeConverter &converter, const CompositorC
 	converter.mapInputSocket(depthSocket, viewerOperation->getInputSocket(2));
 
 	converter.addNodeInputPreview(imageSocket);
+
+	if (do_output)
+		converter.registerViewer(viewerOperation);
 }