VTK

I added two renderers in renderWindow, and set a blue background for the top renderer.

But the color is not visible, here is source code.

#include <iostream>
#include <vtkSmartPointer.h>
#include <vtkSphereSource.h>
#include <vtkActor.h>
#include <vtkConeSource.h>
#include <vtkRenderer.h>
#include <vtkRenderWindow.h>
#include <vtkPolyDataMapper.h>
#include <vtkRenderWindowInteractor.h>

#define vtkSPtr vtkSmartPointer
#define vtkSPtrNew(Var, Type) vtkSPtr<Type> Var = vtkSPtr<Type>::New();

using namespace std;

int main()
{
    vtkSPtrNew( cone, vtkConeSource );
    vtkSPtrNew( mapper, vtkPolyDataMapper );
    mapper->SetInputConnection( cone->GetOutputPort() );

    vtkSPtrNew( actor, vtkActor );
    actor->SetMapper( mapper );

    vtkSPtrNew( renderer, vtkRenderer );
    renderer->AddActor(actor);
    renderer->SetLayer( 0 );
    renderer->SetBackground( 0, 0, 0 );

    vtkSPtrNew( renderer2, vtkRenderer );
    renderer2->SetViewport( 0, 0, 0.5, 1 );
    renderer2->SetLayer( 1 );
    renderer2->SetBackground( 0, 0, 1 );

    vtkSPtrNew( renderWindow, vtkRenderWindow );
    renderWindow->AddRenderer( renderer );
    renderWindow->AddRenderer( renderer2 );
    renderWindow->SetNumberOfLayers( 2 );

    vtkSPtrNew( renderWindowInteractor, vtkRenderWindowInteractor );
    renderWindowInteractor->SetRenderWindow( renderWindow );

    renderer->ResetCamera();
    renderWindow->Render();
    renderWindowInteractor->Start();
    return 0;
}

We can find the description for the usage of the interface SetPreserveColorBuffer.

   /**
   * Set/Get the layer that this renderer belongs to.  This is only used if
   * there are layered renderers.

   * Note: Changing the layer will update the PreserveColorBuffer setting. If
   * the layer is 0, PreserveColorBuffer will be set to false, making the
   * bottom renderer opaque. If the layer is non-zero, PreserveColorBuffer will
   * be set to true, giving the renderer a transparent background. If other
   * PreserveColorBuffer configurations are desired, they must be adjusted after
   * the layer is set.
   */
  virtual void SetLayer(int layer);
 
   /**
   * By default, the renderer at layer 0 is opaque, and all non-zero layer
   * renderers are transparent. This flag allows this behavior to be overridden.
   * If true, this setting will force the renderer to preserve the existing
   * color buffer regardless of layer. If false, it will always be cleared at
   * the start of rendering.

   * This flag influences the Transparent() method, and is updated by calls to
   * SetLayer(). For this reason it should only be set after changing the layer.
   */
  vtkGetMacro(PreserveColorBuffer, vtkTypeBool);
  vtkSetMacro(PreserveColorBuffer, vtkTypeBool);
  vtkBooleanMacro(PreserveColorBuffer, vtkTypeBool);

Add the following code for renderer2.

    renderer2->SetBackgroundAlpha( 1.0 );
    std::cout << renderer2->GetPreserveColorBuffer() << std::endl; // 1
    renderer2->SetPreserveColorBuffer(false);

We can see the blue background of the top renderer now.

if layer is not 0, PreserveColorBuffer is true. The value of PreserveColorBuffer become false if the layer of renderer is 0.

    vtkSPtrNew( renderer, vtkRenderer );
    renderer->AddActor(actor);
    renderer->SetViewport( 0.5, 0, 1, 1 );
    renderer->SetLayer( 1 );
    renderer->SetBackground( 0, 1, 0 );
    renderer->SetBackgroundAlpha( 1.0 );
    // if layer is not 0, PreserveColorBuffer is true
    std::cout << renderer->GetPreserveColorBuffer() << std::endl; // 1
    renderer->SetPreserveColorBuffer(false);

    vtkSPtrNew( renderer2, vtkRenderer );
    renderer2->SetViewport( 0, 0, 0.5, 1 );
    renderer2->SetLayer( 0 );
    renderer2->SetBackground( 0, 0, 1 );
    renderer2->SetBackgroundAlpha( 1.0 );
    // if layer is 0, PreserveColorBuffer is false
    std::cout << renderer2->GetPreserveColorBuffer() << std::endl; // 0
    renderer2->SetPreserveColorBuffer(false);
    renderer2->AddActor(actor);

    vtkSPtrNew( renderWindow, vtkRenderWindow );
    renderWindow->AddRenderer( renderer );
    renderWindow->AddRenderer( renderer2 );
    renderWindow->SetNumberOfLayers( 2 );

Resut: