Qt Quick 3D - Stencil Outline Extension Example

 // Copyright (C) 2023 The Qt Company Ltd.
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR BSD-3-Clause

 #include "outlinerenderextension.h"

 #include <rhi/qrhi.h>

 #include <QtQuick3D/qquick3dobject.h>
 #include <ssg/qquick3dextensionhelpers.h>

 #include <ssg/qssgrenderhelpers.h>
 #include <ssg/qssgrenderextensions.h>
 #include <ssg/qssgrendercontextcore.h>

 class OutlineRenderer : public QSSGRenderExtension
 {
 public:
     OutlineRenderer() = default;

     bool prepareData(QSSGFrameData &data) override;
     void prepareRender(QSSGFrameData &data) override;
     void render(QSSGFrameData &data) override;
     void resetForFrame() override;
     RenderMode mode() const override { return RenderMode::Main; }
     RenderStage stage() const override { return RenderStage::PostColor; };

     QSSGPrepContextId stencilPrepContext { QSSGPrepContextId::Invalid };
     QSSGPrepContextId outlinePrepContext { QSSGPrepContextId::Invalid };
     QSSGPrepResultId stencilPrepResult { QSSGPrepResultId::Invalid };
     QSSGPrepResultId outlinePrepResult { QSSGPrepResultId::Invalid };
     QPointer<QQuick3DObject> model;
     QSSGNodeId modelId { QSSGNodeId::Invalid };
     QPointer<QQuick3DObject> material;
     QSSGResourceId outlineMaterialId {};
     float outlineScale = 1.05f;

     QSSGRenderablesId stencilRenderables;
     QSSGRenderablesId outlineRenderables;
 };

 bool OutlineRenderer::prepareData(QSSGFrameData &data)
 {
     // Make sure we have a model and a material.
     if (!model || !material)
         return false;

     modelId = QQuick3DExtensionHelpers::getNodeId(*model);
     if (modelId == QSSGNodeId::Invalid)
         return false;

     outlineMaterialId = QQuick3DExtensionHelpers::getResourceId(*material);
     if (outlineMaterialId == QSSGResourceId::Invalid)
         return false;

     // This is the active camera for the scene (the camera used to render the QtQuick3D scene)
     QSSGCameraId camera = data.activeCamera();
     if (camera == QSSGCameraId::Invalid)
         return false;

     // We are going to render the same renderable(s) twice so we need to create two contexts.
     stencilPrepContext = QSSGRenderHelpers::prepareForRender(data, *this, camera, 0);
     outlinePrepContext = QSSGRenderHelpers::prepareForRender(data, *this, camera, 1);
     // Create the renderables for the target model. One for the original with stencil write, and one for the outline model.
     // Note that we 'Steal' the model here, that tells QtQuick3D that we'll take over the rendering of the model.
     stencilRenderables = QSSGRenderHelpers::createRenderables(data, stencilPrepContext, { modelId }, QSSGRenderHelpers::CreateFlag::Steal);
     outlineRenderables = QSSGRenderHelpers::createRenderables(data, outlinePrepContext, { modelId });

     // Now we can start setting the data for our models.
     // Here we set a material and a scale for the outline
     QSSGModelHelpers::setModelMaterials(data, outlineRenderables, modelId, { outlineMaterialId });
     QMatrix4x4 globalTransform = QSSGModelHelpers::getGlobalTransform(data, modelId);
     globalTransform.scale(outlineScale);
     QSSGModelHelpers::setGlobalTransform(data, outlineRenderables, modelId, globalTransform);

     // When all changes are done, we need to commit the changes.
     stencilPrepResult = QSSGRenderHelpers::commit(data, stencilPrepContext, stencilRenderables);
     outlinePrepResult = QSSGRenderHelpers::commit(data, outlinePrepContext, outlineRenderables);

     // If there's something to be rendered we return true.
     const bool dataReady = (stencilPrepResult != QSSGPrepResultId::Invalid && outlinePrepResult != QSSGPrepResultId::Invalid);

     return dataReady;
 }

 void OutlineRenderer::prepareRender(QSSGFrameData &data)
 {
     Q_ASSERT(modelId != QSSGNodeId::Invalid);
     Q_ASSERT(stencilPrepResult != QSSGPrepResultId::Invalid && outlinePrepResult != QSSGPrepResultId::Invalid);

     const auto &ctx = data.contextInterface();

     if (const auto &rhiCtx = ctx->rhiContext()) {
         const QSSGRhiGraphicsPipelineState basePs = data.getPipelineState();
         QRhiRenderPassDescriptor *rpDesc = rhiCtx->mainRenderPassDescriptor();
         const int samples = rhiCtx->mainPassSampleCount();

         { // Original model - Write to the stencil buffer.
             QSSGRhiGraphicsPipelineState ps = basePs;
             ps.flags |= { QSSGRhiGraphicsPipelineState::Flag::BlendEnabled,
                           QSSGRhiGraphicsPipelineState::Flag::DepthWriteEnabled,
                           QSSGRhiGraphicsPipelineState::Flag::UsesStencilRef,
                           QSSGRhiGraphicsPipelineState::Flag::DepthTestEnabled };
             ps.stencilWriteMask = 0xff;
             ps.stencilRef = 1;
             ps.samples = samples;
             ps.cullMode = QRhiGraphicsPipeline::Back;

             ps.stencilOpFrontState = { QRhiGraphicsPipeline::Keep,
                                        QRhiGraphicsPipeline::Keep,
                                        QRhiGraphicsPipeline::Replace,
                                        QRhiGraphicsPipeline::Always };

             QSSGRenderHelpers::prepareRenderables(data, stencilPrepResult, rpDesc, ps);
         }

         { // Scaled version - Only draw outside the original.
             QSSGRhiGraphicsPipelineState ps = basePs;
             ps.flags |= { QSSGRhiGraphicsPipelineState::Flag::BlendEnabled,
                           QSSGRhiGraphicsPipelineState::Flag::UsesStencilRef,
                           QSSGRhiGraphicsPipelineState::Flag::DepthTestEnabled };
             ps.flags.setFlag(QSSGRhiGraphicsPipelineState::Flag::DepthWriteEnabled, false);
             ps.stencilWriteMask = 0;
             ps.stencilRef = 1;
             ps.cullMode = QRhiGraphicsPipeline::Back;

             ps.stencilOpFrontState = { QRhiGraphicsPipeline::Keep,
                                        QRhiGraphicsPipeline::Keep,
                                        QRhiGraphicsPipeline::Replace,
                                        QRhiGraphicsPipeline::NotEqual };

             QSSGRenderHelpers::prepareRenderables(data, outlinePrepResult, rpDesc, ps);
         }
     }
 }

 void OutlineRenderer::render(QSSGFrameData &data)
 {
     Q_ASSERT(stencilPrepResult != QSSGPrepResultId::Invalid);

     const auto &ctx = data.contextInterface();
     if (const auto &rhiCtx = ctx->rhiContext()) {
         QRhiCommandBuffer *cb = rhiCtx->commandBuffer();
         cb->debugMarkBegin(QByteArrayLiteral("Stencil outline pass"));
         QSSGRenderHelpers::renderRenderables(data, stencilPrepResult);
         QSSGRenderHelpers::renderRenderables(data, outlinePrepResult);
         cb->debugMarkEnd();
     }
 }

 void OutlineRenderer::resetForFrame()
 {
     stencilPrepContext = { QSSGPrepContextId::Invalid };
     stencilPrepResult = { QSSGPrepResultId::Invalid };
 }

 OutlineRenderExtension::~OutlineRenderExtension() {}

 float OutlineRenderExtension::outlineScale() const
 {
     return m_outlineScale;
 }

 void OutlineRenderExtension::setOutlineScale(float newOutlineScale)
 {
     if (qFuzzyCompare(m_outlineScale, newOutlineScale))
         return;
     m_outlineScale = newOutlineScale;

     markDirty(Dirty::OutlineScale);

     emit outlineScaleChanged();
 }

 QQuick3DObject *OutlineRenderExtension::target() const
 {
     return m_target;
 }

 void OutlineRenderExtension::setTarget(QQuick3DObject *newTarget)
 {
     if (m_target == newTarget)
         return;
     m_target = newTarget;

     markDirty(Dirty::Target);

     emit targetChanged();
 }

 QSSGRenderGraphObject *OutlineRenderExtension::updateSpatialNode(QSSGRenderGraphObject *node)
 {
     if (!node)
         node = new OutlineRenderer;

     OutlineRenderer *renderer = static_cast<OutlineRenderer *>(node);
     renderer->outlineScale = m_outlineScale;
     renderer->model = m_target;
     renderer->material = m_outlineMaterial;

     m_dirtyFlag = {};

     return node;
 }

 void OutlineRenderExtension::markDirty(Dirty v)
 {
     m_dirtyFlag |= v;
     update();
 }

 QQuick3DObject *OutlineRenderExtension::outlineMaterial() const
 {
     return m_outlineMaterial;
 }

 void OutlineRenderExtension::setOutlineMaterial(QQuick3DObject *newOutlineMaterial)
 {
     if (m_outlineMaterial == newOutlineMaterial)
         return;

     m_outlineMaterial = newOutlineMaterial;

     markDirty(Dirty::OutlineMaterial);
     emit outlineMaterialChanged();
 }