How to integrate QML UI into a custom Vulkan renderer without using a separate window

sashakddr

I'm developing a custom Vulkan renderer and want to integrate a QML-based UI into it.

I already have working Vulkan setup and also managed to render QML over Vulkan using a separate QQuickWindow and QQuickRenderControl, but this approach isn't ideal - the QML elements live in a distinct window, so they have their own focus and input handling. I can use QCoreApplication::sendEvent to send events, this works for buttons, but doesnt work for TextField, because for text field underlying window should be activated, but as I said currently my UI elements have distinct QQuickWindow and I dont want to activate them, so that my QWindow I use to render Vulkan onto lose focus.

What I want instead is for QML elements to share focus and input events with my in-game window (i.e. no separate OS-level window, a single rendering surface that draws both game content and QML UI).

Is there a way to embed QML rendering directly into an existing Vulkan surface or swapchain image, so that both systems share focus and input?

Questions:

What’s the best way to render QML directly into an existing Vulkan context or image?
How can I make QML and my in-game input system share the same focus / event handling?
Should I use QQuickRenderControl, or is there a more appropriate approach for Vulkan integration?

Any examples or recommended architecture would be greatly appreciated.

sashakddr

My current setup:

UserInterfaceElement.h

struct UserInterfaceElement
{
    std::unique_ptr<UserInterfaceRenderer> renderer;
    std::unique_ptr<UiCache> cache;
    Texture texture;
    Model model;
    QString qmlPath;
};

UserInterfaceRenderer.h

class UserInterfaceRenderer : public QObject
{
    Q_OBJECT
public:
    UserInterfaceRenderer(QWindow* parentWindow = nullptr);
    ~UserInterfaceRenderer();

    void loadQml(const QSize& size, const QString& qmlPath);
    void render();
    void resize(const QSize& size);
    void createFbo(const QSize& size);
    void deleteFbo();
    QImage grabImage(); // taking screenshot
    QOpenGLFramebufferObject* getFbo() const { return fbo; }
    QQuickWindow* getQuickWindow() const { return quickWindow; }

    void forwardEvent(QEvent* event);
    QQuickItem* getRootItem() const { return rootItem; }

private:
    QQuickRenderControl* renderControl = nullptr;
    QQuickWindow* quickWindow = nullptr;
    QQmlEngine* engine = nullptr;
    QQmlComponent* component = nullptr;
    QQuickItem* rootItem = nullptr;

    QOpenGLFramebufferObject* fbo = nullptr;
    QOpenGLContext* context = nullptr;
    QOffscreenSurface* offscreenSurface = nullptr;
    QSize surfaceSize;
};

UserInterfaceRenderer.cpp

UserInterfaceRenderer::UserInterfaceRenderer(QWindow* parentWindow) {
    renderControl = new QQuickRenderControl(this);
    quickWindow = new QQuickWindow(renderControl);
    quickWindow->setGraphicsApi(QSGRendererInterface::OpenGL);
    quickWindow->setFlags(
        Qt::FramelessWindowHint | Qt::Tool | 
        Qt::WindowStaysOnTopHint | Qt::WindowTransparentForInput
    );
    quickWindow->setColor(Qt::transparent);

    context = new QOpenGLContext(parentWindow);
    context->setFormat(parentWindow->format());
    context->create();

    offscreenSurface = new QOffscreenSurface();
    offscreenSurface->setFormat(context->format());
    offscreenSurface->create();

    context->makeCurrent(offscreenSurface);
    quickWindow->setGraphicsDevice(QQuickGraphicsDevice::fromOpenGLContext(context));
    renderControl->initialize();
    quickWindow->create();

    //quickWindow->setTransientParent(parentWindow);
    //quickWindow->setParent(parentWindow);

    engine = new QQmlEngine(this);
    if (!engine->incubationController()) {
        engine->setIncubationController(quickWindow->incubationController());
    }
}

UserInterfaceRenderer::~UserInterfaceRenderer() {
    if (rootItem) {
        rootItem->setParentItem(nullptr);
        delete rootItem;
    }

    delete renderControl;
    delete quickWindow;
    delete engine;
    delete component;
    deleteFbo();

    if (context) {
        context->doneCurrent();
        delete context;
    }

    if (offscreenSurface) {
        offscreenSurface->destroy();
        delete offscreenSurface;
    }
}

void UserInterfaceRenderer::forwardEvent(QEvent* event) {
    QCoreApplication::sendEvent(quickWindow, event);
}

void UserInterfaceRenderer::loadQml(const QSize& size, const QString& qmlPath) {
    component = new QQmlComponent(engine, QUrl::fromLocalFile(qmlPath));

    rootItem = qobject_cast<QQuickItem*>(component->create());
    if (!rootItem) {
        qWarning() << "[UserInterfaceRenderer] Failed to load QML root item." << qmlPath;
        return;
    }

    rootItem->setParentItem(quickWindow->contentItem());
    rootItem->setSize(size);
    quickWindow->resize(size);

    surfaceSize = size;

    deleteFbo();
    createFbo(size);
}

void UserInterfaceRenderer::resize(const QSize& size) {
    if (!rootItem) return;

    rootItem->setSize(size);
    quickWindow->resize(size);
    surfaceSize = size;

    deleteFbo();
    createFbo(size);
}

void UserInterfaceRenderer::createFbo(const QSize& size) {
    context->makeCurrent(offscreenSurface);

    QOpenGLFramebufferObjectFormat format;
    format.setAttachment(QOpenGLFramebufferObject::CombinedDepthStencil);
    format.setTextureTarget(GL_TEXTURE_2D);
    format.setInternalTextureFormat(GL_RGBA8);
    fbo = new QOpenGLFramebufferObject(size, format);

    QQuickRenderTarget renderTarget = QQuickRenderTarget::fromOpenGLTexture(
        fbo->texture(),
        surfaceSize
    );
    quickWindow->setRenderTarget(renderTarget);
}

void UserInterfaceRenderer::deleteFbo()
{
    if (fbo) {
        context->makeCurrent(offscreenSurface);
        delete fbo;
        fbo = nullptr;
    }
}

void UserInterfaceRenderer::render() {
    if (!rootItem || !quickWindow || !fbo) return;

    if (!context->makeCurrent(offscreenSurface)) {
        qWarning() << "Failed to make OpenGL context current!";
        return;
    }

    QOpenGLFunctions* f = context->functions();
    f->glViewport(0, 0, surfaceSize.width(), surfaceSize.height());
    f->glClearColor(0, 0, 0, 0); // transparent clear
    f->glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

    renderControl->beginFrame();
    renderControl->polishItems();
    renderControl->sync();
    renderControl->render();
    renderControl->endFrame();

    f->glFlush();
}

Model.h

struct Texture {
    VkImage                                 image         = VK_NULL_HANDLE;
    VmaAllocation                           vmaAllocation = VK_NULL_HANDLE;
    VkImageView                             imageView     = VK_NULL_HANDLE;
    VkSampler                               sampler       = VK_NULL_HANDLE;
    uint32_t                                mipLevels     = 0;
    uint32_t                                width         = 0;
    uint32_t                                height        = 0;
};

struct Material {
    Texture                                 diffuseTexture; // basic color
    Texture                                 normalTexture;
    Texture                                 specularTexture;
    Texture                                 emissiveTexture;
};

struct Mesh
{
    std::vector<Vertex>                     vertices;
    std::vector<uint32_t>                   indices;
    glm::mat4                               transform              = glm::mat4(1.0f);

    Material                                material;

    VkBuffer                                vertexBuffer           = VK_NULL_HANDLE;
    VmaAllocation                           vertexBufferAllocation = VK_NULL_HANDLE;
    VkBuffer                                indexBuffer            = VK_NULL_HANDLE;
    VmaAllocation                           indexBufferAllocation  = VK_NULL_HANDLE;

    std::vector<VkDescriptorSet>            descriptorSets         = std::vector<VkDescriptorSet>(MAX_FRAMES_IN_FLIGHT, VK_NULL_HANDLE);
};

struct Model {
    std::vector<Mesh>                       meshes;
    ModelType                               type = ModelType::OTHER;

    glm::vec3                               position;
    glm::vec3                               scale;
    glm::quat                               rotation;

    bool                                    isCollidable = false;
};

called in recordCommandBuffer function, which is called each frame for each UI element

void AetherEngine::recordUiElementToCommandBuffer(UserInterfaceElement& uiElement, VkCommandBuffer commandBuffer)
{
    renderQmlToTexture(uiElement.renderer.get(), uiElement.texture);
    vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelines["ui"]);
    recordModelToCommandBuffer(uiElement.model, commandBuffer);
}
void AetherEngine::renderQmlToTexture(UserInterfaceRenderer* renderer, Texture& texture)
{
    renderer->render();
    QImage image = renderer->getFbo()->toImage().convertToFormat(QImage::Format_RGBA8888);

    if (texture.width != static_cast<uint32_t>(image.width()) ||
        texture.height != static_cast<uint32_t>(image.height())) {
        cleanupTexture(texture);

        modelManager.createSolidColorTexture({ 0, 0, 0, 0 }, image.width(), image.height(), texture);
    }

    modelManager.uploadRawDataToTexture(image.bits(), image.width(), image.height(), texture);
}

sashakddr

If you need some clarifications let me know