How to grab mouse from QViewport surface in Qt3D

[ktalik]

Maybe this question is too Qt3D-specific and should be moved to a different forum. If yes, please let me know.

I have following setup:

As you can see, one Qt3DWindow (let's call it "main Qt3DWindow") is embeded inside QDockWidget. The other is standalone Qt3DWindow (called "topWindow", because it renders the view from the top of the scene).

My problem is picking.

I create 20 spheres of following class in loop:

class HoverableEntity(Qt3DCore.QEntity):

    def hovered(self):
        print('Hovered %s' % self)

and assign pickers to them like this:

        for i in range(20):
            sphereEntity = HoverableEntity(self.rootEntity)
            self.spheres.append(sphereEntity)
            # (...)
            picker = Qt3DRender.QObjectPicker(sphereEntity)
            picker.setHoverEnabled(True)
            picker.entered.connect(sphereEntity.hovered)
            sphereEntity.addComponent(picker)

This picking works successfully on the main Qt3DWindow, printing the output like:

Hovered <__main__.HoverableEntity object at 0x10A8C3C8>
Hovered <__main__.HoverableEntity object at 0x10A86FD0>
Hovered <__main__.HoverableEntity object at 0x10A8CC88>
...

However, it doesn't work for external "topWindow", which renders the viewport, connected to the framegraph like this:

        self.topCamera = Qt3DRender.QCamera(self.camera().parent())
        self.topCamera.setPosition(self.camera().position())
        self.topCamera.setPosition(QVector3D(0, 100, 0))
        self.topCamera.setViewCenter(QVector3D(0, 0, 0))

        self.surfaceSelector2 = Qt3DRender.QRenderSurfaceSelector(self.rootNode)
        self.surfaceSelector2.setSurface(self.mainWindow.topWindow)
        self.viewport2 = Qt3DRender.QViewport(self.surfaceSelector2)
        self.viewport2.setNormalizedRect(QRect(0, 0, 1, 1))
        self.cameraSelector2 = Qt3DRender.QCameraSelector(self.viewport2)
        self.cameraSelector2.setCamera(self.topCamera)
        self.clearBuffers2 = Qt3DRender.QClearBuffers(self.cameraSelector2)
        self.clearBuffers2.setClearColor(Qt.white)
        self.clearBuffers2.setBuffers(Qt3DRender.QClearBuffers.ColorDepthBuffer)
        self.renderStateSet2 = Qt3DRender.QRenderStateSet(self.clearBuffers2)
        self.cullFace2 = Qt3DRender.QCullFace(self.renderStateSet2)
        self.depthTest2 = Qt3DRender.QDepthTest()
        self.depthTest2.setDepthFunction(Qt3DRender.QDepthTest.Less)
        self.renderStateSet2.addRenderState(self.depthTest2)

This is the essence of the code. I suppose something could be wrong here.

In Qt3D documentation of QObjectPicker there is a following sentence:

For every combination of viewport and camera, picking casts a ray through the scene to find entities who's bounding volume intersects the ray.

But no description is given how make sure such ray is catsed correctly.

Here is the entire code combined in one executable main:

import random
import sys
import time

from PySide2 import QtWidgets, QtCore, Qt3DExtras
from PySide2.Qt3DCore import Qt3DCore
from PySide2.Qt3DExtras import Qt3DExtras
from PySide2.Qt3DLogic import Qt3DLogic
from PySide2.Qt3DRender import Qt3DRender
from PySide2.QtCore import QPropertyAnimation, QByteArray, Qt, QRect
from PySide2.QtGui import QVector3D, QQuaternion
from PySide2.QtWidgets import QApplication, QDockWidget, QWidget, QLabel
from PySide2.QtWidgets import QMainWindow


class HoverableEntity(Qt3DCore.QEntity):

    def hovered(self):
        print('Hovered %s' % self)


class View3DWidget(Qt3DExtras.Qt3DWindow):

    def __init__(self, mainWindow):
        Qt3DExtras.Qt3DWindow.__init__(self)
        self.mainWindow = mainWindow

        # Camera
        self.camera().lens().setPerspectiveProjection(45, 16 / 9, 0.1, 1000)
        self.camera().setPosition(QVector3D(0, -10, 40))
        self.camera().setViewCenter(QVector3D(0, 0, 0))

        # Qt3D example rewritten from C++
        # https://doc.qt.io/qt-5/qt3d-simple-cpp-example.html

        # Root entity to begin with
        self.rootEntity = Qt3DCore.QEntity()

        # Material
        self.material = Qt3DExtras.QPhongMaterial(self.rootEntity)

        # Torus
        self.torusEntity = Qt3DCore.QEntity(self.rootEntity)
        self.torusMesh = Qt3DExtras.QTorusMesh()
        self.torusMesh.setRadius(5)
        self.torusMesh.setMinorRadius(1)
        self.torusMesh.setRings(100)
        self.torusMesh.setSlices(20)

        self.torusTransform = Qt3DCore.QTransform()
        self.torusTransform.setScale3D(QVector3D(1.5, 1, 0.5))
        self.torusTransform.setRotation(QQuaternion.fromAxisAndAngle(QVector3D(1, 0, 0), 45))

        self.torusEntity.addComponent(self.torusMesh)
        self.torusEntity.addComponent(self.torusTransform)
        self.torusEntity.addComponent(self.material)

        # Spheres
        self.spheres = []
        self.transforms = []
        self.controllers = []
        self.animations = []
        self.sphereMesh = Qt3DExtras.QSphereMesh()
        self.sphereMesh.setRadius(3)

        for i in range(20):
            sphereEntity = HoverableEntity(self.rootEntity)
            self.spheres.append(sphereEntity)

            sphereTransform = Qt3DCore.QTransform()
            self.transforms.append(sphereTransform)
            controller = OrbitTransformController(sphereTransform)
            controller.setTarget(sphereTransform)
            controller.setRadius(20)
            self.controllers.append(controller)

            sphereRotateTransformAnimation = QPropertyAnimation(sphereTransform)
            sphereRotateTransformAnimation.setTargetObject(controller)
            sphereRotateTransformAnimation.setPropertyName(QByteArray(b'angle'))
            sphereRotateTransformAnimation.setStartValue(0)
            sphereRotateTransformAnimation.setEndValue(360)
            sphereRotateTransformAnimation.setDuration(random.randint(5000, 10000))
            sphereRotateTransformAnimation.setLoopCount(-1)
            sphereRotateTransformAnimation.start()
            self.animations.append(sphereRotateTransformAnimation)

            picker = Qt3DRender.QObjectPicker(sphereEntity)
            picker.setHoverEnabled(True)
            picker.entered.connect(sphereEntity.hovered)
            sphereEntity.addComponent(picker)

            sphereEntity.addComponent(self.sphereMesh)
            sphereEntity.addComponent(sphereTransform)
            sphereEntity.addComponent(self.material)

        # For camera controls
        self.camController = Qt3DExtras.QOrbitCameraController(self.rootEntity)
        self.camController.setLinearSpeed(50)
        self.camController.setLookSpeed(180)
        self.camController.setCamera(self.camera())

        self.setRootEntity(self.rootEntity)

        self.rootNode = Qt3DRender.QRenderTargetSelector()
        self.renderSettings2 = Qt3DRender.QRenderSettings(self.rootNode)
        # self.renderSettings().setRenderPolicy(Qt3DRender.QRenderSettings.OnDemand)

        self.surfaceSelector = Qt3DRender.QRenderSurfaceSelector(self.rootNode)
        self.surfaceSelector.setSurface(self)
        self.viewport = Qt3DRender.QViewport(self.surfaceSelector)
        self.viewport.setNormalizedRect(QRect(0, 0, 1, 1))
        self.cameraSelector = Qt3DRender.QCameraSelector(self.viewport)
        self.cameraSelector.setCamera(self.camera())
        self.clearBuffers = Qt3DRender.QClearBuffers(self.cameraSelector)
        self.clearBuffers.setClearColor(Qt.white)
        self.clearBuffers.setBuffers(Qt3DRender.QClearBuffers.ColorDepthBuffer)
        self.renderStateSet = Qt3DRender.QRenderStateSet(self.clearBuffers)
        self.cullFace = Qt3DRender.QCullFace(self.renderStateSet)
        self.depthTest = Qt3DRender.QDepthTest()
        self.depthTest.setDepthFunction(Qt3DRender.QDepthTest.Less)
        self.renderStateSet.addRenderState(self.depthTest)

        self.topCamera = Qt3DRender.QCamera(self.camera().parent())
        self.topCamera.setPosition(self.camera().position())
        self.topCamera.setPosition(QVector3D(0, 100, 0))
        self.topCamera.setViewCenter(QVector3D(0, 0, 0))

        self.surfaceSelector2 = Qt3DRender.QRenderSurfaceSelector(self.rootNode)
        self.surfaceSelector2.setSurface(self.mainWindow.topWindow)
        self.viewport2 = Qt3DRender.QViewport(self.surfaceSelector2)
        self.viewport2.setNormalizedRect(QRect(0, 0, 1, 1))
        self.cameraSelector2 = Qt3DRender.QCameraSelector(self.viewport2)
        self.cameraSelector2.setCamera(self.topCamera)
        self.clearBuffers2 = Qt3DRender.QClearBuffers(self.cameraSelector2)
        self.clearBuffers2.setClearColor(Qt.white)
        self.clearBuffers2.setBuffers(Qt3DRender.QClearBuffers.ColorDepthBuffer)
        self.renderStateSet2 = Qt3DRender.QRenderStateSet(self.clearBuffers2)
        self.cullFace2 = Qt3DRender.QCullFace(self.renderStateSet2)
        self.depthTest2 = Qt3DRender.QDepthTest()
        self.depthTest2.setDepthFunction(Qt3DRender.QDepthTest.Less)
        self.renderStateSet2.addRenderState(self.depthTest2)

        self.setActiveFrameGraph(self.rootNode)

        # FPS counting
        self.fps_counter = 0
        self.fps_update_time = None
        self.fps_widget = QLabel(self.parent())
        self.fps_widget.setToolTip(self.tr('FPS counter for %s view' % self.title))
        self.mainWindow.statusBar().addPermanentWidget(self.fps_widget)

        self.frameAction = Qt3DLogic.QFrameAction()
        self.rootEntity.addComponent(self.frameAction)
        self.frameAction.triggered.connect(self.frame_action)

    def frame_action(self):
        self.fps_counter += 1

        if self.fps_update_time is None:
            self.fps_update_time = time.time()
            return

        this_frame = time.time()

        if this_frame - self.fps_update_time >= 1:
            self.fps_widget.setText('FPS: %s\t' % self.fps_counter)
            self.fps_counter = 0
            self.fps_update_time = this_frame


class View3DDockWidget(QDockWidget):

    # noinspection PyPep8Naming
    class View3DContainer(QWidget):

        def __init__(self, parent=None):
            QtWidgets.QWidget.__init__(self, parent)

            self.view3dWidget = View3DWidget(parent.parent())

            layout = QtWidgets.QHBoxLayout()

            self.windowContainer = QWidget.createWindowContainer(self.view3dWidget)
            layout.addWidget(self.windowContainer)

            # Set minimum size so that main window does not start having 3D widgets
            # almost invisible. Keep it small to allow user to arrange docks as they want.
            # Let's use a value a bit larger than 4, to be able to arrange all view widgets
            # vertically or horizontally, while the OS task bar / menu start is present.
            self.windowContainer.setMinimumSize(
                int(self.view3dWidget.screen().size().width() / 5.5),
                int(self.view3dWidget.screen().size().height() / 5.5))

            # Do not waste space for margins between views, draw the content instead
            layout.setContentsMargins(0, 0, 0, 0)

            self.setLayout(layout)

    def __init__(self, parent=None, title='3D'):
        QDockWidget.__init__(self, parent)
        self.title = title

        self.setFeatures(self.DockWidgetMovable | self.DockWidgetFloatable)
        self.setWindowTitle(title)

        self.container = View3DDockWidget.View3DContainer(self)
        self.setWidget(self.container)


class MainWindow(QMainWindow):

    def __init__(self, parent):
        QMainWindow.__init__(self, parent=parent)
        self.setWindowFlags(Qt.Widget)

        # self.topWindow = QtGui.QWindow()
        # self.topWindow.setSurfaceType(QtGui.QSurface.OpenGLSurface)
        self.topWindow = Qt3DExtras.Qt3DWindow()
        self.topWindow.renderSettings().deleteLater()
        self.topWindow.activeFrameGraph().deleteLater()
        self.topWindow.setWidth(200)
        self.topWindow.setHeight(100)
        self.topWindow.show()

        self.free3DWidget = View3DDockWidget(self, title=self.tr('3D'))
        self.addDockWidget(QtCore.Qt.LeftDockWidgetArea, self.free3DWidget)


class MainWindowWrapper(QMainWindow):
    """Wrapper for the actual main window, keeping the real window as a central widget

    This solution allows to restart the main window completely with a newly opened file"""

    def __init__(self):
        QMainWindow.__init__(self, parent=None)
        mw = MainWindow(self)
        self.setCentralWidget(mw)
        mw.show()


if __name__ == "__main__":
    app = QApplication(sys.argv)

    window = MainWindowWrapper()
    window.show()

    sys.exit(app.exec_())