QModelIndex.internalPointer() returning random objects and crashing - PySide2

[Flanagan]

Hello, I have a TreeView with TreeModel with the drag & drop implemented as indicated in the documentation:

• https://doc.qt.io/qt-5/model-view-programming.html#using-drag-and-drop-with-item-views
• https://doc.qt.io/qt-5/qtwidgets-itemviews-simpletreemodel-example.html
• https://doc.qt.io/qt-5/qtwidgets-itemviews-editabletreemodel-example.html

Everything was fine but after a certain amount of drag & drop movements the model started bringing random objects from index.internalPointer() in method get_node(index).

def get_node(self, index):
        if index.isValid():
            return index.internalPointer()

        return self.root

The rest of the code at the bottom of the post.

.internalPointer() should give me a CriterionNode object, but sometimes I get corrupt data, CriterionNode semi-empty objects or totally random objects. I even got objects (like Line2D objects) belonging to the matplotlib chart, whose logic is separated from the tree model.

Video proof of the error: https://mega.nz/file/3gwlQLyD#6JJnVFAZF5T13rCeGUUOQ7nx94YQXjquJc5YuBT8SVI

Images of random object returning from internalPointer():

TreeModel

from PySide2 import QtCore
from PySide2.QtCore import QMimeData

from models.tree_node import CriterionNode


def add_children_recursive(parent, children_data):
    if children_data:
        for child_data in children_data:
            child = CriterionNode(parent, *child_data['data'])
            add_children_recursive(child, child_data['children'])
            parent.children.append(child)


class MyTreeModel(QtCore.QAbstractItemModel):
    node_dropped = QtCore.Signal(int, int, QtCore.QModelIndex)

    def __init__(self):
        super().__init__(parent=None)
        self.root = None  # The header data is in the root node data

    def __len__(self):
        return len(self.root) - 1

    def __str__(self):
        return str(self.root)

    @property
    def last_index(self):
        return self.index(self.rowCount() - 1)

    def add(self, new_node):
        self.insertRow(self.rowCount())
        self.setData(self.last_index, new_node)

    def clear(self):
        self.beginResetModel()
        self.root = CriterionNode(name='root')
        self.endResetModel()

    def columnCount(self, index):
        return 1

    def data(self, index, role):
        if not index.isValid():
            return None

        if role == QtCore.Qt.DisplayRole:
            node: CriterionNode = self.get_node(index)
            return node.data_at(0)

    def dropMimeData(self, data, action, row, column, parent_index) -> bool:
        if row != -1:  # Inserting between nodes
            insert_pos = row
        else:  # Inserting in the node at the end
            insert_pos = self.get_node(parent_index).child_count()

        encoded_data = data.data('nodes')
        stream = QtCore.QDataStream(encoded_data, QtCore.QIODevice.ReadOnly)
        dropped_nodes = []
        while not stream.atEnd():
            droped_node = stream.readQVariant()
            dropped_nodes.append(droped_node)

        parent = self.get_node(parent_index)

        if row == -1 and dropped_nodes[0].parent == parent:
            return False

        correct_actual_index = insert_pos
        if dropped_nodes[0].parent == parent and insert_pos > dropped_nodes[0].row():
            correct_actual_index -= len(dropped_nodes)

        self.insertRows(insert_pos, len(dropped_nodes), parent_index)
        insert_pos = insert_pos
        for droped_node in dropped_nodes:
            index = self.index(insert_pos, 0, parent_index)
            self.setData(index, droped_node)
            insert_pos += 1

        self.node_dropped.emit(correct_actual_index, 0, parent_index)

        return True

    def flags(self, index):
        if index.isValid():
            return QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsDragEnabled | QtCore.Qt.ItemIsDropEnabled

        return QtCore.Qt.ItemIsDropEnabled

    def get_node(self, index):
        if index.isValid():
            return index.internalPointer()

        return self.root

    def headerData(self, section, orientation, role):
        if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
            return self.root.data[section]

        return None

    def index(self, row, column=0, parent_index=None):
        parent_index = parent_index or QtCore.QModelIndex()

        if not self.hasIndex(row, column, parent_index):
            return QtCore.QModelIndex()

        parent: CriterionNode = self.get_node(parent_index)
        child: CriterionNode = parent.children[row]
        if child:
            return self.createIndex(row, column, child)

        return QtCore.QModelIndex()

    def insertRows(self, row, count, parent_index=None) -> bool:
        parent_index = parent_index or QtCore.QModelIndex()

        self.beginInsertRows(parent_index, row, row + count - 1)
        node = self.get_node(parent_index)
        node.insert_children(row, count)
        self.endInsertRows()

        return True

    def load_data(self, criteria_data):
        self.beginResetModel()
        self.root = CriterionNode(name='root')
        add_children_recursive(self.root, criteria_data)
        self.endResetModel()

    def mimeData(self, indexes):
        mime_data = QMimeData()
        encoded_data = QtCore.QByteArray()
        stream = QtCore.QDataStream(encoded_data, QtCore.QIODevice.WriteOnly)

        for index in indexes:
            if not index.isValid():
                continue

            node = self.get_node(index)
            stream.writeQVariant(node)

        mime_data.setData('nodes', encoded_data)
        return mime_data

    def mimeTypes(self):
        return ['nodes']

    def parent(self, index):
        if not index.isValid():
            return QtCore.QModelIndex()

        child = self.get_node(index)
        if not child or child == self.root:
            return QtCore.QModelIndex()

        parent = child.parent

        if not parent or parent == self.root:
            return QtCore.QModelIndex()

        return self.createIndex(parent.row(), 0, parent)

    def removeRows(self, row, count, parent_index) -> bool:
        self.beginRemoveRows(parent_index, row, row + count - 1)
        node = self.get_node(parent_index)
        node.remove_children(row, count)
        self.endRemoveRows()
        return True

    def rowCount(self, parent_index=None):
        parent_index = parent_index or QtCore.QModelIndex()

        if parent_index.column() > 0:
            return 0

        parent = self.get_node(parent_index)

        return parent.child_count() if parent else 0

    def setData(self, index, new_node, role=QtCore.Qt.EditRole) -> bool:
        node = self.get_node(index)
        node.update_data(new_node)
        self.dataChanged.emit(index, index, [QtCore.Qt.DisplayRole, QtCore.Qt.EditRole])
        return True

    def supportedDropActions(self):
        return QtCore.Qt.MoveAction | QtCore.Qt.CopyAction

.
.
.
.
TreeNode

import json

from models.base_model import BaseModel


def json_encoder_mini(obj):
    if type(obj) is set:
        return list(obj)
    elif isinstance(obj, TreeNode):
        return [
            obj._data[0] if obj._data else '---',
            *obj.children
        ]
    else:
        return vars(obj)


class TreeNode(BaseModel):
    def __init__(self, parent=None, data=None):
        self.parent = parent
        self._data = data or []
        self.children = []

    def __eq__(self, other):
        return self._data == other._data if isinstance(other, TreeNode) else False

    def __len__(self):
        if not self.children:
            return 1
        return sum(len(child) for child in self.children) + 1

    def __str__(self):
        return f'{self.__class__.__name__}\n{self.to_json(indent=4)}'

    def child_count(self):
        return len(self.children)

    def data_at(self, column):
        try:
            return self._data[column]
        except IndexError:
            return None

    def row(self):
        if self.parent:
            return self.parent.children.index(self)

        return 0  # Never used

    def column_count(self):
        return len(self._data)

    def insert_children(self, position, count):
        if position not in range(0, self.child_count() + 1):
            return False

        for row in range(count):
            node = self.__class__(self)
            self.children.insert(position, node)

        return True

    def insert_columns(self, position, data_columns):
        if position not in range(0, self.child_count() + 1):
            return False

        for column in range(data_columns):
            self._data.insert(position, None)

        for child in self.children:
            child.insert_columns(position, data_columns)

    def remove_children(self, position, count):
        if position not in range(0, self.child_count() + 1 - count):
            return False

        i_to_delete = range(position, position + count)
        self.children = [child for i, child in enumerate(self.children) if i not in i_to_delete]

    def set_data(self, column, value) -> bool:
        try:
            self._data[column] = value
            return True
        except IndexError:
            return False

    def to_json_mini(self, indent=None):
        return json.dumps(self, default=json_encoder_mini, indent=indent)

    def update_data(self, other: 'TreeNode'):
        other.parent = self.parent
        self.parent.children[self.row()] = other


class CriterionNode(TreeNode):
    def __init__(self,
                 parent=None,
                 name='',
                 description='',
                 measurement_unit='%',
                 minimum='0',
                 maximum='100',
                 intermiedate_points=None):
        super().__init__(parent, [name, description, measurement_unit, minimum, maximum, intermiedate_points or []])

    @property
    def name(self):
        return self.data_at(0)

    @name.setter
    def name(self, name):
        self.set_data(0, name)

    @property
    def description(self):
        return self.data_at(1)

    @description.setter
    def description(self, desription):
        self.set_data(1, desription)

    @property
    def measurement_unit(self):
        return self.data_at(2)

    @measurement_unit.setter
    def measurement_unit(self, measurement_unit):
        self.set_data(2, measurement_unit)

    @property
    def minimum(self):
        return self.data_at(3)

    @minimum.setter
    def minimum(self, minimum):
        self.set_data(3, minimum)

    @property
    def maximum(self):
        return self.data_at(4)

    @maximum.setter
    def maximum(self, maximum):
        self.set_data(4, maximum)

    @property
    def intermediate_points(self):
        return self.data_at(5)

    @intermediate_points.setter
    def intermediate_points(self, intermediate_points):
        self.set_data(5, intermediate_points)

Thank you so much for everything.

[Christian Ehrlicher]

Please show us your createIndex() function where you put the internal pointer into the index.

[Flanagan]

@Christian-Ehrlicher you mean where in the code do i call createIndex()?
Well I do it in MyTreeModel, in index() and in parent() methods.
If you mean reimplement the createIndex() method, I haven't, I thought it wasn't necessary.

def index(self, row, column=0, parent_index=None):
        parent_index = parent_index or QtCore.QModelIndex()

        if not self.hasIndex(row, column, parent_index):
            return QtCore.QModelIndex()

        parent: CriterionNode = self.get_node(parent_index)
        child: CriterionNode = parent.children[row]
        if child:
            return self.createIndex(row, column, child)

        return QtCore.QModelIndex()

def parent(self, index):
        if not index.isValid():
            return QtCore.QModelIndex()

        child = self.get_node(index)
        if not child or child == self.root:
            return QtCore.QModelIndex()

        parent = child.parent

        if not parent or parent == self.root:
            return QtCore.QModelIndex()

        return self.createIndex(parent.row(), 0, parent)

[Flanagan]

Maybe I should report a bug?

[VRonin]

I'm pretty sure it's not a Qt bug.
Could you create an QAbstractItemModelTester next to your model and reproduce the bug to see if it gives more details on where the problem is?

[Flanagan]

@VRonin

QAbstractItemModelTester is missing in Pyside2:
https://wiki.qt.io/Qt_for_Python_Missing_Bindings

However, trying to make a minimal reproducible example, I removed the chart, which I thought dont affect my problem, and it turns out that it do.

I am using matplotlib embedded within Qt, something that is supposed to be used quite a bit. It seems that when cleaning the axes something breaks internally after doing several drag & drop movements.

I have been commenting lines until I have found the line that causes the application to fail:

canvas.py

from dataclasses import dataclass, field
from typing import List

from matplotlib import ticker
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg
from matplotlib.figure import Figure


@dataclass
class ChartValues:
    x_values: List[float] = field(default_factory=lambda: [])
    y_values: List[float] = field(default_factory=lambda: [])
    x_label: str = ''
    y_label: str = ''
    min_x: float = 0.0
    max_x: float = 0.0


class ChartCanvas(FigureCanvasQTAgg):
    def __init__(self, parent=None):
        self.fig = Figure(figsize=(1, 1), dpi=100)
        self.axes = self.fig.add_subplot(1, 1, 1)
        self.fig.subplots_adjust(bottom=0.15, top=0.92)
        super().__init__(self.fig)
        self.chart_values = ChartValues()

        self.setParent(parent)

    def update_figure(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self.chart_values, k, v)

        # Clear axes
        self.axes.cla()  # same as self.axes.clear()

        # Axis ranges
        self.axes.set_xlim(self.chart_values.min_x, self.chart_values.max_x)
        self.axes.set_ylim(0, 1)

        # y-axis step
        self.axes.yaxis.set_major_locator(ticker.MultipleLocator(0.2))

        # Background grid
        self.axes.xaxis.grid(color='gray', linewidth=0.4, linestyle='dotted')
        self.axes.yaxis.grid(color='gray', linewidth=0.4, linestyle='dotted')

        # Axis labels
        self.axes.set_xlabel(self.chart_values.x_label)
        self.axes.set_ylabel(self.chart_values.y_label)

        # Plot & draw
        self.axes.plot([0, *self.chart_values.x_values, self.chart_values.max_x],
                       [0, *self.chart_values.y_values, 1],
                       'green')
        # self.fig.tight_layout()
        self.draw()

[Flanagan]

I have made a minimal reproducible example in a single script main.py. I cannot make it shorter.

It is necessary to have PySide2 and matplotlib installed. I use Python 3.8.2. Although having 3.6 or superior is fine.

pip install Pyside2
pip install matplotlib

Run the script below and follow the steps in the next video to get random objects:
https://mega.nz/file/6sxnjKSb#w7uT0IzoGSgChTH5ZR-mzgbKS1CF5g6gkFTZ_PgMRDo

main.py:

import sys

from PySide2 import QtCore
from PySide2 import QtWidgets
from PySide2.QtCore import QMimeData
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg
from matplotlib.figure import Figure

data = [
    {
        'data': [
            'Criterio 1',
            'descripcion\n1',
            'points',
            '0',
            '100',
            ['35', '38']
        ],
        'children': [
            {
                'data': [
                    'Criterio 2',
                    'descripcion\n2',
                    'metros',
                    '5',
                    '10',
                    ['2', '7']
                ],
                'children': [
                    {
                        'data': [
                            'Criterio 4',
                            'descripcion\n4',
                            'vatios',
                            '50',
                            '2500',
                            []
                        ],
                        'children': []
                    }
                ]
            },
            {
                'data': [
                    'Criterio 3',
                    'descripcion\n3',
                    'julios',
                    '1',
                    '15468',
                    ['3', '5', '7']
                ],
                'children': []
            }
        ]
    },
    {
        'data': [
            'Criterio 5',
            'descripcion\n5',
            'voltios',
            '9',
            '5000',
            ['50']
        ],
        'children': [
            {
                'data': [
                    'Criterio 6',
                    'descripcion\n6',
                    'litros',
                    '0',
                    '999',
                    ['2', '8', '24', '68', '187']
                ],
                'children': []
            }
        ]
    }
]


class ChartCanvas(FigureCanvasQTAgg):
    def __init__(self):
        self.fig = Figure(figsize=(1, 1), dpi=100)
        self.axes = self.fig.add_subplot(1, 1, 1)
        self.fig.subplots_adjust(bottom=0.15, top=0.92)
        super().__init__(self.fig)

    def update_figure(self):
        # Clear axes
        self.axes.cla()  # same as self.axes.clear()
        self.draw()


class MyTreeView(QtWidgets.QTreeView):
    def __init__(self, canvas):
        super().__init__()
        self.setDragDropMode(self.InternalMove)
        self.canvas = canvas

    def setModel(self, model):
        super().setModel(model)
        self.model().node_dropped.connect(self.update_selection, QtCore.Qt.QueuedConnection)
        self.expandAll()

    def update_selection(self, row, column, parent_index):
        self.expandRecursively(parent_index)
        self.setCurrentIndex(self.model().index(row, column, parent_index))

    def currentChanged(self, current, previous):
        super().currentChanged(current, previous)
        if not current.isValid():
            return
        self.canvas.update_figure()


def add_children_recursive(parent, children_data):
    if children_data:
        for child_data in children_data:
            child = TreeNode(parent, child_data['data'])
            add_children_recursive(child, child_data['children'])
            parent.children.append(child)


class MyTreeModel(QtCore.QAbstractItemModel):
    node_dropped = QtCore.Signal(int, int, QtCore.QModelIndex)

    def __init__(self):
        super().__init__(parent=None)
        self.root = None  # The header data is in the root node data
        self._internal_objects = []

    def __len__(self):
        return len(self.root) - 1

    def __str__(self):
        return str(self.root)

    def columnCount(self, index):
        return 1

    def data(self, index, role):
        if not index.isValid():
            return None

        if role == QtCore.Qt.DisplayRole:
            node: TreeNode = self.get_node(index)
            return node.data_at(0)

    def dropMimeData(self, data, action, row, column, parent_index) -> bool:
        if row != -1:  # Inserting between nodes
            insert_pos = row
        else:  # Inserting in the node at the end
            insert_pos = self.get_node(parent_index).child_count()

        encoded_data = data.data('nodes')
        stream = QtCore.QDataStream(encoded_data, QtCore.QIODevice.ReadOnly)
        dropped_nodes = []
        while not stream.atEnd():
            droped_node = stream.readQVariant()
            dropped_nodes.append(droped_node)

        parent = self.get_node(parent_index)

        if row == -1 and dropped_nodes[0].parent == parent:
            return False

        correct_actual_index = insert_pos
        if dropped_nodes[0].parent == parent and insert_pos > dropped_nodes[0].row():
            correct_actual_index -= len(dropped_nodes)

        self.insertRows(insert_pos, len(dropped_nodes), parent_index)
        insert_pos = insert_pos
        for droped_node in dropped_nodes:
            index = self.index(insert_pos, 0, parent_index)
            self.setData(index, droped_node)
            insert_pos += 1

        self.node_dropped.emit(correct_actual_index, 0, parent_index)

        return True

    def flags(self, index):
        if index.isValid():
            return QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsDragEnabled | QtCore.Qt.ItemIsDropEnabled

        return QtCore.Qt.ItemIsDropEnabled

    def get_node(self, index):
        if index.isValid():
            return index.internalPointer()

        return self.root

    def headerData(self, section, orientation, role):
        if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
            return self.root.data_at(section)

        return None

    def index(self, row, column=0, parent_index=None):
        parent_index = parent_index or QtCore.QModelIndex()

        if not self.hasIndex(row, column, parent_index):
            return QtCore.QModelIndex()

        parent: TreeNode = self.get_node(parent_index)
        child: TreeNode = parent.children[row]
        if child:
            return self.createIndex(row, column, child)

        return QtCore.QModelIndex()

    def insertRows(self, row, count, parent_index=None) -> bool:
        parent_index = parent_index or QtCore.QModelIndex()

        self.beginInsertRows(parent_index, row, row + count - 1)
        node = self.get_node(parent_index)
        node.insert_children(row, count)
        self.endInsertRows()

        return True

    def load_data(self, criteria_data):
        self.beginResetModel()
        self.root = TreeNode(data=['root'])
        add_children_recursive(self.root, criteria_data)
        self.endResetModel()

    def mimeData(self, indexes):
        mime_data = QMimeData()
        encoded_data = QtCore.QByteArray()
        stream = QtCore.QDataStream(encoded_data, QtCore.QIODevice.WriteOnly)

        for index in indexes:
            if not index.isValid():
                continue

            node = self.get_node(index)
            stream.writeQVariant(node)

        mime_data.setData('nodes', encoded_data)
        return mime_data

    def mimeTypes(self):
        return ['nodes']

    def parent(self, index):
        if not index.isValid():
            return QtCore.QModelIndex()

        child = self.get_node(index)
        if not child or child == self.root:
            return QtCore.QModelIndex()

        parent = child.parent

        if parent is None or parent == self.root:
            return QtCore.QModelIndex()

        return self.createIndex(parent.row(), 0, parent)

    def removeRows(self, row, count, parent_index) -> bool:
        self.beginRemoveRows(parent_index, row, row + count - 1)
        node = self.get_node(parent_index)
        node.remove_children(row, count)
        self.endRemoveRows()
        return True

    def rowCount(self, parent_index=None):
        parent_index = parent_index or QtCore.QModelIndex()

        if parent_index.column() > 0:
            return 0

        parent = self.get_node(parent_index)

        return parent.child_count() if parent else 0

    def setData(self, index, new_node, role=QtCore.Qt.EditRole) -> bool:
        node = self.get_node(index)
        node.update_data(new_node)
        self.dataChanged.emit(index, index, [QtCore.Qt.DisplayRole, QtCore.Qt.EditRole])
        return True

    def supportedDropActions(self):
        return QtCore.Qt.MoveAction | QtCore.Qt.CopyAction


class TreeNode:
    def __init__(self, parent=None, data=None):
        self.parent = parent
        self._data = data or []
        self.children = []

    def __eq__(self, other):
        return self._data == other._data if isinstance(other, TreeNode) else False

    def __len__(self):
        if not self.children:
            return 1
        return sum(len(child) for child in self.children) + 1

    def child_count(self):
        return len(self.children)

    def data_at(self, column):
        try:
            return self._data[column]
        except IndexError:
            return None

    def row(self):
        if self.parent:
            return self.parent.children.index(self)

        return 0  # Never used

    def column_count(self):
        return len(self._data)

    def insert_children(self, position, count):
        if position not in range(0, self.child_count() + 1):
            return False

        for row in range(count):
            node = self.__class__(self)
            self.children.insert(position, node)

        return True

    def insert_columns(self, position, data_columns):
        if position not in range(0, self.child_count() + 1):
            return False

        for column in range(data_columns):
            self._data.insert(position, None)

        for child in self.children:
            child.insert_columns(position, data_columns)

    def remove_children(self, position, count):
        if position not in range(0, self.child_count() + 1 - count):
            return False

        i_to_delete = range(position, position + count)
        self.children = [child for i, child in enumerate(self.children) if i not in i_to_delete]

    def set_data(self, column, value) -> bool:
        try:
            self._data[column] = value
            return True
        except IndexError:
            return False

    def update_data(self, other: 'TreeNode'):
        other.parent = self.parent
        self.parent.children[self.row()] = other


app = QtWidgets.QApplication([])

model = MyTreeModel()
model.load_data(data)

canvas = ChartCanvas()
tree_view = MyTreeView(canvas)
tree_view.setModel(model)
tree_view.setHeaderHidden(True)

layout = QtWidgets.QHBoxLayout()
layout.addWidget(tree_view)
layout.addWidget(canvas)
layout.setStretch(0, 1)
layout.setStretch(1, 1)

w = QtWidgets.QWidget()
w.setLayout(layout)
w.show()

sys.exit(app.exec_())

[Flanagan]

Should this thread be moved to the Qt for python subforum?

Really i need help. I can clearly run the script above and see the memory leak by Qt. I get Affine2D, Line2D, tuple and more random objects...

Maybe I should use QTreeWidget or make my own internalPointer in python.

[Thar]

Necroing the old thread, but since I ran into at least a similar issue, I figured I'd respond to hopefully give closure.

As above, I was getting a random object out of internalPointer. The cause of the issue is that pyside2 does not maintain a handle of the object so Python was destroying the objects between creating the index and accessing them later.