Okay, I have come up with a solution. I have setup a better handling for the hittest evaluation. I first check which side of the intersected boundingbox the moved item is closest to. Then I set the x, or y coordinate accordingly. I added some helper functions for this, to compute point-line distance and closest side of a rect to a point. They are all included in the snippet below.

NOTE: the collision test will also evaluate and prohibit the item to exceed the scene bounds. It will also only solve, overlapping one other item. If the border test results in another overlap I simply set the position back to where it was prior to mouse move. Works a lot nicer than my initial implementation though.

enum CustomGraphicsItem::BOX_SIDE {
LEFT,
RIGHT,
UPPER,
LOWER
};
void CustomGraphicsItem::mouseMoveEvent(QGraphicsSceneMouseEvent* e)
{
if(mode_ == MOVE) {
QPointF p = pos();
QGraphicsItem::mouseMoveEvent(e);
QPointF p_new = pos();
QList<QGraphicsItem*> col_it = collidingItems(Qt::IntersectsItemBoundingRect);
if(col_it.size() > 0) {
qreal x_min = col_it[0]->pos().x() - boundingRect().width();
qreal x_max = col_it[0]->pos().x() + col_it[0]->boundingRect().width();
qreal y_min = col_it[0]->pos().y() - boundingRect().height();
qreal y_max = col_it[0]->pos().y() + col_it[0]->boundingRect().height();
QRectF rect(QPointF(x_min, y_min), QPointF(x_max, y_max));
switch(closestSide(p_new, rect)) {
case LEFT:
p_new.setX(x_min);
break;
case RIGHT:
p_new.setX(x_max);
break;
case UPPER:
p_new.setY(y_min);
break;
case LOWER:
p_new.setY(y_max);
break;
}
setPos(p_new);
}
// check if item in scene bounds
qreal max_x = scene()->width() - boundingRect().width();
qreal max_y = scene()->height() - boundingRect().height();
if (x() < 0)
setPos(0, y());
else if (x() > max_x)
setPos(max_x, y());
if (y() < 0)
setPos(x(), 0);
else if (y() > max_y)
setPos(x(), max_y);
// if still colliding set pos back to start
col_it = collidingItems(Qt::IntersectsItemBoundingRect);
if(col_it.size() > 0)
setPos(p);
}
}
qreal CustomGraphicsItem::distance(const QPointF &p, const QLineF &l)
{
QPointF p1 = l.p1();
QPointF p2 = l.p2();
qreal x = p.x() - p1.x();
qreal y = p.y() - p1.y();
qreal x2 = p2.x() - p1.x();
qreal y2 = p2.y() - p1.y();
// if line is a point, return distance between point and one line node
qreal norm = sqrt(x2*x2 + y2*y2);
if (norm <= std::numeric_limits<int>::epsilon())
return sqrt(x*x + y*y);
// distance
return fabs(x*y2 - y*x2) / norm;
}
CustomGraphicsItem::BOX_SIDE CustomGraphicsItem::closestSide(const QPointF &p, const QRectF &rect)
{
qreal x_min = rect.x();
qreal x_max = rect.x() + rect.width();
qreal y_min = rect.y();
qreal y_max = rect.y() + rect.height();
qreal temp_dist = 0;
// left
QLineF l(QPointF(x_min, y_min), QPointF(x_min, y_max));
qreal min_dist = distance(p,l);
BOX_SIDE side = LEFT;
// right
l.setPoints(QPointF(x_max,y_min), QPointF(x_max, y_max));
temp_dist = distance(p,l);
if(temp_dist < min_dist) {
min_dist = temp_dist;
side = RIGHT;
}
// upper
l.setPoints(QPointF(x_min, y_min), QPointF(x_max, y_min));
temp_dist = distance(p,l);
if(temp_dist < min_dist) {
min_dist = temp_dist;
side = UPPER;
}
// lower
l.setPoints(QPointF(x_min, y_max), QPointF(x_max, y_max));
temp_dist = distance(p,l);
if(temp_dist < min_dist) {
min_dist = temp_dist;
side = LOWER;
}
return side;
}