//// MenuExtensions.h
#include <QAction>
#include <QFileDialog>
#include <QObject>
#include <QStandardPaths>
#include <QStringList>
#include <QWidget>

class MenuExtensions : public QObject
{
    Q_OBJECT
public:
    MenuExtensions(QWidget* parent)
        : m_fileOpen(new QAction(parent))
        , m_parent(parent)
        , m_lastOpenDirectory(QStandardPaths::writableLocation(QStandardPaths::DocumentsLocation))
    {
        connect(m_fileOpen, &QAction::triggered, this, &MenuExtensions::HandleOpenFile);
    }
private slots:
    void HandleOpenFile()
    {
        QStringList requestedFiles = OpenFileDialog(QString("Open experiment"), QString("Experiment files (*.emd)"));
        // call the loading interface in the business logic 
    }
private:
    QStringList OpenFileDialog(const QString& caption, const QString& filter)
    {
        QFileDialog dialog(m_parent, caption, m_lastOpenDirectory, filter);
        dialog.setObjectName("OpenFileDialog");
        dialog.setFileMode(QFileDialog::ExistingFiles);
        connect(&dialog, &QFileDialog::directoryEntered, this, &MenuExtensions::LastOpenDirectoryChanged);
        if (dialog.exec())
        {
            return dialog.selectedFiles();
        }
        return QStringList();
    }
    void LastOpenDirectoryChanged(const QString& dir)
    {
        m_lastOpenDirectory = dir;
    }
public: // public just for the purpose of this minimal example
    QAction* m_fileOpen;
    QWidget* m_parent;
    QString m_lastOpenDirectory;
};

/// TestMinimum.cpp
#include <QApplication>
#include <QDir>
#include <QFileDialog>
#include <QFileInfo>
#include <QMainWindow>
#include <QStandardPaths>
#include <QTemporaryFile>
#include <QtTest/QtTest>
#include <QTimer>
#include "MenuExtensions.h"

void SetDirectoryAndFile(QWidget* window, const QFileInfo& file)
{
    QFileDialog* dialog = window->findChild<QFileDialog*>("OpenFileDialog");
    dialog->setDirectory(file.absolutePath());
    dialog->directoryEntered(file.absolutePath());
    dialog->selectFile(QDir::toNativeSeparators(file.fileName()));

    // Next follows the problematic 'enter', which will produce 'The filename is not valid',
    // because the file dialog changed the native separator back into '/'
    QTest::keyClick(dialog, Qt::Key_Enter);
}

int main(int argc, char** argv)
{
    QApplication app(argc, argv);

    QDir folder(QStandardPaths::writableLocation(QStandardPaths::DocumentsLocation));
    QDir::setCurrent(folder.absolutePath());
    QTemporaryFile emdFile("ExperimentXXXXXX.emd");
    emdFile.open();

    QMainWindow window;
    MenuExtensions extensions(&window);

    QTimer::singleShot(50, [&window, &folder, &emdFile](){ SetDirectoryAndFile(&window, QFileInfo(emdFile.fileName())); });
    extensions.m_fileOpen->trigger();
}

BTW, I also realized I didn't mention that this is for the Native dialog. Could it be I hit some bug?

You're looking for QDir::toNativeSeparators