QDataStream fail to send large strings
-
Why should a string be 500MB large?
How do you send it/receive it? -
Sender
port = 500 socket = QTcpSocket() # Connect to the server socket.connectToHost("localhost", port) msg = str(uuid.uuid4()) for a in range(0, 28): msg += msg print(len(msg) / 1024 / 1024) # Wait for the connection to be established if socket.waitForConnected(5000): print("sending...") # Create a QByteArray to hold the data data = QByteArray("Hello World!".encode()) data = json.dumps({"hello": "world", "this": [{"this": "is"}, {"awe": "some"}], "stupidLongStuff": msg, "END": 1}) # Create a QDataStream to write the data stream = QDataStream(socket) dataSize = len(data) print("sending data size : ", dataSize) # Write the data to the stream stream.writeInt32(dataSize) stream.writeString(str(data)) # Wait for the data to be sent socket.waitForBytesWritten() print("Data sent successfully") else: print("Error:", socket.errorString())Receiver is just
def _handleReadyRead(self): client_socket: QTcpSocket = self.sender() stream = QDataStream(client_socket) jobSize = client_socket.bytesAvailable() print("Thread dat : ", self.thread(), jobSize) size = stream.readInt32() data = stream.readString()The int works fine, but string after is fail. No matter waht I try, dataStream wont push it.
-
readRead() is called when data is available at the socket, not when all data is received (what a socket can't know) so you have to reassemble your data by yourself.
-
@Christian-Ehrlicher Should DataStream not deal with it? Handle chunking/etc/etc?
Regards
Dariusz -
@Dariusz said in QDataStream fail to send large strings:
Should DataStream not deal with it? Handle chunking/etc/etc?
Why? "The QDataStream class provides serialization of binary data to a QIODevice" - nothing more and nothing less.
You can look at the transaction support for QDataStream though.
-
@Christian-Ehrlicher Hmmm I think because its class is responsible for serializing/deserializing/managing data so I thought it would handle the chunking/etc with socket. Else it feels like half-measure and I still have to solve the chunking/disconnection/etc.
-
Hi,
QDataStream serializes something to a QIODevice. What happens after that is not its concern. For example, why would it need to know how to write to a socket more than knowing how to write to a file ? That's what the QIODevice are responsible for. If you need higher level structure management, make your own QIODevice.
-
Please read about the transactions.
Also writing this as a string is nonsense - it just doubles the size of transferred bytes for no reason. Just sent a bytearray directly. Also there is no need to send the dataSize since QDataStream adds this information to the stream by itself. -
@Christian-Ehrlicher Hmmm I'm not having any luck with this method so far :/
Code below is self contained test. The DataStream is not commitTransaction and thus still waits for more data but never reads it/deals with it.
Socket dies, no more data comes in, handle_ready_read never gets called again, even if I force call again it wont read any more data... :/ Where am I hitting wall here?import json import uuid from functools import partial from PySide2.QtNetwork import * from PySide2.QtCore import * from PySide2.QtWidgets import * class BasicServer(QTcpServer): s_newJob = Signal(object) def __init__(self, port: int): QTcpServer.__init__(self) self._mStatus = self.listen(QHostAddress.LocalHost, port) self._mSocketList = {} self._mJobs = {} self.newConnection.connect(self._handleNewConnection) print("Starting logic server on port : ", port, " Status : ", self._mStatus) self._mTimer = QElapsedTimer() self._mTimer.start() def getStatus(self) -> int: return self._mStatus def _handleNewConnection(self): client_socket: QTcpSocket = self.nextPendingConnection() client_socket.readyRead.connect(partial(self.handle_ready_read, client_socket)) stream = QDataStream(client_socket) self._mSocketList[client_socket] = [stream, QByteArray()] client_socket.disconnected.connect(partial(self._handleSocketDisconnect, client_socket)) def handle_ready_read(self, client_socket): streamItem = self._mSocketList.get(client_socket, None) if streamItem is None: return print("reading?") stream = streamItem[0] stream.startTransaction() streamData = streamItem[1] stream >> streamData print(len(streamData)) self._mSocketList[client_socket][1] = streamData if not stream.commitTransaction(): return print("IM FINISH! ", streamData) def _handleSocketDisconnect(self, socketPtr: QTcpSocket): print("Socket is gone...") return # for now keep socket "alive" killPort = socketPtr.peerPort() if killPort in self._mJobs: if not self._mJobs[killPort]["emitted"]: if self._mJobs[killPort]["toDownload"] == 0: pass del self._mJobs[killPort] if socketPtr in self._mSocketList: del self._mSocketList[socketPtr] print("Killing socket", killPort, socketPtr) socketPtr.deleteLater() def clientTest(): port = 500 socket = QTcpSocket() # Connect to the server socket.connectToHost("localhost", port) msg = str(uuid.uuid4()) for a in range(0, 10): msg += msg print(len(msg) / 1024 / 1024) # Wait for the connection to be established if socket.waitForConnected(5000): print("sending...") # Create a QByteArray to hold the data data = QByteArray("Hello World!".encode()) jDumped = json.dumps({"hello": "world", "this": [{"this": "is"}, {"awe": "some"}], "stupidLongStuff": msg, "END": 1}) data = QByteArray(jDumped.encode()) # Create a QDataStream to write the data stream = QDataStream(socket) # stream.setVersion(QDataStream.Qt_5_15) print("Data size : ", len(data)) stream << data # stream.writeString("hellooo") # dataSize = len(data) # print("sending data size : ", dataSize) # Write the data to the stream # stream.writeInt64(dataSize) # socket.waitForBytesWritten() # socket.write(struct.pack("l", len(data))) # socket.write(data.encode()) socket.waitForBytesWritten() print("Data sent successfully") else: print("Error:", socket.errorString()) app = QApplication([]) s = BasicServer(500) clientTest() app.exec_()
