Delay of picture refreshing increases and suddenly not responding.
-
Hello everyone,
my target is to show two camera videos synchronically. As mentioned in many post, I need to use Threads to actualize the window. When I ran the code below, I can see an increase of delay between camera image and shown image, and suddenly the window is not responding. After a few seconds, the window closed itself. I don't really understand the reason and it would be very nice if someone can help me.
Update thread(problem with print)
class UpdateThread(QThread): changePixmap = pyqtSignal(list) def __init__(self,gen:Union[Generator,callable],size:QSize,window:QWidget): super().__init__() self.gen = gen self.mode = isinstance(gen,Generator) self.size = size self.stop_flag = False self.window = window def get_next(self): if self.mode: return next(self.gen) else: return (self.gen)() def run(self) -> None: while not self.stop_flag: ims = self.get_next() res = [] for i in range(len(ims)): res.append(self.process_im(ims[i])) self.changePixmap.emit(res) print('"',end='') def process_im(self,im:np.ndarray): if isinstance(im,QImage): return im h,w,ch = im.shape im = QImage(im,w,h,ch*w,QImage.Format_RGB888) im = im.scaled(self.size/20*9, Qt.KeepAspectRatio) return im def get_latest(self): return self.latest def stop(self): self.stop_flag = Trueclass App, the whole controller
# using PyQt5 class App: def __init__(self, video_save_path:str, image_save_path:str,cam_num:int,warning:bool=False): self.cam_num = cam_num self.video_path = video_save_path self.image_save_path = image_save_path self.use_warning = warning self.PCB_FLAG = False self.SAVER_FLAG = False self.TI_STARTED = False self.stop_flag = False self.init_camera() self.init_grabber_and_saver(cfg.MAX_LENGTH,self.video_path,cfg.FPS,cfg.IMAGE_SIZE) self.init_pcb_detector() self.init_gui() def init_grabber_and_saver(self,cache_len:int,video_save_path:str,fps:Union[int,float],image_size:Union[list,tuple,np.ndarray]): if self.gas is not None: del self.gas self.set_video_path(video_save_path) video_name = [f'cam{i+1}-'+datetime.now().strftime("%Y%m%d,%H-%M") + '.mp4' for i in range(self.cam_num)] self.gas = GrabberAndSaver(self.gen,cache_len,self.cam_num,cfg.IMAGE_SHAPE, [cv2.VideoWriter(self.video_path + video_name[i],cv2.VideoWriter_fourcc(*'mp4v'),fps,image_size) for i in range(self.cam_num)]) self.SAVER_FLAG = True self.start_origin() def init_camera(self): self.cam = Camera(self.cam_num) self.gen = self.cam.generate_image_all() self.gas = None print('###############################camera opened#############################') def init_gui(self): self.app = QApplication(sys.argv) self.window_size = self.app.screens()[0].availableSize() self.gui = VideoDisplay() self.gui.init_all_close(self.close_all) self.up_thread = UpdateThread(self.gas.get_latest,QSize(*cfg.IMAGE_SIZE),self.gui.wd) self.up_thread.changePixmap.connect(self.gui.wd.setImage) self.up_thread.start() self.gui.wd.button_connect(None,None,{0:self.pcb_detect,1:self.TI_detect}) self.gui.show() print('########################gui initialization completed#####################') def init_pcb_detector(self): self.pcb = YOLOv7Detector(cfg.YOLOV7_PCB,(cfg.INPUT_SIZE_PCB,cfg.INPUT_SIZE_PCB),cfg.IOU_PCB,cfg.SCORE_PCB) def init_TI_detector(self): if not self.PCB_FLAG: raise ExecutionOrderError('PCB Detection or camera calibration must be executed first.') moore = [Moore(cfg.MAX_LENGTH,cfg.NUM_CLASS_TIME,cfg.TOLERANCE,cfg.FPS) for _ in range(len(self.box)*self.cam_num)] self.ti = TIDetection(cfg.YOLOV7_TI,cfg.INPUT_SIZE_TI,cfg.IOU_TI,cfg.SCORE_TI,moore,self.image_save_path,self.warning if self.use_warning else None) print('########################ti initialization completed######################') def pcb_detect(self): ims = self.gas.get_latest() for im in ims: self.box,score,_,_ = self.pcb.predict(im) if self.box.shape[-1] == 0: print("No PCB detected. Please try again with another camera direction. If this warning was shown several times, this program can't work with the pcb.") if self.cam_num==2: self.box = np.array([[[0,0,1440,1080]],[[0,0,1440,1080]]],dtype=int)# for tests elif self.cam_num ==1: self.box = np.array([[[0,0,1440,1080]]],dtype=int)# for tests self.PCB_FLAG = True return self.PCB_FLAG = True image.draw_box(Image.fromarray(im),self.box,score).show() def TI_detect(self): if self.TI_STARTED: self.ti.stop() self.TI_STARTED = False else: self.start_TI_detect() self.TI_STARTED = True def start_TI_detect(self): if not self.SAVER_FLAG: raise ExecutionOrderError('Resource pool and video writer must be initialized before starting TI detection.') if hasattr(self,'pcb'): del self.pcb self.init_TI_detector() self.ti_thread = Thread(target=self.ti.proceed,args=[self.box,None,self.gas.get_latest]) self.ti_thread.start() def start_origin(self): self.grabber = Thread(target=self.gas.proceed) self.grabber.start() def warning(self): Msg(cfg.MSG_TITLE,cfg.MSG_TEXT,cfg.SOUND_FILE) def set_video_path(self, video_path:str): self.video_path = video_path if video_path.endswith(('/','\\')) else video_path + '/' def close_all(self): # clr = {0:'r',1:'g'} # for i in range(len(self.ti.times)): # time_test = [self.ti.times[i][j+1]-self.ti.times[i][j] for j in range(len(self.ti.times[i])-1)] # print(time_test) # if len(time_test)>0: # print(f'TI-processing average time needed:{sum(time_test)/len(time_test)}') # plt.plot(time_test[1:],clr[i]) # plt.savefig('test.jpg') self.stop_flag = True if hasattr(self,'ti'): self.ti.stop() self.gas.stop()all gui classes except
UpdateThreadclass VideoDisplay(QMainWindow): def __init__(self) -> None: super().__init__() self.setStyleSheet("background-color: #fff5f6;") self.setWindowTitle("Original Camera Video") self.wd = VideoWindow() self.setCentralWidget(self.wd) self.move(QPoint(60,20)) def init_all_close(self,func:Callable): self.func = func def closeEvent(self, a0: QCloseEvent) -> None: self.func() a0.accept() class VideoWindow(QWidget): def __init__(self): super().__init__() self.video1 = QLabel(self) self.video2 = QLabel(self) self.gbox = QGridLayout() self.vbox = QVBoxLayout() self.b1 = QPushButton('detect PCB',self) self.b2 = QPushButton('start/stop',self) self.vbox.addWidget(self.b1) self.vbox.addWidget(self.b2) self.gbox.addWidget(self.video1,0,0,1,1) self.gbox.addLayout(self.vbox,0,1,1,2) self.gbox.addWidget(self.video2,1,1,2,2) self.setLayout(self.gbox) self.widget_dict = { 0:self.video1, 1:self.video2 } self.button_dict = { 0:self.b1, 1:self.b2, } def button_connect(self,idx:int,func:Callable,container:dict=None): if container: for idx,func in container.items(): self.button_dict[idx].clicked.connect(func) else: self.button_dict[idx].clicked.connect(func) @pyqtSlot(list) def setImage(self, image): for i in range(len(image)): self.widget_dict[i].setPixmap(QPixmap.fromImage(image[i]))class TI,
proceed()is used by threadclass TIDetection(YOLOv7Detector): def __init__(self, model: str, input_size: int, iou: float, score: float, automat:Union[Moore,list,tuple], image_save:str, warn:callable) -> None: super().__init__(model, input_size, iou, score) self.automat = automat if isinstance(automat,Iterable) else [automat] self.warn = warn self.res_detect = 0 if image_save and not image_save.endswith(('/','\\')): image_save += '/' self.im_path = image_save self.pre = datetime.now() self.stop_flag = False self.times = [[],[]] def proceed_one(self, im:np.ndarray,au_idx:tuple): self.now = datetime.now() box,score,color,max_cls = self.predict(im) im = Image.fromarray(im).convert('RGB') if self.im_path and (self.now-self.pre).seconds>10 and max_cls > 1: im_name = self.im_path + self.now.strftime("%H-%M-%S")+'.jpg' self.pre = self.now else: im_name = None im = image.draw_box(im,box,score,color,im_name) ti = self._update(max_cls,au_idx) if ti == 4: self._warn() return im def proceed(self,pos:np.ndarray,gen:Generator=None, func: callable=None): self.times[0].append(time.time()) self.times[1].append(time.time()) cycle = 0 if gen: while not self.stop_flag: ims = next(gen) for i in range(len(pos)): im = ims[i] boxes = pos[i] for j in range(len(boxes)): pcb = image.box_image(im,boxes[j]) self.proceed_one(pcb,i*len(boxes)+j) self.times[i].append(time.time()-self.times[i][-1]) #print(self.times) cycle += 1 if cycle>100: break elif func: while not self.stop_flag: ims = func() #print(pos) for i in range(len(pos)): im = ims[i] boxes = pos[i] #print(boxes) for j in range(len(boxes)): pcb = image.box_image(im,boxes[j]) self.proceed_one(pcb,i*len(boxes)+j) #print('proceeded') self.times[i].append(time.time()) print(f'time:{time.time()},cycle{cycle}') #print(self.times) cycle += 1 if cycle>100: break del self.model print('ti deleted') def one(self,pos:np.ndarray,im:np.ndarray): for j in range(len(pos)): pcb = image.box_image(im,pos[j]) self.proceed_one(pcb,0) def set_image_save_dir(self,image_save:str): self.im_path = image_save def _update(self, value:int, au_idx:int): return self.automat[au_idx].update(value) def get_automat_latest_condition(self,au_idx:int): return self.automat[au_idx].get_latest() def _warn(self): if not self.warn: return (self.warn)() def stop(self): self.stop_flag = Trueclass
GrabberAndSaver, grabs images from cameras and pushing them in a ringbuffer. And all images will be saved to a video. Methodproceedis called by thread.class GrabberAndSaver: cache = None ptr_latest = 0 ptr_earliest = 0 def __init__(self,gen: Generator,cache_len:int,cam_num:int,im_shape:Union[list,tuple,np.ndarray],video_writer:Union[list,tuple]) -> None: self.gen = gen self.cache = np.zeros((cache_len,cam_num,*im_shape),dtype = np.uint8) self.saver = Saver(video_writer) self.stop_flag = False def proceed(self): while not self.stop_flag: im = next(self.gen) #self.saver.write(im[...,::-1]) self._push(im) #print(f'thread res cache updated') def _push(self,value:Any): for i in range(len(value)): self.cache[self.ptr_latest,i] = value[i] if self.ptr_latest == len(self.cache) - 1: self.ptr_latest = 0 else: self.ptr_latest += 1 def get_latest(self): return self.cache[self.ptr_latest-1] def get_all(self): self.ptr_earliest += 1 return self.cache[self.ptr_earliest-1] def stop(self): self.stop_flag = True self.saver.stop()By the way, I saw a huge improvement by updating directly in thread instead of using slots(as commented).Why does the program behaves so? Thanks for helping!!!
@Kaninchen
You cannot/must not access any UI element, such as a widget, in a thread in Qt. No matter how much you would like to, you can't.Your code passes a
window:QWidgetto the thread, and that goesself.window.setImage(res)(naughty you are calling that on aQWidget, I think you're getting that fromVideoWindow...). Which does something to the widget, like set its pixmap. And you can't do that. So at some point your code goes wrong..... -
Alright, so I have to use signal and slots as codes commented. In this case, the performance is even worse, about 70% of all time not responding. Should this be my programming error or full usage of hardware resource? The values in the system monitor is totally normal. By the way, this is my first time programming frontend. And I have changed the codes so that the whole process can be seen.
-
Alright, so I have to use signal and slots as codes commented. In this case, the performance is even worse, about 70% of all time not responding. Should this be my programming error or full usage of hardware resource? The values in the system monitor is totally normal. By the way, this is my first time programming frontend. And I have changed the codes so that the whole process can be seen.
-
If I add one line in UpdateThread.run(), which is print('x'), then the program runs very smoothly until I start the neural network detection thread. It's unbelievable for me.
@Kaninchen said in Delay of picture refreshing increases and suddenly not responding.:
It's unbelievable for me.
Then stick to using
print(), or similar. The issue arises when you have threads running which wish to access the Qt UI, like widgets. Which are much more complex than just printing to stdout. And can only be accessed in their own GUI thread.So, yes, signals are needed. Or, go find a different UI library from Qt which does allow multi-threaded access to windowing-system elements in the way you want.
-
@Kaninchen said in Delay of picture refreshing increases and suddenly not responding.:
It's unbelievable for me.
Then stick to using
print(), or similar. The issue arises when you have threads running which wish to access the Qt UI, like widgets. Which are much more complex than just printing to stdout. And can only be accessed in their own GUI thread.So, yes, signals are needed. Or, go find a different UI library from Qt which does allow multi-threaded access to windowing-system elements in the way you want.
@JonB I haven't understood well. What I replied is based on signals and slot, so there is already no GUI access outside main-thread. I'm sorry to have discarded changes in my codes. Now it should be the proper version. Since I used the proper way to actualize images, at least I think, how can stdout impact on GUI? Even if I used
print()in the thread(againsetImagethroughPyQtSlot), the window starts to be not responding after I start the detection thread. If I don't use GUI, everything goes well. -
Hi,
What does your detection thread do ?
How is it implemented ?
How do you start it ? -
@SGaist Hi,
My detection thread takes batch of images from cache pool
GrabberAndSaverand used the initialized YOLOv7 model to make predictions. Maybe I should explain my codes. There are two detectors in my program. One detects PCB and the other detects Flame. Both detectors are YOLOv7 model in Pytorch. PCB will be detected by clicking button. Another button controls Flame detection. Once clicked, the PCB detector will be deleted and the flame detection model will be loaded to GPU. I started the thread by using Python threads. The codes can be found in classApp. By the way, the cache pool is also a python thread which is started before gui initialization. Implementation also updated . -
while not self.stop_flag: ims = self.get_next() res = [] for i in range(len(ims)): res.append(self.process_im(ims[i])) self.changePixmap.emit(res) print('"',end='')this might be incredibly fast and overwhelm the signal system. Consider timing one loop, and restricting the execution to something reasonable, once every 20 ms for example
-
while not self.stop_flag: ims = self.get_next() res = [] for i in range(len(ims)): res.append(self.process_im(ims[i])) self.changePixmap.emit(res) print('"',end='')this might be incredibly fast and overwhelm the signal system. Consider timing one loop, and restricting the execution to something reasonable, once every 20 ms for example
