OSADL QA Farm on Real-time of Mainline Linux

About - Hardware - CPUs - Benchmarks - Graphics - Benchmarks - Kernels - Boards/Distros - Latency monitoring - Latency plots - System data - Profiles - Compare - Awards

Default latency plot of system in rack #0, slot #2

Click here to display the system's profile data or here to proceed to next system.

Click on a legend element to toggle display of that core, ctrl-click inverts display, shift-click enables all.
Data to construct the above plot have been generated using the RT test utility cyclictest.
Unexpectedly long latencies may be caused by SMIs
Total number of samples: 100 million
Resolution of latency scale: normal
Duration: 5 hours, 33 minutes, lowest P state: performance
Compare latency of primary with shadow system

Characteristics of the 100 highest latencies:

System rack0slot2.osadl.org (updated Sat Mar 28, 2026 00:43:33)
Delayed (victim)					Switcher (culprit)			Timestamp	CPU
PID	Prio	Total latency (µs)	T^(,W^*) latency (µs)	Cmd	PID	Prio	Cmd	Timestamp	CPU
255563	2	97	0,1	sleep7	255562	-21	grep	23:12:57	7
3904668	99	49	47,2	cyclictest	160415	-21	kworker/u16:5+flush-8:0	23:18:19	6
3904668	99	49	47,2	cyclictest	160415	-21	kworker/u16:5+flush-8:0	23:18:19	6
3904149	2	48	38,7	sleep4	0	-21	swapper/4	18:59:57	4
3904668	99	47	45,2	cyclictest	62875	-21	kworker/u16:4+flush-8:0	22:08:21	6
3904668	99	46	44,2	cyclictest	299854	-21	kworker/u16:1+events_unbound	23:46:15	6
326686	2	45	0,1	sleep5	0	-21	swapper/5	23:42:28	5
3904668	99	43	41,2	cyclictest	327094	-21	kworker/u16:2+events_unbound	23:56:27	6
3904668	99	43	41,2	cyclictest	327094	-21	kworker/u16:2+events_unbound	23:52:44	6
3904668	99	43	41,2	cyclictest	299854	-21	kworker/u16:1+events_unbound	00:17:53	6
3904668	99	43	41,2	cyclictest	267077	-21	kworker/u16:0+events_unbound	00:01:47	6
3904668	99	43	41,2	cyclictest	195975	-21	kworker/u16:1+events_unbound	23:12:19	6
3904668	99	43	41,2	cyclictest	16099	-21	kworker/u16:5+events_unbound	21:49:43	6
3904668	99	43	40,2	cyclictest	195975	-21	kworker/u16:1+events_unbound	23:12:58	6
3904668	99	42	40,2	cyclictest	51900	-21	kworker/u16:1+events_unbound	21:59:02	6
3904668	99	42	40,2	cyclictest	160415	-21	kworker/u16:5+events_unbound	22:36:43	6
3904670	99	41	34,6	cyclictest	650	-21	systemd-journal	19:12:57	7
3904668	99	41	39,2	cyclictest	16099	-21	kworker/u16:5+events_unbound	21:43:55	6
3904668	99	41	39,2	cyclictest	160415	-21	kworker/u16:5+events_unbound	22:40:12	6
3904141	2	41	16,8	sleep5	0	-21	swapper/5	18:59:50	5
3904668	99	40	38,2	cyclictest	4138081	-21	kworker/u16:3+events_unbound	21:05:31	6
3904668	99	40	38,2	cyclictest	4089870	-21	kworker/u16:2+events_unbound	21:13:59	6
3904668	99	40	38,2	cyclictest	398728	-21	kworker/u16:3+events_unbound	00:26:15	6
3904668	99	40	38,2	cyclictest	195975	-21	kworker/u16:1+events_unbound	22:55:52	6
3904668	99	40	38,2	cyclictest	160415	-21	kworker/u16:5+events_unbound	23:04:47	6
3904668	99	39	37,2	cyclictest	4089870	-21	kworker/u16:2+events_unbound	21:30:43	6
3904668	99	39	37,2	cyclictest	4013541	-21	kworker/u16:4+flush-8:0	20:33:19	6
3904668	99	39	37,2	cyclictest	267077	-21	kworker/u16:0+events_unbound	23:42:40	6
3904668	99	39	37,2	cyclictest	267077	-21	kworker/u16:0+events_unbound	23:35:11	6
3904668	99	39	37,2	cyclictest	267077	-21	kworker/u16:0+events_unbound	00:21:22	6
3904668	99	39	37,2	cyclictest	207500	-21	kworker/u16:2+events_unbound	23:26:19	6
3904668	99	39	37,2	cyclictest	160415	-21	kworker/u16:5+events_unbound	23:02:54	6
3904668	99	39	37,2	cyclictest	160415	-21	kworker/u16:5+events_unbound	22:50:04	6
3904668	99	39	37,2	cyclictest	146040	-21	kworker/u16:1+events_unbound	22:27:24	6
3904198	2	39	29,7	sleep6	0	-21	swapper/6	19:00:39	6
3904668	99	38	36,2	cyclictest	62875	-21	kworker/u16:4+events_unbound	22:06:04	6
3904668	99	38	36,2	cyclictest	4186003	-21	kworker/u16:0+events_unbound	21:40:59	6
3904668	99	38	36,2	cyclictest	4138081	-21	kworker/u16:3+events_unbound	21:21:40	6
3904668	99	38	36,2	cyclictest	299854	-21	kworker/u16:1+events_unbound	23:32:23	6
3895272	2	38	3,30	sleep7	0	-21	swapper/7	18:57:59	7
3904670	99	37	35,1	cyclictest	64250	-21	ssh	21:52:58	7
3904668	99	37	35,2	cyclictest	62875	-21	kworker/u16:4+events_unbound	21:56:18	6
3904668	99	37	35,2	cyclictest	4089870	-21	kworker/u16:2+events_unbound	21:24:11	6
3904668	99	37	35,2	cyclictest	267077	-21	kworker/u16:0+events_unbound	00:05:38	6
3904666	99	37	35,2	cyclictest	62875	-21	kworker/u16:4+flush-8:0	22:02:51	5
3904670	99	36	31,4	cyclictest	4185595	-21	latency_hist	21:22:56	7
3904668	99	36	34,2	cyclictest	62875	-21	kworker/u16:4+events_unbound	22:31:16	6
3904668	99	36	34,2	cyclictest	4138081	-21	kworker/u16:3+events_unbound	21:33:55	6
3904668	99	36	34,2	cyclictest	4138081	-21	kworker/u16:3+events_unbound	21:09:08	6
3904668	99	36	34,2	cyclictest	299854	-21	kworker/u16:1+events_unbound	00:29:36	6
3904666	99	36	33,2	cyclictest	299854	-21	kworker/u16:1+flush-8:0	00:03:02	5
3904668	99	35	33,2	cyclictest	87889	-21	kworker/u16:0+events_unbound	22:47:30	6
3904668	99	35	33,2	cyclictest	62826	-21	kworker/u16:2+events_unbound	22:20:35	6
3904668	99	35	33,2	cyclictest	112761	-21	kworker/u16:1+events_unbound	22:14:40	6
3904668	99	35	32,2	cyclictest	4013541	-21	kworker/u16:4+events_unbound	20:47:59	6
3904666	99	35	33,2	cyclictest	3914068	-21	kworker/u16:2+flush-8:0	19:18:22	5
3904668	99	34	33,1	cyclictest	267077	-21	kworker/u16:0+events_unbound	00:09:27	6
3904668	99	33	31,2	cyclictest	3841765	-21	kworker/u16:0+flush-8:0	19:07:56	6
3904670	99	32	31,0	cyclictest	0	-21	swapper/7	22:33:28	7
3904668	99	30	28,2	cyclictest	3946617	-21	kworker/u16:3+events_unbound	20:17:11	6
3904668	99	30	28,2	cyclictest	3841765	-21	kworker/u16:0+events_unbound	19:34:27	6
3904668	99	30	28,2	cyclictest	3841765	-21	kworker/u16:0+events_unbound	19:22:08	6
3904666	99	29	27,2	cyclictest	267077	-21	kworker/u16:0+flush-8:0	00:13:30	5
3904666	99	29	27,2	cyclictest	207500	-21	kworker/u16:2+flush-8:0	22:58:27	5
3904662	99	29	27,2	cyclictest	51900	-21	kworker/u16:1+events_unbound	21:58:52	4
3904662	99	29	26,2	cyclictest	267077	-21	kworker/u16:0+events_unbound	23:43:10	4
3904666	99	28	25,2	cyclictest	16099	-21	kworker/u16:5+flush-8:0	21:43:03	5
3904662	99	28	25,2	cyclictest	4071089	-21	kworker/u16:1+flush-8:0	21:08:33	4
3904670	99	27	25,1	cyclictest	0	-21	swapper/7	22:43:12	7
3904668	99	27	25,2	cyclictest	3946823	-21	kworker/u16:4+events_unbound	19:29:12	6
3904668	99	27	25,2	cyclictest	3946617	-21	kworker/u16:3+events_unbound	19:27:07	6
3904666	99	27	25,2	cyclictest	87889	-21	kworker/u16:0+events_unbound	22:23:55	5
3904666	99	27	25,2	cyclictest	4013541	-21	kworker/u16:4+flush-8:0	20:47:56	5
3904666	99	27	25,2	cyclictest	4013541	-21	kworker/u16:4+flush-8:0	20:47:56	5
3904662	99	27	25,2	cyclictest	62826	-21	kworker/u16:2+events_unbound	22:18:21	4
3904662	99	27	25,2	cyclictest	267077	-21	kworker/u16:0+flush-8:0	23:42:13	4
3904662	99	27	25,2	cyclictest	16099	-21	kworker/u16:5+flush-8:0	21:38:35	4
3904662	99	27	24,3	cyclictest	207500	-21	kworker/u16:2+events_unbound	23:13:08	4
3904662	99	27	24,2	cyclictest	87889	-21	kworker/u16:0+flush-8:0	22:03:16	4
3904662	99	27	24,2	cyclictest	299854	-21	kworker/u16:1+flush-8:0	00:28:06	4
3904670	99	26	25,0	cyclictest	0	-21	swapper/7	23:58:22	7
3904670	99	26	0,1	cyclictest	0	-21	swapper/7	23:03:22	7
3904668	99	26	24,2	cyclictest	4061131	-21	kworker/u16:0+events_unbound	20:32:08	6
3904668	99	26	24,2	cyclictest	3841765	-21	kworker/u16:0+events_unbound	19:11:43	6
3904666	99	26	24,2	cyclictest	4186003	-21	kworker/u16:0+events_unbound	21:37:59	5
3904670	99	25	25,0	cyclictest	0	-21	swapper/7	21:50:00	7
3904668	99	25	23,2	cyclictest	3994673	-21	kworker/u16:2+flush-8:0	19:53:07	6
3904668	99	25	23,2	cyclictest	3946617	-21	kworker/u16:3+events_unbound	20:22:55	6
3904668	99	25	23,2	cyclictest	3946617	-21	kworker/u16:3+events_unbound	20:04:32	6
3904666	99	25	23,2	cyclictest	4071089	-21	kworker/u16:1+flush-8:0	20:53:22	5
3904662	99	25	24,1	cyclictest	87889	-21	kworker/u16:0+events_unbound	22:08:30	4
3904662	99	25	23,2	cyclictest	299854	-21	kworker/u16:1+flush-8:0	23:58:11	4
3904670	99	24	23,0	cyclictest	0	-21	swapper/7	20:12:43	7
3904668	99	24	22,2	cyclictest	3946617	-21	kworker/u16:3+events_unbound	19:44:09	6
3904666	99	24	22,2	cyclictest	4089870	-21	kworker/u16:2+flush-8:0	21:22:56	5
3904666	99	24	22,2	cyclictest	4013541	-21	kworker/u16:4+events_unbound	20:48:27	5
3904666	99	24	22,2	cyclictest	195975	-21	kworker/u16:1+events_unbound	23:10:04	5
3904662	99	24	22,2	cyclictest	62826	-21	kworker/u16:2+events_unbound	21:53:45	4
3904662	99	24	22,2	cyclictest	207500	-21	kworker/u16:2+flush-8:0	22:58:40	4
3904670	99	23	22,0	cyclictest	0	-21	swapper/7	19:38:11	7
^Timer ^*Wakeup (latency=timer+wakeup+contextswitch)


Access: public		Short link to this page
This page was last modified on 2019-05-28 - 00:17