OSADL QA Farm on Real-time of Mainline Linux

About - Hardware - CPUs - Benchmarks - Graphics - Benchmarks - Kernels - Boards/Distros - Latency monitoring - Latency plots - System data - Profiles - Compare - Awards

Default latency plot of shadow in rack #3, slot #5

Rack #0/#0 #1 #2 #3 #4 #5 #6 #7 #8 - Rack #1/#0 #1 #2 #3 #4 #5 #6 #7 #8 -
Rack #2/#0 #1 #2 #3 #4 #5 #6 #7 #8 - Rack #3/#0 #1 #2 #3 #4 #5 #6 #7 #8 -
Rack #4/#0 #1 #2 #3 #4 #5 #6 #7 #8 - Rack #5/#0 #1 #2 #3 #4 #5 #6 #7 #8 -
Rack #6/#0 #1 #2 #3 #4 #5 #6 #7 #8 - Rack #7/#0 #1 #2 #3 #4 #5 #6 #7 #8 -
Rack #8/#0 #1 #2 #3 #4 #5 #6 #7 #8 - Rack #9/#0 #1 #2 #3 #4 #5 #6 #7 #8 -
Rack #a/#0 #1 #2 #3 #4 #5 #6 #7 #8 - Rack #b/#0 #1 #2 #3 #4 #5 #6 #7 #8 -
Rack #c/#0 #1 #2 #3 #4 #5 #6 #7 #8 - Rack #d/#0 #1 #2 #3 #4 #5 #6 #7 #8 -
Rack #e/#0 #1 #2 #3 #4 #5 #6 #7 #8 - Rack #f/#0 #1 #2 #3 #4 #5 #6 #7 #8
Special All - All RT - Optimization - Ethernet - Thumbnails - Next

Click here to display the system's profile data or here to proceed to next system.

Click on a legend element to toggle display of that core, ctrl-click inverts display, shift-click enables all.
Data to construct the above plot have been generated using the RT test utility cyclictest.

Total number of samples: 100 million
Resolution of latency scale: normal
Duration: 5 hours, 33 minutes
Compare latency of primary with shadow system

Characteristics of the highest latencies:

System rack3slot5s.osadl.org (updated Fri Apr 10, 2026 00:43:30)
Delayed (victim)					Switcher (culprit)			Timestamp	CPU
PID	Prio	Total latency (µs)	T^(,W^*) latency (µs)	Cmd	PID	Prio	Cmd	Timestamp	CPU
24668	2	135	0	sleep1	31296	50	irq/29-eth0_g0_	23:54:48	1
12	1	85	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	23:34:41	0
12	1	85	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	23:34:41	0
12	1	62	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	00:03:21	0
12	1	52	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	21:30:09	1
12	1	49	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	23:45:36	0
31297	50	48	0	irq/30-eth0_g0_	30754	-21	grep	00:19:47	1
31297	50	48	0	irq/30-eth0_g0_	30754	-21	grep	00:19:47	1
12	1	48	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	23:26:08	1
12	1	47	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	22:17:39	1
12	1	47	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	23:25:40	0
12	1	47	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	22:47:03	1
12	1	47	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	22:47:03	1
12	1	47	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	22:09:28	1
12	1	46	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	00:38:41	1
31297	50	45	0	irq/30-eth0_g0_	31295	-21	kworker/1:1	23:03:08	1
31296	50	45	0	irq/29-eth0_g0_	22431	-21	netstat	21:34:39	0
12	1	45	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	23:32:54	1
12	1	45	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	22:22:00	1
12	1	45	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	00:33:40	1
12	1	45	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	00:08:54	1
12	1	45	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	00:01:59	1
12	1	45	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	23:05:16	1
12	1	45	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	22:34:39	1
12	1	45	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	21:12:39	1
12	1	45	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	20:59:10	1
31297	50	44	0	irq/30-eth0_g0_	3	1	ksoftirqd/0	00:04:39	0
31297	50	44	0	irq/30-eth0_g0_	12	1	ksoftirqd/1	22:32:46	1
25289	2	44	0	sleep0	31297	50	irq/30-eth0_g0_	21:45:18	0
12	1	44	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	23:52:29	0
12	1	44	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	23:39:39	1
12	1	44	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	22:50:38	1
12	1	44	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	21:49:06	1
31297	50	43	0	irq/30-eth0_g0_	24417	-21	/usr/sbin/munin	21:44:24	1
17849	2	43	0	sleep0	31297	50	irq/30-eth0_g0_	21:14:48	0
12	1	43	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	23:36:05	1
12	1	43	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	23:36:05	1
12	1	43	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	23:17:30	0
12	1	43	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	21:25:37	0
12	1	43	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	21:39:16	1
31296	50	42	0	irq/29-eth0_g0_	3565	-21	proc_pri	22:29:43	0
31296	50	42	0	irq/29-eth0_g0_	13	-21	kworker/0:1	22:14:39	0
12	1	42	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	23:19:37	0
12	1	42	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	22:09:35	0
31297	50	41	0	irq/30-eth0_g0_	31295	-21	kworker/1:1	00:18:09	1
31297	50	41	0	irq/30-eth0_g0_	31295	-21	kworker/1:1	00:13:25	1
31296	50	41	0	irq/29-eth0_g0_	18855	-21	latencyplot	19:09:23	0
31296	50	41	0	irq/29-eth0_g0_	1875	-21	runrttasks	21:02:01	0
31296	50	41	0	irq/29-eth0_g0_	12	1	ksoftirqd/1	21:22:33	0
12	1	41	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	21:59:45	1
31296	50	40	0	irq/29-eth0_g0_	13	-21	kworker/0:1	22:53:38	0
31296	50	40	0	irq/29-eth0_g0_	13	-21	kworker/0:1	21:09:37	0
12	1	40	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	22:19:36	0
12	1	40	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	00:12:46	0
10006	2	40	0	sleep1	31296	50	irq/29-eth0_g0_	22:55:14	1
31297	50	39	0	irq/30-eth0_g0_	31295	-21	kworker/1:1	21:22:44	1
31297	50	39	0	irq/30-eth0_g0_	12	1	ksoftirqd/1	23:49:41	1
31296	50	39	0	irq/29-eth0_g0_	23354	-21	df_abs	21:39:35	0
31296	50	39	0	irq/29-eth0_g0_	13	-21	kworker/0:1	22:54:37	0
31296	50	39	0	irq/29-eth0_g0_	13	-21	kworker/0:1	22:44:25	0
31296	50	39	0	irq/29-eth0_g0_	13	-21	kworker/0:1	22:44:25	0
31296	50	39	0	irq/29-eth0_g0_	13	-21	kworker/0:1	22:04:38	0
31296	50	39	0	irq/29-eth0_g0_	13	-21	kworker/0:1	21:58:43	0
31296	50	39	0	irq/29-eth0_g0_	13	-21	kworker/0:1	21:58:43	0
31296	50	39	0	irq/29-eth0_g0_	13	-21	kworker/0:1	00:27:55	0
18857	99	39	26	cyclictest	1894	-21	/usr/sbin/munin	20:22:18	1
18857	99	39	1	cyclictest	2990	-21	wc	20:14:44	1
18857	99	39	1	cyclictest	2003	-21	snmpd	21:54:41	1
18857	99	39	1	cyclictest	2003	-21	snmpd	21:54:41	1
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	23:57:17	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	23:31:02	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	23:13:17	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	23:02:36	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	22:03:37	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	21:08:39	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	20:48:14	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	20:48:14	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	20:37:23	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	19:50:23	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	19:35:27	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	19:28:13	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	19:22:14	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	00:23:21	0
12	1	39	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	00:23:21	0
12	1	39	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	23:09:41	1
12	1	39	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	20:34:35	1
31297	50	38	0	irq/30-eth0_g0_	31295	-21	kworker/1:1	23:22:07	1
31297	50	38	0	irq/30-eth0_g0_	31295	-21	kworker/1:1	22:25:09	1
31297	50	38	0	irq/30-eth0_g0_	17958	-21	latency	19:04:36	1
31297	50	38	0	irq/30-eth0_g0_	12	1	ksoftirqd/1	21:53:42	1
31297	50	38	0	irq/30-eth0_g0_	0	-21	swapper/1	22:44:22	1
31297	50	38	0	irq/30-eth0_g0_	0	-21	swapper/1	21:25:36	1
31296	50	38	0	irq/29-eth0_g0_	3	1	ksoftirqd/0	22:41:38	0
31296	50	38	0	irq/29-eth0_g0_	1875	-21	runrttasks	20:03:20	0
31296	50	38	0	irq/29-eth0_g0_	1875	-21	runrttasks	00:16:23	0
31296	50	38	0	irq/29-eth0_g0_	13	-21	kworker/0:1	23:43:15	0
31296	50	38	0	irq/29-eth0_g0_	13	-21	kworker/0:1	22:37:38	0
31296	50	38	0	irq/29-eth0_g0_	13	-21	kworker/0:1	22:24:50	0
31296	50	38	0	irq/29-eth0_g0_	13	-21	kworker/0:1	00:31:04	0
18857	99	38	3	cyclictest	8869	-21	memory	20:39:37	1
18857	99	38	2	cyclictest	29871	-21	processes	22:04:43	1
18857	99	38	1	cyclictest	14801	-21	grep	23:14:46	1
12	1	38	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	23:05:14	0
12	1	38	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	21:50:38	0
12	1	38	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	20:27:14	0
12	1	38	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	19:54:29	0
12	1	38	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	19:47:21	0
12	1	38	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	19:47:21	0
12	1	38	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	20:00:03	1
12	1	38	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	19:44:38	1
12	1	38	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	19:44:38	1
31297	50	37	0	irq/30-eth0_g0_	12	1	ksoftirqd/1	21:14:25	1
31297	50	37	0	irq/30-eth0_g0_	0	-21	swapper/1	00:28:59	1
31296	50	37	0	irq/29-eth0_g0_	21473	-21	unixbench_multi	21:29:47	0
31296	50	37	0	irq/29-eth0_g0_	18950	-21	/usr/sbin/munin	19:09:33	0
31296	50	37	0	irq/29-eth0_g0_	1875	-21	runrttasks	19:34:02	0
31296	50	37	0	irq/29-eth0_g0_	13	-21	kworker/0:1	00:38:38	0
31296	50	37	0	irq/29-eth0_g0_	12786	-21	runrttasks	20:54:43	0
18857	99	37	2	cyclictest	20641	-21	proc_pri	19:14:42	1
18857	99	37	1	cyclictest	1167	-21	latency_hist	20:09:24	1
18857	99	37	1	cyclictest	10200	-21	seq	20:44:41	1
18857	99	37	1	cyclictest	10200	-21	seq	20:44:41	1
12	1	37	0	ksoftirqd/1	31296	50	irq/29-eth0_g0_	20:09:42	1
31296	50	36	0	irq/29-eth0_g0_	1875	-21	runrttasks	20:22:48	0
31296	50	36	0	irq/29-eth0_g0_	1875	-21	runrttasks	20:10:49	0
31296	50	36	0	irq/29-eth0_g0_	1875	-21	runrttasks	20:06:15	0
31296	50	36	0	irq/29-eth0_g0_	1875	-21	runrttasks	19:40:34	0
31296	50	36	0	irq/29-eth0_g0_	1875	-21	runrttasks	19:40:34	0
31296	50	36	0	irq/29-eth0_g0_	0	-21	swapper/1	20:59:44	1
18857	99	36	1	cyclictest	25469	-21	ntp_states	19:34:41	1
18857	99	36	1	cyclictest	24234	-21	taskset	19:29:41	1
18857	99	36	1	cyclictest	14768	-21	sed	21:04:34	1
31296	50	35	0	irq/29-eth0_g0_	8897	-21	missed_timers	20:39:38	0
31296	50	35	0	irq/29-eth0_g0_	1842	-21	sshd	20:17:35	0
18857	99	35	1	cyclictest	5612	-21	sed	20:24:46	1
18857	99	35	1	cyclictest	28802	2	sleep1	19:49:34	1
18857	99	35	1	cyclictest	0	-21	swapper/1	19:26:22	1
12	1	35	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	20:29:31	0
12	1	35	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	19:16:03	0
31297	50	34	0	irq/30-eth0_g0_	6279	-21	seq	20:29:36	1
31297	50	34	0	irq/30-eth0_g0_	0	-21	swapper/1	23:48:59	1
31297	50	34	0	irq/30-eth0_g0_	0	-21	swapper/1	20:49:35	1
18857	99	34	1	cyclictest	20087	-21	grep	19:14:23	1
12	1	34	0	ksoftirqd/1	31297	50	irq/30-eth0_g0_	20:51:13	0
31296	50	33	0	irq/29-eth0_g0_	18857	99	cyclictest	19:39:44	1
31296	50	33	0	irq/29-eth0_g0_	18857	99	cyclictest	19:39:44	1
18857	99	33	1	cyclictest	0	-21	swapper/1	19:54:43	1
18857	99	33	1	cyclictest	0	-21	swapper/1	19:21:44	1
^Timer ^*Wakeup (latency=timer+wakeup+contextswitch)


Access: public		Short link to this page
This page was last modified on 2019-05-28 - 00:17