From 994b452ebf73c8f770155ca8ccd71724c7af6d2d Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" <a.darwish@linutronix.de> Date: Sat, 16 Oct 2021 10:49:08 +0200 Subject: [PATCH 044/166] net: sched: Use _bstats_update/set() instead of raw writes The Qdisc::running sequence counter, used to protect Qdisc::bstats reads from parallel writes, is in the process of being removed. Qdisc::bstats read/writes will synchronize using an internal u64_stats sync point instead. Modify all bstats writes to use _bstats_update(). This ensures that the internal u64_stats sync point is always acquired and released as appropriate. Signed-off-by: Ahmed S. Darwish <a.darwish@linutronix.de> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> Signed-off-by: David S. Miller <davem@davemloft.net> --- net/core/gen_stats.c | 9 +++++---- net/sched/sch_cbq.c | 3 +-- net/sched/sch_gred.c | 7 ++++--- net/sched/sch_htb.c | 25 +++++++++++++++---------- net/sched/sch_qfq.c | 3 +-- 5 files changed, 26 insertions(+), 21 deletions(-) Index: linux-5.15.32-rt39/net/core/gen_stats.c =================================================================== @ linux-5.15.32-rt39/net/core/gen_stats.c:129 @ EXPORT_SYMBOL(gnet_stats_basic_packed_in static void gnet_stats_add_basic_cpu(struct gnet_stats_basic_packed *bstats, struct gnet_stats_basic_cpu __percpu *cpu) { + u64 t_bytes = 0, t_packets = 0; int i; for_each_possible_cpu(i) { @ linux-5.15.32-rt39/net/core/gen_stats.c:143 @ static void gnet_stats_add_basic_cpu(str packets = bcpu->bstats.packets; } while (u64_stats_fetch_retry_irq(&bcpu->syncp, start)); - bstats->bytes += bytes; - bstats->packets += packets; + t_bytes += bytes; + t_packets += packets; } + _bstats_update(bstats, t_bytes, t_packets); } void gnet_stats_add_basic(const seqcount_t *running, @ linux-5.15.32-rt39/net/core/gen_stats.c:169 @ void gnet_stats_add_basic(const seqcount packets = b->packets; } while (running && read_seqcount_retry(running, seq)); - bstats->bytes += bytes; - bstats->packets += packets; + _bstats_update(bstats, bytes, packets); } EXPORT_SYMBOL(gnet_stats_add_basic); Index: linux-5.15.32-rt39/net/sched/sch_cbq.c =================================================================== --- linux-5.15.32-rt39.orig/net/sched/sch_cbq.c +++ linux-5.15.32-rt39/net/sched/sch_cbq.c @ linux-5.15.32-rt39/net/core/gen_stats.c:568 @ cbq_update(struct cbq_sched_data *q) long avgidle = cl->avgidle; long idle; - cl->bstats.packets++; - cl->bstats.bytes += len; + _bstats_update(&cl->bstats, len, 1); /* * (now - last) is total time between packet right edges. Index: linux-5.15.32-rt39/net/sched/sch_gred.c =================================================================== --- linux-5.15.32-rt39.orig/net/sched/sch_gred.c +++ linux-5.15.32-rt39/net/sched/sch_gred.c @ linux-5.15.32-rt39/net/core/gen_stats.c:356 @ static int gred_offload_dump_stats(struc { struct gred_sched *table = qdisc_priv(sch); struct tc_gred_qopt_offload *hw_stats; + u64 bytes = 0, packets = 0; unsigned int i; int ret; @ linux-5.15.32-rt39/net/core/gen_stats.c:385 @ static int gred_offload_dump_stats(struc table->tab[i]->bytesin += hw_stats->stats.bstats[i].bytes; table->tab[i]->backlog += hw_stats->stats.qstats[i].backlog; - _bstats_update(&sch->bstats, - hw_stats->stats.bstats[i].bytes, - hw_stats->stats.bstats[i].packets); + bytes += hw_stats->stats.bstats[i].bytes; + packets += hw_stats->stats.bstats[i].packets; sch->qstats.qlen += hw_stats->stats.qstats[i].qlen; sch->qstats.backlog += hw_stats->stats.qstats[i].backlog; sch->qstats.drops += hw_stats->stats.qstats[i].drops; sch->qstats.requeues += hw_stats->stats.qstats[i].requeues; sch->qstats.overlimits += hw_stats->stats.qstats[i].overlimits; } + _bstats_update(&sch->bstats, bytes, packets); kfree(hw_stats); return ret; Index: linux-5.15.32-rt39/net/sched/sch_htb.c =================================================================== --- linux-5.15.32-rt39.orig/net/sched/sch_htb.c +++ linux-5.15.32-rt39/net/sched/sch_htb.c @ linux-5.15.32-rt39/net/core/gen_stats.c:1311 @ nla_put_failure: static void htb_offload_aggregate_stats(struct htb_sched *q, struct htb_class *cl) { + u64 bytes = 0, packets = 0; struct htb_class *c; unsigned int i; @ linux-5.15.32-rt39/net/core/gen_stats.c:1327 @ static void htb_offload_aggregate_stats( if (p != cl) continue; - cl->bstats.bytes += c->bstats_bias.bytes; - cl->bstats.packets += c->bstats_bias.packets; + bytes += c->bstats_bias.bytes; + packets += c->bstats_bias.packets; if (c->level == 0) { - cl->bstats.bytes += c->leaf.q->bstats.bytes; - cl->bstats.packets += c->leaf.q->bstats.packets; + bytes += c->leaf.q->bstats.bytes; + packets += c->leaf.q->bstats.packets; } } } + _bstats_update(&cl->bstats, bytes, packets); } static int @ linux-5.15.32-rt39/net/core/gen_stats.c:1363 @ htb_dump_class_stats(struct Qdisc *sch, cl->bstats = cl->leaf.q->bstats; else gnet_stats_basic_packed_init(&cl->bstats); - cl->bstats.bytes += cl->bstats_bias.bytes; - cl->bstats.packets += cl->bstats_bias.packets; + _bstats_update(&cl->bstats, + cl->bstats_bias.bytes, + cl->bstats_bias.packets); } else { htb_offload_aggregate_stats(q, cl); } @ linux-5.15.32-rt39/net/core/gen_stats.c:1584 @ static int htb_destroy_class_offload(str WARN_ON(old != q); if (cl->parent) { - cl->parent->bstats_bias.bytes += q->bstats.bytes; - cl->parent->bstats_bias.packets += q->bstats.packets; + _bstats_update(&cl->parent->bstats_bias, + q->bstats.bytes, + q->bstats.packets); } offload_opt = (struct tc_htb_qopt_offload) { @ linux-5.15.32-rt39/net/core/gen_stats.c:1952 @ static int htb_change_class(struct Qdisc htb_graft_helper(dev_queue, old_q); goto err_kill_estimator; } - parent->bstats_bias.bytes += old_q->bstats.bytes; - parent->bstats_bias.packets += old_q->bstats.packets; + _bstats_update(&parent->bstats_bias, + old_q->bstats.bytes, + old_q->bstats.packets); qdisc_put(old_q); } new_q = qdisc_create_dflt(dev_queue, &pfifo_qdisc_ops, Index: linux-5.15.32-rt39/net/sched/sch_qfq.c =================================================================== --- linux-5.15.32-rt39.orig/net/sched/sch_qfq.c +++ linux-5.15.32-rt39/net/sched/sch_qfq.c @ linux-5.15.32-rt39/net/core/gen_stats.c:1238 @ static int qfq_enqueue(struct sk_buff *s return err; } - cl->bstats.bytes += len; - cl->bstats.packets += gso_segs; + _bstats_update(&cl->bstats, len, gso_segs); sch->qstats.backlog += len; ++sch->q.qlen;