net: openvswitch: Decouple flow operations from RTNL
Subsystems (3)
v4 cover net: openvswitch: Decouple flow operations from RTNL
When RTNL is contended, network-related control-plane operations can be delayed.
In such scenario, if OVS control-plane operations (such as vport creation) take a bit longer, it's acceptable. However, flow installation operations happen as part of upcall processing, executed in the context of of handler threads. If they get delayed, it affects the data-plane and can even result in packet drops.
Because flow operations also use ovs_mutex for concurrency protection and given RTNL can nest under ovs_mutex, contention can be easily transferred from RTNL to ovs_mutex, causing delay in flow operations.
In order to protect flow operations from RTNL delays, this series decouples them from ovs_mutex. First, the flow_table is converted into an rcu-protected pointer. Then two locking mechanisms are introduced: a per-table mutex and a refcount.
The mutex protects the flow_table against concurrent modifications, while the refcount is used to extend the lifetime of the flow_table beyond the rcu read-protected region used to dereference it.
The following is an example of how concurrent datapath deletion and flow modifcation would work.
Datapath deletion:
ovs_lock(); table = rcu_dereference_protected(dp->table, ...); rcu_assign_pointer(dp->table, NULL); ovs_flow_tbl_put(table); ovs_unlock();
Flow modification:
rcu_read_lock(); dp = get_dp(...); table = rcu_dereference(dp->table); ovs_flow_tbl_get(table); rcu_read_unlock(); mutex_lock(&table->lock); /* Perform modifications on the flow_table */ mutex_unlock(&table->lock); ovs_flow_tbl_put(table);
v4:
- Further split patches: move lockdep checks to an independent patch (Paolo)
- Add an additional patch to reduce the nested rcu callback scheduling (Paolo)
- consolidate flow_table_put after if/else (Paolo)
v3:
- Split in 2 patches (Paolo)
- Improve locking in get_dp_stats (Paolo and Sashiko)
- Use __always_unused in lockdep stubs (Paolo)
- Use READ_ONCE/WRITE_ONCE for table->count (Aaron)
- Take a reference in ovs_dp_masks_rebalance (Aaron)
v2: Fix argument in ovs_flow_tbl_put (sparse)
Remove rcu checks in ovs_dp_masks_rebalance
Adrian Moreno (4):
net: openvswitch: make flow_table an rcu pointer
net: openvswitch: add per-flow_table lockdep checks
net: openvswitch: decouple flow_table from ovs_mutex
net: openvswitch: avoid double-rcu wait period
net/openvswitch/datapath.c | 294 ++++++++++++++++++++++++----------- net/openvswitch/datapath.h | 2 +- net/openvswitch/flow.c | 13 +- net/openvswitch/flow.h | 9 +- net/openvswitch/flow_table.c | 211 ++++++++++++++++--------- net/openvswitch/flow_table.h | 56 ++++++- 6 files changed, 406 insertions(+), 179 deletions(-)
Signature · 1 line
2.54.0
PATCH v4 1/4 net: openvswitch: make flow_table an rcu pointer
This patch turns "flow_table" from being embedded into "datapath" to being an rcu protected pointer. No functional change intended.
--- net/openvswitch/datapath.c | 113 ++++++++++++++++++++++++++--------- net/openvswitch/datapath.h | 2 +- net/openvswitch/flow_table.c | 23 ++++--- net/openvswitch/flow_table.h | 5 +- 4 files changed, 105 insertions(+), 38 deletions(-)
net/openvswitch/datapath.c +86 -27
@@ -166,7 +166,6 @@ static void destroy_dp_rcu(struct rcu_head *rcu) { struct datapath *dp = container_of(rcu, struct datapath, rcu); -ovs_flow_tbl_destroy (&dp->table);free_percpu (dp->stats_percpu);kfree (dp->ports);ovs_meters_exit (dp);
@@ -247,6 +246,7 @@ void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)struct ovs_pcpu_storage *ovs_pcpu =this_cpu_ptr (ovs_pcpu_storage);const struct vport *p =OVS_CB (skb)->input_vport;struct datapath *dp = p->dp; +struct flow_table *table;struct sw_flow *flow;struct sw_flow_actions *sf_acts;struct dp_stats_percpu *stats;
@@ -257,9 +257,16 @@ void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)int error; stats =this_cpu_ptr (dp->stats_percpu); + table =rcu_dereference (dp->table); +if (!table) { +net_dbg_ratelimited ("ovs: no flow table on datapath %s\n" , +ovs_dp_name (dp)); +kfree_skb (skb); +return ; + }/* Look up flow. */ - flow =ovs_flow_tbl_lookup_stats (&dp->table, key,skb_get_hash (skb), + flow =ovs_flow_tbl_lookup_stats (table, key,skb_get_hash (skb), &n_mask_hit, &n_cache_hit);if (unlikely (!flow)) {struct dp_upcall_info upcall;
@@ -752,12 +759,16 @@ static struct genl_family dp_packet_genl_family __ro_after_init = {static void get_dp_stats (const struct datapath *dp,struct ovs_dp_stats *stats,struct ovs_dp_megaflow_stats *mega_stats) { +struct flow_table *table =ovsl_dereference (dp->table);int i;memset (mega_stats,0 ,sizeof (*mega_stats)); +memset (stats,0 ,sizeof (*stats)); - stats->n_flows =ovs_flow_tbl_count (&dp->table); - mega_stats->n_masks =ovs_flow_tbl_num_masks (&dp->table); +if (table) { + stats->n_flows =ovs_flow_tbl_count (table); + mega_stats->n_masks =ovs_flow_tbl_num_masks (table); + } stats->n_hit = stats->n_missed = stats->n_lost =0 ;
@@ -998,6 +1009,7 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)struct nlattr **a = info->attrs;struct ovs_header *ovs_header =genl_info_userhdr (info);struct sw_flow *flow =NULL , *new_flow; +struct flow_table *table;struct sw_flow_mask mask;struct sk_buff *reply;struct datapath *dp;
@@ -1070,17 +1082,22 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) error = -ENODEV ; goto err_unlock_ovs; } + table =ovsl_dereference (dp->table); +if (!table) { + error = -ENODEV ; + goto err_unlock_ovs; + }/* Check if this is a duplicate flow */ if (ovs_identifier_is_ufid (&new_flow->id)) - flow =ovs_flow_tbl_lookup_ufid (&dp->table, &new_flow->id); + flow =ovs_flow_tbl_lookup_ufid (table, &new_flow->id);if (!flow) - flow =ovs_flow_tbl_lookup (&dp->table, key); + flow =ovs_flow_tbl_lookup (table, key);if (likely (!flow)) {rcu_assign_pointer (new_flow->sf_acts, acts);/* Put flow in bucket. */ - error =ovs_flow_tbl_insert (&dp->table, new_flow, &mask); + error =ovs_flow_tbl_insert (table, new_flow, &mask);if (unlikely (error)) { acts =NULL ; goto err_unlock_ovs;
@@ -1115,7 +1132,7 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)*/ if (unlikely (!ovs_flow_cmp (flow, &match))) {if (ovs_identifier_is_key (&flow->id)) - flow =ovs_flow_tbl_lookup_exact (&dp->table, + flow =ovs_flow_tbl_lookup_exact (table, &match);else /* UFID matches but key is different */ flow =NULL ;
@@ -1244,6 +1261,7 @@ static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)struct net *net =sock_net (skb->sk);struct nlattr **a = info->attrs;struct ovs_header *ovs_header =genl_info_userhdr (info); +struct flow_table *table;struct sw_flow_key key;struct sw_flow *flow;struct sk_buff *reply =NULL ;
@@ -1284,11 +1302,16 @@ static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info) error = -ENODEV ; goto err_unlock_ovs; } + table =ovsl_dereference (dp->table); +if (!table) { + error = -ENODEV ; + goto err_unlock_ovs; + }/* Check that the flow exists. */ if (ufid_present) - flow =ovs_flow_tbl_lookup_ufid (&dp->table, &sfid); + flow =ovs_flow_tbl_lookup_ufid (table, &sfid);else - flow =ovs_flow_tbl_lookup_exact (&dp->table, &match); + flow =ovs_flow_tbl_lookup_exact (table, &match);if (unlikely (!flow)) { error = -ENOENT ; goto err_unlock_ovs;
@@ -1346,6 +1369,7 @@ static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)struct nlattr **a = info->attrs;struct ovs_header *ovs_header =genl_info_userhdr (info);struct net *net =sock_net (skb->sk); +struct flow_table *table;struct sw_flow_key key;struct sk_buff *reply;struct sw_flow *flow;
@@ -1376,11 +1400,16 @@ static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info) err = -ENODEV ; goto unlock; } + table =ovsl_dereference (dp->table); +if (!table) { + err = -ENODEV ; + goto unlock; + }if (ufid_present) - flow =ovs_flow_tbl_lookup_ufid (&dp->table, &ufid); + flow =ovs_flow_tbl_lookup_ufid (table, &ufid);else - flow =ovs_flow_tbl_lookup_exact (&dp->table, &match); + flow =ovs_flow_tbl_lookup_exact (table, &match);if (!flow) { err = -ENOENT ; goto unlock;
@@ -1405,6 +1434,7 @@ static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)struct nlattr **a = info->attrs;struct ovs_header *ovs_header =genl_info_userhdr (info);struct net *net =sock_net (skb->sk); +struct flow_table *table;struct sw_flow_key key;struct sk_buff *reply;struct sw_flow *flow =NULL ;
@@ -1431,22 +1461,27 @@ static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info) err = -ENODEV ; goto unlock; } + table =ovsl_dereference (dp->table); +if (!table) { + err = -ENODEV ; + goto unlock; + }if (unlikely (!a[OVS_FLOW_ATTR_KEY ] && !ufid_present)) { - err =ovs_flow_tbl_flush (&dp->table); + err =ovs_flow_tbl_flush (table); goto unlock; }if (ufid_present) - flow =ovs_flow_tbl_lookup_ufid (&dp->table, &ufid); + flow =ovs_flow_tbl_lookup_ufid (table, &ufid);else - flow =ovs_flow_tbl_lookup_exact (&dp->table, &match); + flow =ovs_flow_tbl_lookup_exact (table, &match);if (unlikely (!flow)) { err = -ENOENT ; goto unlock; } -ovs_flow_tbl_remove (&dp->table, flow); +ovs_flow_tbl_remove (table, flow);ovs_unlock (); reply = ovs_flow_cmd_alloc_info((const struct sw_flow_actions __force *) flow->sf_acts,
@@ -1485,6 +1520,7 @@ static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)struct nlattr *a[__OVS_FLOW_ATTR_MAX];struct ovs_header *ovs_header =genlmsg_data (nlmsg_data (cb->nlh));struct table_instance *ti; +struct flow_table *table;struct datapath *dp;u32 ufid_flags;int err;
@@ -1501,8 +1537,13 @@ static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)rcu_read_unlock ();return -ENODEV ; } + table =rcu_dereference_ovsl (dp->table); +if (!table) { +rcu_read_unlock (); +return -ENODEV ; + } - ti =rcu_dereference (dp->table.ti); + ti =rcu_dereference (table->ti);for (;;) {struct sw_flow *flow;u32 bucket, obj;
@@ -1598,8 +1639,13 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,struct ovs_dp_stats dp_stats;struct ovs_dp_megaflow_stats dp_megaflow_stats;struct dp_nlsk_pids *pids =ovsl_dereference (dp->upcall_portids); +struct flow_table *table;int err, pids_len; + table =ovsl_dereference (dp->table); +if (!table) +return -ENODEV ; + ovs_header =genlmsg_put (skb, portid, seq, &dp_datapath_genl_family, flags, cmd);if (!ovs_header)
@@ -1625,7 +1671,7 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb, goto nla_put_failure;if (nla_put_u32 (skb,OVS_DP_ATTR_MASKS_CACHE_SIZE , -ovs_flow_tbl_masks_cache_size (&dp->table))) +ovs_flow_tbl_masks_cache_size (table))) goto nla_put_failure;if (dp->user_features &OVS_DP_F_DISPATCH_UPCALL_PER_CPU && pids) {
@@ -1736,6 +1782,7 @@ u32 ovs_dp_get_upcall_portid(const struct datapath *dp, uint32_t cpu_id)static int ovs_dp_change (struct datapath *dp,struct nlattr *a[]) {u32 user_features =0 , old_features = dp->user_features; +struct flow_table *table;int err;if (a[OVS_DP_ATTR_USER_FEATURES ]) {
@@ -1757,8 +1804,12 @@ static int ovs_dp_change(struct datapath *dp, struct nlattr *a[])int err;u32 cache_size; + table =ovsl_dereference (dp->table); +if (!table) +return -ENODEV ; + cache_size =nla_get_u32 (a[OVS_DP_ATTR_MASKS_CACHE_SIZE ]); - err =ovs_flow_tbl_masks_cache_resize (&dp->table, cache_size); + err =ovs_flow_tbl_masks_cache_resize (table, cache_size);if (err)return err; }
@@ -1810,6 +1861,7 @@ static int ovs_dp_vport_init(struct datapath *dp)static int ovs_dp_cmd_new (struct sk_buff *skb,struct genl_info *info) {struct nlattr **a = info->attrs; +struct flow_table *table;struct vport_parms parms;struct sk_buff *reply;struct datapath *dp;
@@ -1833,9 +1885,12 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)ovs_dp_set_net (dp,sock_net (skb->sk));/* Allocate table. */ - err =ovs_flow_tbl_init (&dp->table); -if (err) + table =ovs_flow_tbl_alloc (); +if (IS_ERR (table)) { + err =PTR_ERR (table); goto err_destroy_dp; + } +rcu_assign_pointer (dp->table, table); err =ovs_dp_stats_init (dp);if (err)
@@ -1905,7 +1960,7 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info) err_destroy_stats:free_percpu (dp->stats_percpu); err_destroy_table: -ovs_flow_tbl_destroy (&dp->table); +call_rcu (&table->rcu, ovs_flow_tbl_destroy_rcu); err_destroy_dp:kfree (dp); err_destroy_reply:
@@ -1917,7 +1972,7 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)/* Called with ovs_mutex. */ static void __dp_destroy (struct datapath *dp) { -struct flow_table *table = &dp->table; +struct flow_table *table =ovsl_dereference (dp->table);int i;if (dp->user_features &OVS_DP_F_TC_RECIRC_SHARING )
@@ -1948,6 +2003,7 @@ static void __dp_destroy(struct datapath *dp)/* RCU destroy the ports, meters and flow tables. */ call_rcu (&dp->rcu, destroy_dp_rcu); +call_rcu (&table->rcu, ovs_flow_tbl_destroy_rcu); }static int ovs_dp_cmd_del (struct sk_buff *skb,struct genl_info *info)
@@ -2585,13 +2641,16 @@ static void ovs_dp_masks_rebalance(struct work_struct *work) { struct ovs_net *ovs_net = container_of(work, struct ovs_net, masks_rebalance.work); +struct flow_table *table;struct datapath *dp;ovs_lock (); - -list_for_each_entry (dp, &ovs_net->dps, list_node) -ovs_flow_masks_rebalance (&dp->table); - +list_for_each_entry (dp, &ovs_net->dps, list_node) { + table =ovsl_dereference (dp->table); +if (!table) +continue ; +ovs_flow_masks_rebalance (table); + }ovs_unlock (); schedule_delayed_work(&ovs_net->masks_rebalance,
net/openvswitch/datapath.h +1 -1
@@ -90,7 +90,7 @@ struct datapath {struct list_head list_node;/* Flow table. */ -struct flow_table table; +struct flow_table __rcu *table;/* Switch ports. */ struct hlist_head *ports;
net/openvswitch/flow_table.c +15 -8
@@ -406,15 +406,19 @@ int ovs_flow_tbl_masks_cache_resize(struct flow_table *table, u32 size)return 0 ; } -int ovs_flow_tbl_init (struct flow_table *table) +struct flow_table *ovs_flow_tbl_alloc (void ) {struct table_instance *ti, *ufid_ti; +struct flow_table *table;struct mask_cache *mc;struct mask_array *ma; + table =kzalloc_obj (*table,GFP_KERNEL ); +if (!table) +return ERR_PTR (-ENOMEM ); mc =tbl_mask_cache_alloc (MC_DEFAULT_HASH_ENTRIES );if (!mc) -return -ENOMEM ; + goto free_table; ma =tbl_mask_array_alloc (MASK_ARRAY_SIZE_MIN );if (!ma)
@@ -435,7 +439,7 @@ int ovs_flow_tbl_init(struct flow_table *table) table->last_rehash = jiffies; table->count =0 ; table->ufid_count =0 ; -return 0 ; +return table; free_ti:__table_instance_destroy (ti);
@@ -443,7 +447,9 @@ int ovs_flow_tbl_init(struct flow_table *table)__mask_array_destroy (ma); free_mask_cache:__mask_cache_destroy (mc); -return -ENOMEM ; +free_table: +kfree (table); +return ERR_PTR (-ENOMEM ); }static void flow_tbl_destroy_rcu_cb (struct rcu_head *rcu)
@@ -505,11 +511,11 @@ static void table_instance_destroy(struct table_instance *ti,call_rcu (&ufid_ti->rcu, flow_tbl_destroy_rcu_cb); } -/* No need for locking this function is called from RCU callback or -* error path. -*/ -void ovs_flow_tbl_destroy (struct flow_table *table) +/* No need for locking this function is called from RCU callback. */ +void ovs_flow_tbl_destroy_rcu (struct rcu_head *rcu) { + struct flow_table *table = container_of(rcu, struct flow_table, rcu); +struct table_instance *ti =rcu_dereference_raw (table->ti);struct table_instance *ufid_ti =rcu_dereference_raw (table->ufid_ti);struct mask_cache *mc =rcu_dereference_raw (table->mask_cache);
@@ -518,6 +524,7 @@ void ovs_flow_tbl_destroy(struct flow_table *table)call_rcu (&mc->rcu, mask_cache_rcu_cb);call_rcu (&ma->rcu, mask_array_rcu_cb);table_instance_destroy (ti, ufid_ti); +kfree (table); } struct sw_flow *ovs_flow_tbl_dump_next(struct table_instance *ti,
net/openvswitch/flow_table.h +3 -2
@@ -60,6 +60,7 @@ struct table_instance { };struct flow_table { +struct rcu_head rcu;struct table_instance __rcu *ti;struct table_instance __rcu *ufid_ti;struct mask_cache __rcu *mask_cache;
@@ -77,9 +78,9 @@ void ovs_flow_exit(void);struct sw_flow *ovs_flow_alloc (void );void ovs_flow_free (struct sw_flow *,bool deferred); -int ovs_flow_tbl_init (struct flow_table *); +struct flow_table *ovs_flow_tbl_alloc (void ); +void ovs_flow_tbl_destroy_rcu (struct rcu_head *table);int ovs_flow_tbl_count (const struct flow_table *table); -void ovs_flow_tbl_destroy (struct flow_table *table);int ovs_flow_tbl_flush (struct flow_table *flow_table); int ovs_flow_tbl_insert(struct flow_table *table, struct sw_flow *flow,
Signature · 1 line
2.54.0
Eelco Chaudron
maintainerRe: [PATCH net-next v4 1/4] net: openvswitch: make flow_table an rcu pointer
Quoted 4 lines · Adrian Moreno
This patch turns "
flow_table" from being embedded into "datapath" to being an rcu protected pointer. No functional change intended.Signed-off-by Adrian Moreno <[email protected]>
Hi Adrian,
Thanks for the series. I need some time to take a look, as this is the first time I reviewed this series. I'll probably reply to the other patches early next week.
Some small comments below.
Cheers,
Eelco
[...]
Quoted 1 line
net/openvswitch/datapath.c +0
-0
[...]
Quoted 6 lines
net/openvswitch/datapath.c +1
-0@@ -752,12 +759,16 @@ static struct genl_family dp_packet_genl_family __ro_after_init = {static void get_dp_stats (const struct datapath *dp,struct ovs_dp_stats *stats,struct ovs_dp_megaflow_stats *mega_stats) { +struct flow_table *table =ovsl_dereference (dp->table);int i;
net/openvswitch/datapath.c static struct genl_family dp_packet_genl_family __ro_after_init = { · line 771
Quoted 10 lines
net/openvswitch/datapath.c +5
-2memset (mega_stats,0 ,sizeof (*mega_stats)); +memset (stats,0 ,sizeof (*stats)); - stats->n_flows =ovs_flow_tbl_count (&dp->table); - mega_stats->n_masks =ovs_flow_tbl_num_masks (&dp->table); +if (table) { + stats->n_flows =ovs_flow_tbl_count (table); + mega_stats->n_masks =ovs_flow_tbl_num_masks (table); + }
The memset above seems like an overkill, why not do this:
if (table) {
stats->n_flows = ovs_flow_tbl_count(table);
mega_stats->n_masks = ovs_flow_tbl_num_masks(table);
} else {
stats->n_flows = 0;
}Quoted 2 lines
net/openvswitch/datapath.c +0
-0stats->n_hit = stats->n_missed = stats->n_lost =0 ;
[...]
net/openvswitch/datapath.c static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb, · line 1647
Quoted 10 lines
net/openvswitch/datapath.c +4
-0@@ -1598,8 +1639,13 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,struct ovs_dp_stats dp_stats;struct ovs_dp_megaflow_stats dp_megaflow_stats;struct dp_nlsk_pids *pids =ovsl_dereference (dp->upcall_portids); +struct flow_table *table;int err, pids_len; + table =ovsl_dereference (dp->table); +if (!table) +return -ENODEV ;
ovs_dp_cmd_fill_info() can now return -ENODEV when table is NULL, but every caller immediately follows with:
BUG_ON(err < 0);
so this would panic the kernel. I guess you fix this in the 3rd patch, but just mentioning this as patch 1 is unsafe to apply in isolation.
Quoted 3 lines
net/openvswitch/datapath.c +0
-0ovs_header =genlmsg_put (skb, portid, seq, &dp_datapath_genl_family, flags, cmd);if (!ovs_header)
[...]
net/openvswitch/datapath.c static void __dp_destroy(struct datapath *dp) · line 2005
Quoted 13 lines
net/openvswitch/datapath.c +1
-1@@ -1917,7 +1972,7 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)/* Called with ovs_mutex. */ static void __dp_destroy (struct datapath *dp) { -struct flow_table *table = &dp->table; +struct flow_table *table =ovsl_dereference (dp->table);int i;if (dp->user_features &OVS_DP_F_TC_RECIRC_SHARING )@@ -1948,6 +2003,7 @@ static void __dp_destroy(struct datapath *dp)/* RCU destroy the ports, meters and flow tables. */ call_rcu (&dp->rcu, destroy_dp_rcu);
In theory the table pointer cannot be NULL here, but would it make sense to add a WARN_ON(!table) to catch any future mistakes?
Quoted 3 lines
net/openvswitch/datapath.c +1
-0+call_rcu (&table->rcu, ovs_flow_tbl_destroy_rcu); }
[...]
Quoted 9 lines
net/openvswitch/flow_table.c +0
-1@@ -406,15 +406,19 @@ int ovs_flow_tbl_masks_cache_resize(struct flow_table *table, u32 size)return 0 ; } -int ovs_flow_tbl_init (struct flow_table *table)
net/openvswitch/flow_table.c int ovs_flow_tbl_masks_cache_resize(struct flow_table *table, u32 size) · line 418
Quoted 10 lines
net/openvswitch/flow_table.c +5
-0+struct flow_table *ovs_flow_tbl_alloc (void ) {struct table_instance *ti, *ufid_ti; +struct flow_table *table;struct mask_cache *mc;struct mask_array *ma; + table =kzalloc_obj (*table,GFP_KERNEL ); +if (!table) +return ERR_PTR (-ENOMEM );
Add an extra blank line?
Quoted 25 lines
net/openvswitch/flow_table.c +5
-3mc =tbl_mask_cache_alloc (MC_DEFAULT_HASH_ENTRIES );if (!mc) -return -ENOMEM ; + goto free_table; ma =tbl_mask_array_alloc (MASK_ARRAY_SIZE_MIN );if (!ma)@@ -435,7 +439,7 @@ int ovs_flow_tbl_init(struct flow_table *table) table->last_rehash = jiffies; table->count =0 ; table->ufid_count =0 ; -return 0 ; +return table; free_ti:__table_instance_destroy (ti);@@ -443,7 +447,9 @@ int ovs_flow_tbl_init(struct flow_table *table)__mask_array_destroy (ma); free_mask_cache:__mask_cache_destroy (mc); -return -ENOMEM ; +free_table: +kfree (table); +return ERR_PTR (-ENOMEM ); }
[...]
PATCH v4 2/4 net: openvswitch: add per-flow_table lockdep checks
In reply to Adrian MorenoA future patch will introduce a per-flow_table mutex that will protect flow operations independently. In preparation for that, this patch introduces a flow_table lockdep macro, and modifies some function signatures to allow lockdep assertions to run.
For now, the actual lockdep check logic is a no-op, but adding the infrastructure helps reduce the size of the upcoming patch.
--- net/openvswitch/datapath.c | 57 +++++++++------- net/openvswitch/flow.c | 13 ++-- net/openvswitch/flow.h | 9 ++- net/openvswitch/flow_table.c | 127 ++++++++++++++++++++--------------- net/openvswitch/flow_table.h | 16 +++++ 5 files changed, 137 insertions(+), 85 deletions(-)
net/openvswitch/datapath.c +33 -24
@@ -840,15 +840,16 @@ static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts, +nla_total_size_64bit (8 );/* OVS_FLOW_ATTR_USED */ } -/* Called with ovs_mutex or RCU read lock. */ +/* Called with table->lock or RCU read lock. */ static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow, + const struct flow_table *table, struct sk_buff *skb) { struct ovs_flow_stats stats; __be16 tcp_flags; unsigned long used; - ovs_flow_stats_get(flow, &stats, &used, &tcp_flags); + ovs_flow_stats_get(flow, table, &stats, &used, &tcp_flags); if (used && nla_put_u64_64bit(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used),
@@ -868,8 +869,9 @@ static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,return 0 ; } -/* Called with ovs_mutex or RCU read lock. */ +/* Called with RCU read lock or table->lock held. */ static int ovs_flow_cmd_fill_actions (const struct sw_flow *flow, +const struct flow_table *table,struct sk_buff *skb,int skb_orig_len) {struct nlattr *start;
@@ -889,7 +891,7 @@ static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,if (start) {const struct sw_flow_actions *sf_acts; - sf_acts =rcu_dereference_ovsl (flow->sf_acts); + sf_acts =rcu_dereference_ovs_tbl (flow->sf_acts, table); err =ovs_nla_put_actions (sf_acts->actions, sf_acts->actions_len, skb);
@@ -908,8 +910,10 @@ static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,return 0 ; } -/* Called with ovs_mutex or RCU read lock. */ -static int ovs_flow_cmd_fill_info (const struct sw_flow *flow,int dp_ifindex, +/* Called with table->lock or RCU read lock. */ +static int ovs_flow_cmd_fill_info (const struct sw_flow *flow, +const struct flow_table *table, +int dp_ifindex,struct sk_buff *skb,u32 portid,u32 seq,u32 flags,u8 cmd,u32 ufid_flags) {
@@ -940,12 +944,12 @@ static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex, goto error; } - err =ovs_flow_cmd_fill_stats (flow, skb); + err =ovs_flow_cmd_fill_stats (flow, table, skb);if (err) goto error;if (should_fill_actions (ufid_flags)) { - err =ovs_flow_cmd_fill_actions (flow, skb, skb_orig_len); + err =ovs_flow_cmd_fill_actions (flow, table, skb, skb_orig_len);if (err) goto error; }
@@ -979,8 +983,9 @@ static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *actreturn skb; } -/* Called with ovs_mutex. */ +/* Called with table->lock. */ static struct sk_buff *ovs_flow_cmd_build_info (const struct sw_flow *flow, +const struct flow_table *table,int dp_ifindex,struct genl_info *info,u8 cmd,bool always,u32 ufid_flags)
@@ -988,12 +993,12 @@ static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,struct sk_buff *skb;int retval; - skb =ovs_flow_cmd_alloc_info (ovsl_dereference (flow->sf_acts), + skb =ovs_flow_cmd_alloc_info (ovs_tbl_dereference (flow->sf_acts, table), &flow->id, info, always, ufid_flags);if (IS_ERR_OR_NULL (skb))return skb; - retval =ovs_flow_cmd_fill_info (flow, dp_ifindex, skb, + retval =ovs_flow_cmd_fill_info (flow, table, dp_ifindex, skb, info->snd_portid, info->snd_seq,0 , cmd, ufid_flags);if (WARN_ON_ONCE (retval <0 )) {
@@ -1104,7 +1109,7 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) }if (unlikely (reply)) { - error = ovs_flow_cmd_fill_info(new_flow, + error = ovs_flow_cmd_fill_info(new_flow, table, ovs_header->dp_ifindex, reply, info->snd_portid, info->snd_seq, 0,
@@ -1142,11 +1147,11 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) } }/* Update actions. */ - old_acts =ovsl_dereference (flow->sf_acts); + old_acts =ovs_tbl_dereference (flow->sf_acts, table);rcu_assign_pointer (flow->sf_acts, acts);if (unlikely (reply)) { - error = ovs_flow_cmd_fill_info(flow, + error = ovs_flow_cmd_fill_info(flow, table, ovs_header->dp_ifindex, reply, info->snd_portid, info->snd_seq, 0,
@@ -1319,11 +1324,11 @@ static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)/* Update actions, if present. */ if (likely (acts)) { - old_acts =ovsl_dereference (flow->sf_acts); + old_acts =ovs_tbl_dereference (flow->sf_acts, table);rcu_assign_pointer (flow->sf_acts, acts);if (unlikely (reply)) { - error = ovs_flow_cmd_fill_info(flow, + error = ovs_flow_cmd_fill_info(flow, table, ovs_header->dp_ifindex, reply, info->snd_portid, info->snd_seq, 0,
@@ -1333,7 +1338,8 @@ static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info) } }else {/* Could not alloc without acts before locking. */ - reply =ovs_flow_cmd_build_info (flow, ovs_header->dp_ifindex, + reply =ovs_flow_cmd_build_info (flow, table, + ovs_header->dp_ifindex, info,OVS_FLOW_CMD_SET , false, ufid_flags);
@@ -1345,7 +1351,7 @@ static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)/* Clear stats. */ if (a[OVS_FLOW_ATTR_CLEAR ]) -ovs_flow_stats_clear (flow); +ovs_flow_stats_clear (flow, table);ovs_unlock ();if (reply)
@@ -1415,8 +1421,9 @@ static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info) goto unlock; } - reply =ovs_flow_cmd_build_info (flow, ovs_header->dp_ifindex, info, -OVS_FLOW_CMD_GET , true, ufid_flags); + reply =ovs_flow_cmd_build_info (flow, table, ovs_header->dp_ifindex, + info,OVS_FLOW_CMD_GET , true, + ufid_flags);if (IS_ERR (reply)) { err =PTR_ERR (reply); goto unlock;
@@ -1489,7 +1496,8 @@ static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)if (likely (reply)) {if (!IS_ERR (reply)) {rcu_read_lock ();/*To keep RCU checker happy. */ - err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, + err = ovs_flow_cmd_fill_info(flow, table, + ovs_header->dp_ifindex, reply, info->snd_portid, info->snd_seq, 0, OVS_FLOW_CMD_DEL,
@@ -1554,8 +1562,8 @@ static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)if (!flow)break ; -if (ovs_flow_cmd_fill_info (flow, ovs_header->dp_ifindex, skb, -NETLINK_CB (cb->skb).portid, +if (ovs_flow_cmd_fill_info (flow, table, ovs_header->dp_ifindex, + skb,NETLINK_CB (cb->skb).portid, cb->nlh->nlmsg_seq,NLM_F_MULTI ,OVS_FLOW_CMD_GET , ufid_flags) <0 )break ;
@@ -1972,7 +1980,8 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)/* Called with ovs_mutex. */ static void __dp_destroy (struct datapath *dp) { -struct flow_table *table =ovsl_dereference (dp->table); +struct flow_table *table =rcu_dereference_protected (dp->table, +lockdep_ovsl_is_held ());int i;if (dp->user_features &OVS_DP_F_TC_RECIRC_SHARING )
net/openvswitch/flow.c +8 -5
@@ -124,8 +124,9 @@ void ovs_flow_stats_update(struct sw_flow *flow, __be16 tcp_flags,spin_unlock (&stats->lock); } -/* Must be called with rcu_read_lock or ovs_mutex. */ +/* Must be called with rcu_read_lock or table->lock held. */ void ovs_flow_stats_get (const struct sw_flow *flow, +const struct flow_table *table,struct ovs_flow_stats *ovs_stats,unsigned long *used,__be16 *tcp_flags) {
@@ -136,7 +137,8 @@ void ovs_flow_stats_get(const struct sw_flow *flow,memset (ovs_stats,0 ,sizeof (*ovs_stats));for_each_cpu (cpu, flow->cpu_used_mask) { -struct sw_flow_stats *stats =rcu_dereference_ovsl (flow->stats[cpu]); +struct sw_flow_stats *stats = +rcu_dereference_ovs_tbl (flow->stats[cpu], table);if (stats) {/* Local CPU may write on non-local stats, so we must
@@ -153,13 +155,14 @@ void ovs_flow_stats_get(const struct sw_flow *flow, } } -/* Called with ovs_mutex. */ -void ovs_flow_stats_clear (struct sw_flow *flow) +/* Called with table->lock held. */ +void ovs_flow_stats_clear (struct sw_flow *flow,struct flow_table *table) {unsigned int cpu;for_each_cpu (cpu, flow->cpu_used_mask) { -struct sw_flow_stats *stats =ovsl_dereference (flow->stats[cpu]); +struct sw_flow_stats *stats = +ovs_tbl_dereference (flow->stats[cpu], table);if (stats) {spin_lock_bh (&stats->lock);
net/openvswitch/flow.h +6 -3
@@ -23,6 +23,7 @@#include <net/dst_metadata.h> #include <net/nsh.h> +struct flow_table ;struct sk_buff ;enum sw_flow_mac_proto {
@@ -280,9 +281,11 @@ static inline bool ovs_identifier_is_key(const struct sw_flow_id *sfid)void ovs_flow_stats_update (struct sw_flow *,__be16 tcp_flags,const struct sk_buff *); -void ovs_flow_stats_get (const struct sw_flow *,struct ovs_flow_stats *, -unsigned long *used,__be16 *tcp_flags); -void ovs_flow_stats_clear (struct sw_flow *); +void ovs_flow_stats_get (const struct sw_flow *flow, +const struct flow_table *table, +struct ovs_flow_stats *stats,unsigned long *used, +__be16 *tcp_flags); +void ovs_flow_stats_clear (struct sw_flow *flow,struct flow_table *table);u64 ovs_flow_used_time (unsigned long flow_jiffies);int ovs_flow_key_update (struct sk_buff *skb,struct sw_flow_key *key);
net/openvswitch/flow_table.c +74 -53
377 lines not on this page · raw message
net/openvswitch/flow_table.h +16 -0
@@ -72,6 +72,22 @@ struct flow_table {extern struct kmem_cache *flow_stats_cache; +static inline int lockdep_ovs_tbl_is_held (const struct flow_table *table +__always_unused ) +{ +return 1 ; +} + +#define ASSERT_OVS_TBL (tbl) WARN_ON(!lockdep_ovs_tbl_is_held(tbl)) + +/* Lock-protected update-allowed dereferences.*/ +#define ovs_tbl_dereference (p, tbl) \ + rcu_dereference_protected(p, lockdep_ovs_tbl_is_held(tbl)) + +/* Read dereferences can be protected by either RCU, table lock. */ +#define rcu_dereference_ovs_tbl (p, tbl) \ + rcu_dereference_check(p, lockdep_ovs_tbl_is_held(tbl)) +int ovs_flow_init (void );void ovs_flow_exit (void );
Signature · 1 line
2.54.0
Eelco Chaudron
maintainerRe: [PATCH net-next v4 2/4] net: openvswitch: add per-flow_table lockdep checks
Quoted 9 lines · Adrian Moreno
A future patch will introduce a per-flow_table mutex that will protect flow operations independently. In preparation for that, this patch introduces a
flow_tablelockdep macro, and modifies some function signatures to allow lockdep assertions to run.For now, the actual lockdep check logic is a no-op, but adding the infrastructure helps reduce the size of the upcoming patch.
Signed-off-by Adrian Moreno <[email protected]>
Hi Adrian,
See some comments below, and maybe address the Sashiko comment in the commit message.
Cheers,
Eelco
[...]
net/openvswitch/datapath.c static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts, · line 845
Quoted 12 lines
net/openvswitch/datapath.c +2
-1@@ -840,15 +840,16 @@ static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts, +nla_total_size_64bit (8 );/* OVS_FLOW_ATTR_USED */ } -/* Called with ovs_mutex or RCU read lock. */ +/* Called with table->lock or RCU read lock. */ static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow, + const struct flow_table *table,
Should 'table' come before 'flow' to be consistent with ovs_flow_tbl_insert() and ovs_flow_tbl_remove()? This applies to all functions in this patch adding the 'table' parameter.
Quoted 5 lines
net/openvswitch/datapath.c +0
-0struct sk_buff *skb) {struct ovs_flow_stats stats;__be16 tcp_flags;unsigned long used;
[...]
Quoted 11 lines
net/openvswitch/flow_table.h +3
-0@@ -72,6 +72,22 @@ struct flow_table {extern struct kmem_cache *flow_stats_cache; +static inline int lockdep_ovs_tbl_is_held (const struct flow_table *table +__always_unused ) +{
net/openvswitch/flow_table.h struct flow_table { · line 87
Quoted 10 lines
net/openvswitch/flow_table.h +10
-0+return 1 ; +} + +#define ASSERT_OVS_TBL (tbl) WARN_ON(!lockdep_ovs_tbl_is_held(tbl)) + +/* Lock-protected update-allowed dereferences.*/ +#define ovs_tbl_dereference (p, tbl) \ + rcu_dereference_protected(p, lockdep_ovs_tbl_is_held(tbl)) + +/* Read dereferences can be protected by either RCU, table lock. */
nit; This comment reads odd, maybe;
/* Read dereferences can be protected by either RCU or table lock. */
Quoted 5 lines
net/openvswitch/flow_table.h +3
-0+#define rcu_dereference_ovs_tbl (p, tbl) \ + rcu_dereference_check(p, lockdep_ovs_tbl_is_held(tbl)) +int ovs_flow_init (void );void ovs_flow_exit (void );
PATCH v4 3/4 net: openvswitch: decouple flow_table from ovs_mutex
In reply to Adrian MorenoIn order to protect flow operations from RTNL contention, this patch decouples flow_table modifications from ovs_mutex by means of the following:
1 - Create a new mutex inside the flow_table that protects it from concurrent modifications.
Putting the mutex inside flow_table makes it easier to consume for functions inside flow_table.c that do not currently take pointers to the datapath.
Some function signatures need to be changed to accept flow_table so that lockdep checks can be performed.
2 - Create a reference count to temporarily extend rcu protection from the datapath to the flow_table.
One reference is held by the datapath, the other is temporarily increased during flow modifications.
--- net/openvswitch/datapath.c | 174 ++++++++++++++++++++++------------- net/openvswitch/flow_table.c | 46 +++++++-- net/openvswitch/flow_table.h | 37 +++++++- 3 files changed, 181 insertions(+), 76 deletions(-)
net/openvswitch/datapath.c +111 -63
@@ -88,13 +88,17 @@ static void ovs_notify(struct genl_family *family,* DOC: Locking: * * All writes e.g. Writes to device state (add/remove datapath, port, set -* operations on vports, etc.), Writes to other state (flow table -* modifications, set miscellaneous datapath parameters, etc.) are protected -* by ovs_lock. +* operations on vports, etc.) and writes to other datapath parameters +* are protected by ovs_lock. +* +* Writes to the flow table are NOT protected by ovs_lock. Instead, a per-table +* mutex and reference count are used (see comment above "struct flow_table" +* definition). On some few occasions, the per-flow table mutex is nested +* inside ovs_mutex. * * Reads are protected by RCU. * -* There are a few special cases (mostly stats) that have their own +* There are a few other special cases (mostly stats) that have their own * synchronization but they nest under all of above and don't interact with * each other. *
@@ -759,16 +763,19 @@ static struct genl_family dp_packet_genl_family __ro_after_init = {static void get_dp_stats (const struct datapath *dp,struct ovs_dp_stats *stats,struct ovs_dp_megaflow_stats *mega_stats) { -struct flow_table *table =ovsl_dereference (dp->table); +struct flow_table *table;int i;memset (mega_stats,0 ,sizeof (*mega_stats));memset (stats,0 ,sizeof (*stats)); +rcu_read_lock (); + table =rcu_dereference (dp->table);if (table) { stats->n_flows =ovs_flow_tbl_count (table); mega_stats->n_masks =ovs_flow_tbl_num_masks (table); } +rcu_read_unlock (); stats->n_hit = stats->n_missed = stats->n_lost =0 ;
@@ -1081,17 +1088,25 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) goto err_kfree_acts; } -ovs_lock (); +rcu_read_lock (); dp =get_dp (net, ovs_header->dp_ifindex);if (unlikely (!dp)) { error = -ENODEV ; - goto err_unlock_ovs; +rcu_read_unlock (); + goto err_kfree_reply; } - table =ovsl_dereference (dp->table); -if (!table) { + table =rcu_dereference (dp->table); +if (!table || !ovs_flow_tbl_get (table)) { error = -ENODEV ; - goto err_unlock_ovs; +rcu_read_unlock (); + goto err_kfree_reply; } +rcu_read_unlock (); + +/* It is safe to dereference "table" after leaving rcu read-protected +* region because it's pinned by refcount. +*/ +mutex_lock (&table->lock);/* Check if this is a duplicate flow */ if (ovs_identifier_is_ufid (&new_flow->id))
@@ -1105,7 +1120,7 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) error =ovs_flow_tbl_insert (table, new_flow, &mask);if (unlikely (error)) { acts =NULL ; - goto err_unlock_ovs; + goto err_unlock_tbl; }if (unlikely (reply)) {
@@ -1117,7 +1132,6 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) ufid_flags);BUG_ON (error <0 ); } -ovs_unlock (); }else {struct sw_flow_actions *old_acts;
@@ -1130,7 +1144,7 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)if (unlikely (info->nlhdr->nlmsg_flags & (NLM_F_CREATE |NLM_F_EXCL ))) { error = -EEXIST ; - goto err_unlock_ovs; + goto err_unlock_tbl; }/* The flow identifier has to be the same for flow updates. * Look for any overlapping flow.
@@ -1143,7 +1157,7 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) flow =NULL ;if (!flow) { error = -ENOENT ; - goto err_unlock_ovs; + goto err_unlock_tbl; } }/* Update actions. */
@@ -1159,20 +1173,23 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) ufid_flags);BUG_ON (error <0 ); } -ovs_unlock (); -ovs_nla_free_flow_actions_rcu (old_acts);ovs_flow_free (new_flow, false); } +mutex_unlock (&table->lock); +ovs_flow_tbl_put (table); +if (reply)ovs_notify (&dp_flow_genl_family, reply, info);kfree (key);return 0 ; -err_unlock_ovs: -ovs_unlock (); +err_unlock_tbl: +mutex_unlock (&table->lock); +ovs_flow_tbl_put (table); +err_kfree_reply:kfree_skb (reply); err_kfree_acts:ovs_nla_free_flow_actions (acts);
@@ -1301,17 +1318,26 @@ static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info) } } -ovs_lock (); +rcu_read_lock (); dp =get_dp (net, ovs_header->dp_ifindex);if (unlikely (!dp)) { error = -ENODEV ; - goto err_unlock_ovs; +rcu_read_unlock (); + goto err_free_reply; } - table =ovsl_dereference (dp->table); -if (!table) { + table =rcu_dereference (dp->table); +if (!table || !ovs_flow_tbl_get (table)) { +rcu_read_unlock (); error = -ENODEV ; - goto err_unlock_ovs; + goto err_free_reply; } +rcu_read_unlock (); + +/* It is safe to dereference "table" after leaving rcu read-protected +* region because it's pinned by refcount. +*/ +mutex_lock (&table->lock); +/* Check that the flow exists. */ if (ufid_present) flow =ovs_flow_tbl_lookup_ufid (table, &sfid);
@@ -1319,7 +1345,7 @@ static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info) flow =ovs_flow_tbl_lookup_exact (table, &match);if (unlikely (!flow)) { error = -ENOENT ; - goto err_unlock_ovs; + goto err_unlock_tbl; }/* Update actions, if present. */
@@ -1345,14 +1371,16 @@ static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)if (IS_ERR (reply)) { error =PTR_ERR (reply); - goto err_unlock_ovs; + goto err_unlock_tbl; } }/* Clear stats. */ if (a[OVS_FLOW_ATTR_CLEAR ])ovs_flow_stats_clear (flow, table); -ovs_unlock (); + +mutex_unlock (&table->lock); +ovs_flow_tbl_put (table);if (reply)ovs_notify (&dp_flow_genl_family, reply, info);
@@ -1361,8 +1389,10 @@ static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)return 0 ; -err_unlock_ovs: -ovs_unlock (); +err_unlock_tbl: +mutex_unlock (&table->lock); +ovs_flow_tbl_put (table); +err_free_reply:kfree_skb (reply); err_kfree_acts:ovs_nla_free_flow_actions (acts);
@@ -1400,17 +1430,24 @@ static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)if (err)return err; -ovs_lock (); +rcu_read_lock (); dp =get_dp (sock_net (skb->sk), ovs_header->dp_ifindex);if (!dp) { - err = -ENODEV ; - goto unlock; +rcu_read_unlock (); +return -ENODEV ; } - table =ovsl_dereference (dp->table); -if (!table) { - err = -ENODEV ; - goto unlock; + table =rcu_dereference (dp->table); +if (!table || !ovs_flow_tbl_get (table)) { +rcu_read_unlock (); +return -ENODEV ; } +rcu_read_unlock (); + +/* It is safe to dereference "table" after leaving rcu read-protected +* region because it's pinned by refcount. +*/ +mutex_lock (&table->lock); +if (ufid_present) flow =ovs_flow_tbl_lookup_ufid (table, &ufid);
@@ -1429,10 +1466,12 @@ static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info) goto unlock; } -ovs_unlock (); +mutex_unlock (&table->lock); +ovs_flow_tbl_put (table);return genlmsg_reply (reply, info); unlock: -ovs_unlock (); +mutex_unlock (&table->lock); +ovs_flow_tbl_put (table);return err; }
@@ -1462,17 +1501,24 @@ static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)return err; } -ovs_lock (); +rcu_read_lock (); dp =get_dp (sock_net (skb->sk), ovs_header->dp_ifindex);if (unlikely (!dp)) { - err = -ENODEV ; - goto unlock; +rcu_read_unlock (); +return -ENODEV ; } - table =ovsl_dereference (dp->table); -if (!table) { - err = -ENODEV ; - goto unlock; + table =rcu_dereference (dp->table); +if (!table || !ovs_flow_tbl_get (table)) { +rcu_read_unlock (); +return -ENODEV ; } +rcu_read_unlock (); + +/* It is safe to dereference "table" after leaving rcu read-protected +* region because it's pinned by refcount. +*/ +mutex_lock (&table->lock); +if (unlikely (!a[OVS_FLOW_ATTR_KEY ] && !ufid_present)) { err =ovs_flow_tbl_flush (table);
@@ -1489,7 +1535,7 @@ static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info) }ovs_flow_tbl_remove (table, flow); -ovs_unlock (); +mutex_unlock (&table->lock); reply =ovs_flow_cmd_alloc_info ((const struct sw_flow_actions __force *) flow->sf_acts, &flow->id, info, false, ufid_flags);
@@ -1516,10 +1562,12 @@ static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info) } out_free: +ovs_flow_tbl_put (table);ovs_flow_free (flow, true);return 0 ; unlock: -ovs_unlock (); +mutex_unlock (&table->lock); +ovs_flow_tbl_put (table);return err; }
@@ -1545,7 +1593,7 @@ static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)rcu_read_unlock ();return -ENODEV ; } - table =rcu_dereference_ovsl (dp->table); + table =rcu_dereference (dp->table);if (!table) {rcu_read_unlock ();return -ENODEV ;
@@ -1650,10 +1698,6 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,struct flow_table *table;int err, pids_len; - table =ovsl_dereference (dp->table); -if (!table) -return -ENODEV ; - ovs_header =genlmsg_put (skb, portid, seq, &dp_datapath_genl_family, flags, cmd);if (!ovs_header)
@@ -1678,8 +1722,12 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,if (nla_put_u32 (skb,OVS_DP_ATTR_USER_FEATURES , dp->user_features)) goto nla_put_failure; -if (nla_put_u32 (skb,OVS_DP_ATTR_MASKS_CACHE_SIZE , -ovs_flow_tbl_masks_cache_size (table))) +rcu_read_lock (); + table =rcu_dereference (dp->table); + err = table ?nla_put_u32 (skb,OVS_DP_ATTR_MASKS_CACHE_SIZE , +ovs_flow_tbl_masks_cache_size (table)) :0 ; +rcu_read_unlock (); +if (err) goto nla_put_failure;if (dp->user_features &OVS_DP_F_DISPATCH_UPCALL_PER_CPU && pids) {
@@ -1817,7 +1865,9 @@ static int ovs_dp_change(struct datapath *dp, struct nlattr *a[])return -ENODEV ; cache_size =nla_get_u32 (a[OVS_DP_ATTR_MASKS_CACHE_SIZE ]); +mutex_lock (&table->lock); err =ovs_flow_tbl_masks_cache_resize (table, cache_size); +mutex_unlock (&table->lock);if (err)return err; }
@@ -1968,7 +2018,7 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info) err_destroy_stats:free_percpu (dp->stats_percpu); err_destroy_table: -call_rcu (&table->rcu, ovs_flow_tbl_destroy_rcu); +ovs_flow_tbl_put (table); err_destroy_dp:kfree (dp); err_destroy_reply:
@@ -2003,16 +2053,11 @@ static void __dp_destroy(struct datapath *dp)*/ ovs_dp_detach_port (ovs_vport_ovsl (dp,OVSP_LOCAL )); -/* Flush sw_flow in the tables. RCU cb only releases resource -* such as dp, ports and tables. That may avoid some issues -* such as RCU usage warning. -*/ -table_instance_flow_flush (table,ovsl_dereference (table->ti), -ovsl_dereference (table->ufid_ti)); +rcu_assign_pointer (dp->table,NULL ); +ovs_flow_tbl_put (table); -/* RCU destroy the ports, meters and flow tables. */ +/* RCU destroy the ports and meters. */ call_rcu (&dp->rcu, destroy_dp_rcu); -call_rcu (&table->rcu, ovs_flow_tbl_destroy_rcu); }static int ovs_dp_cmd_del (struct sk_buff *skb,struct genl_info *info)
@@ -2656,9 +2701,12 @@ static void ovs_dp_masks_rebalance(struct work_struct *work)ovs_lock ();list_for_each_entry (dp, &ovs_net->dps, list_node) { table =ovsl_dereference (dp->table); -if (!table) +if (!table || !ovs_flow_tbl_get (table))continue ; +mutex_lock (&table->lock);ovs_flow_masks_rebalance (table); +mutex_unlock (&table->lock); +ovs_flow_tbl_put (table); }ovs_unlock ();
net/openvswitch/flow_table.c +37 -9
109 lines not on this page · raw message
net/openvswitch/flow_table.h +33 -4
@@ -59,7 +59,31 @@ struct table_instance {u32 hash_seed; }; +/* Locking: +* +* flow_table is _not_ protected by ovs_lock (see comment above ovs_mutex +* in datapath.c). +* +* All writes to flow_table are protected by the embedded "lock". +* In order to ensure datapath destruction does not trigger the destruction +* of the flow_table, "refcnt" is used. Therefore, writers must: +* 1 - Enter rcu read-protected section +* 2 - Increase "table->refcnt" +* 3 - Leave rcu read-protected section (to avoid using mutexes inside rcu) +* 4 - Lock "table->lock" +* 5 - Perform modifications +* 6 - Release "table->lock" +* 7 - Decrease "table->refcnt" +* +* Reads are protected by RCU. +* +* Note with this schema, it's possible that a flow operation is performed on a +* flow_table that is about to be freed. +*/ struct flow_table { +/* Locks flow table writes. */ +struct mutex lock; +refcount_t refcnt;struct rcu_head rcu;struct table_instance __rcu *ti;struct table_instance __rcu *ufid_ti;
@@ -72,11 +96,15 @@ struct flow_table {extern struct kmem_cache *flow_stats_cache; +#ifdef CONFIG_LOCKDEP +int lockdep_ovs_tbl_is_held (const struct flow_table *table); +#else static inline int lockdep_ovs_tbl_is_held (const struct flow_table *table__always_unused ) {return 1 ; } +#endif #define ASSERT_OVS_TBL (tbl) WARN_ON(!lockdep_ovs_tbl_is_held(tbl))
@@ -95,7 +123,11 @@ struct sw_flow *ovs_flow_alloc(void);void ovs_flow_free (struct sw_flow *,bool deferred);struct flow_table *ovs_flow_tbl_alloc (void ); -void ovs_flow_tbl_destroy_rcu (struct rcu_head *table); +void ovs_flow_tbl_put (struct flow_table *table); +static inline bool ovs_flow_tbl_get (struct flow_table *table) +{ +return refcount_inc_not_zero (&table->refcnt); +}int ovs_flow_tbl_count (const struct flow_table *table);int ovs_flow_tbl_flush (struct flow_table *flow_table);
@@ -125,8 +157,5 @@ void ovs_flow_mask_key(struct sw_flow_key *dst, const struct sw_flow_key *src, bool full, const struct sw_flow_mask *mask);void ovs_flow_masks_rebalance (struct flow_table *table); -void table_instance_flow_flush (struct flow_table *table, -struct table_instance *ti, -struct table_instance *ufid_ti);#endif /* flow_table.h */
Signature · 1 line
2.54.0
Eelco Chaudron
maintainerRe: [PATCH net-next v4 3/4] net: openvswitch: decouple flow_table from ovs_mutex
Quoted 12 lines · Adrian Moreno
In order to protect flow operations from RTNL contention, this patch decouples
flow_tablemodifications fromovs_mutexby means of the following:1 - Create a new mutex inside the
flow_tablethat protects it from concurrent modifications.
Putting the mutex insideflow_tablemakes it easier to consume for functions insideflow_table.cthat do not currently take pointers to the datapath.
Some function signatures need to be changed to acceptflow_tableso that lockdep checks can be performed.
Quoted 6 lines
2 - Create a reference count to temporarily extend rcu protection from the datapath to the
flow_table.
One reference is held by the datapath, the other is temporarily increased during flow modifications.Signed-off-by Adrian Moreno <[email protected]>
Hi Adrian,
Thanks for this patch. I did spend quite some time analyzing the implications of decoupling the table from ovs_mutex, and I think all the concerns I had are covered. I did not do any extensive traffic-based testing, but if time allows I will do so with your next revision.
Find some comments below.
Cheers,
Eelco
[...]
Quoted 16 lines
net/openvswitch/datapath.c +6
-2@@ -1678,8 +1722,12 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,if (nla_put_u32 (skb,OVS_DP_ATTR_USER_FEATURES , dp->user_features)) goto nla_put_failure; -if (nla_put_u32 (skb,OVS_DP_ATTR_MASKS_CACHE_SIZE , -ovs_flow_tbl_masks_cache_size (table))) +rcu_read_lock (); + table =rcu_dereference (dp->table); + err = table ?nla_put_u32 (skb,OVS_DP_ATTR_MASKS_CACHE_SIZE , +ovs_flow_tbl_masks_cache_size (table)) :0 ; +rcu_read_unlock (); +if (err) goto nla_put_failure;if (dp->user_features &OVS_DP_F_DISPATCH_UPCALL_PER_CPU && pids) {
net/openvswitch/datapath.c static int ovs_dp_change(struct datapath *dp, struct nlattr *a[]) · line 1870
Quoted 6 lines
net/openvswitch/datapath.c +2
-0@@ -1817,7 +1865,9 @@ static int ovs_dp_change(struct datapath *dp, struct nlattr *a[])return -ENODEV ; cache_size =nla_get_u32 (a[OVS_DP_ATTR_MASKS_CACHE_SIZE ]); +mutex_lock (&table->lock); err =ovs_flow_tbl_masks_cache_resize (table, cache_size); +mutex_unlock (&table->lock);
The locking schema in flow_table.h does not document that ovs_mutex-held writers may skip the refcount steps. Should this either be documented as an exception, or should the full 7-step protocol be followed here for consistency?
Quoted 3 lines
net/openvswitch/datapath.c +0
-0if (err)return err; }
[...]
net/openvswitch/datapath.c static void ovs_dp_masks_rebalance(struct work_struct *work) · line 2708
Quoted 10 lines
net/openvswitch/datapath.c +3
-1@@ -2656,9 +2701,12 @@ static void ovs_dp_masks_rebalance(struct work_struct *work)ovs_lock ();list_for_each_entry (dp, &ovs_net->dps, list_node) { table =ovsl_dereference (dp->table); -if (!table) +if (!table || !ovs_flow_tbl_get (table))continue ; +mutex_lock (&table->lock);ovs_flow_masks_rebalance (table); +mutex_unlock (&table->lock);
Same question as above, but here the RCU steps are skipped instead of the refcount steps.
Quoted 5 lines
net/openvswitch/datapath.c +1
-0+ovs_flow_tbl_put (table); }ovs_unlock ();
[...]
Quoted 10 lines
net/openvswitch/flow_table.c +1
-1@@ -513,7 +528,7 @@ static void table_instance_destroy(struct table_instance *ti, }/* No need for locking this function is called from RCU callback. */ -void ovs_flow_tbl_destroy_rcu (struct rcu_head *rcu) +static void ovs_flow_tbl_destroy_rcu (struct rcu_head *rcu) { struct flow_table *table = container_of(rcu, struct flow_table, rcu);
net/openvswitch/flow_table.c void ovs_flow_tbl_destroy_rcu(struct rcu_head *rcu) · line 549
Quoted 10 lines
net/openvswitch/flow_table.c +4
-0@@ -525,9 +540,22 @@ void ovs_flow_tbl_destroy_rcu(struct rcu_head *rcu)call_rcu (&mc->rcu, mask_cache_rcu_cb);call_rcu (&ma->rcu, mask_array_rcu_cb);table_instance_destroy (ti, ufid_ti); +mutex_destroy (&table->lock);kfree (table); } +void ovs_flow_tbl_put (struct flow_table *table) +{ +if (refcount_dec_and_test (&table->refcnt)) {
Do we need a comment explaining why the 7-step protocol is not followed here? Something like:
/* Last reference dropped, no concurrent writers possible. * Lock is only needed for lockdep assertions below. */
Quoted 12 lines
net/openvswitch/flow_table.c +9
-0+mutex_lock (&table->lock); +table_instance_flow_flush (table, +ovs_tbl_dereference (table->ti, table), +ovs_tbl_dereference (table->ufid_ti, table)); +mutex_unlock (&table->lock); +call_rcu (&table->rcu, ovs_flow_tbl_destroy_rcu); + } +} +struct sw_flow *ovs_flow_tbl_dump_next (struct table_instance *ti,u32 *bucket,u32 *last) {
[...]
PATCH v4 4/4 net: openvswitch: avoid double-rcu wait period
In reply to Adrian MorenoAvoid waiting for two rcu periods by scheduling the deletion of the flow_table and all of its referenced structs at the same time.
--- net/openvswitch/flow_table.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-)
net/openvswitch/flow_table.c +15 -12
@@ -527,30 +527,33 @@ static void table_instance_destroy(struct table_instance *ti,call_rcu (&ufid_ti->rcu, flow_tbl_destroy_rcu_cb); } -/* No need for locking this function is called from RCU callback. */ static void ovs_flow_tbl_destroy_rcu (struct rcu_head *rcu) { struct flow_table *table = container_of(rcu, struct flow_table, rcu); -struct table_instance *ti =rcu_dereference_raw (table->ti); -struct table_instance *ufid_ti =rcu_dereference_raw (table->ufid_ti); -struct mask_cache *mc =rcu_dereference_raw (table->mask_cache); -struct mask_array *ma =rcu_dereference_raw (table->mask_array); - -call_rcu (&mc->rcu, mask_cache_rcu_cb); -call_rcu (&ma->rcu, mask_array_rcu_cb); -table_instance_destroy (ti, ufid_ti);mutex_destroy (&table->lock);kfree (table); }void ovs_flow_tbl_put (struct flow_table *table) { +struct table_instance *ufid_ti; +struct table_instance *ti; +struct mask_cache *mc; +struct mask_array *ma; +if (refcount_dec_and_test (&table->refcnt)) {mutex_lock (&table->lock); -table_instance_flow_flush (table, -ovs_tbl_dereference (table->ti, table), -ovs_tbl_dereference (table->ufid_ti, table)); + ufid_ti =ovs_tbl_dereference (table->ufid_ti, table); + ti =ovs_tbl_dereference (table->ti, table); +table_instance_flow_flush (table, ti, ufid_ti); +table_instance_destroy (ti, ufid_ti); + + mc =ovs_tbl_dereference (table->mask_cache, table); + ma =ovs_tbl_dereference (table->mask_array, table); +call_rcu (&mc->rcu, mask_cache_rcu_cb); +call_rcu (&ma->rcu, mask_array_rcu_cb); +mutex_unlock (&table->lock);call_rcu (&table->rcu, ovs_flow_tbl_destroy_rcu); }
Signature · 1 line
2.54.0
Eelco Chaudron
maintainerRe: [PATCH net-next v4 4/4] net: openvswitch: avoid double-rcu wait period
Quoted 2 lines · Adrian Moreno
Avoid waiting for two rcu periods by scheduling the deletion of the
flow_tableand all of its referenced structs at the same time.
Hi Adrian,
One small nit below; the code itself looks fine to me.
Cheers,
Eelco
Quoted 41 lines
Signed-off-by Adrian Moreno <[email protected]>--- net/openvswitch/flow_table.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-)net/openvswitch/flow_table.c +5
-11@@ -527,30 +527,33 @@ static void table_instance_destroy(struct table_instance *ti,call_rcu (&ufid_ti->rcu, flow_tbl_destroy_rcu_cb); } -/* No need for locking this function is called from RCU callback. */ static void ovs_flow_tbl_destroy_rcu (struct rcu_head *rcu) { struct flow_table *table = container_of(rcu, struct flow_table, rcu); -struct table_instance *ti =rcu_dereference_raw (table->ti); -struct table_instance *ufid_ti =rcu_dereference_raw (table->ufid_ti); -struct mask_cache *mc =rcu_dereference_raw (table->mask_cache); -struct mask_array *ma =rcu_dereference_raw (table->mask_array); - -call_rcu (&mc->rcu, mask_cache_rcu_cb); -call_rcu (&ma->rcu, mask_array_rcu_cb); -table_instance_destroy (ti, ufid_ti);mutex_destroy (&table->lock);kfree (table); }void ovs_flow_tbl_put (struct flow_table *table) { +struct table_instance *ufid_ti; +struct table_instance *ti; +struct mask_cache *mc; +struct mask_array *ma; +if (refcount_dec_and_test (&table->refcnt)) {mutex_lock (&table->lock); - table_instance_flow_flush(table, - ovs_tbl_dereference(table->ti, table),
net/openvswitch/flow_table.c static void table_instance_destroy(struct table_instance *ti, · line 555
Quoted 10 lines
net/openvswitch/flow_table.c +9
-1- ovs_tbl_dereference(table->ufid_ti, table)); + ufid_ti =ovs_tbl_dereference (table->ufid_ti, table); + ti =ovs_tbl_dereference (table->ti, table); +table_instance_flow_flush (table, ti, ufid_ti); +table_instance_destroy (ti, ufid_ti); + + mc =ovs_tbl_dereference (table->mask_cache, table); + ma =ovs_tbl_dereference (table->mask_array, table); +call_rcu (&mc->rcu, mask_cache_rcu_cb); +call_rcu (&ma->rcu, mask_array_rcu_cb);
nit: Would it be cleaner to extract the destruction logic in ovs_flow_tbl_put() into a separate static function, e.g. ovs_flow_tbl_destroy(), to separate the refcount handling from the actual cleanup?
Quoted 3 lines
net/openvswitch/flow_table.c +0
-0mutex_unlock (&table->lock);call_rcu (&table->rcu, ovs_flow_tbl_destroy_rcu); }