tests: py: adapt this to new add element command semantics
Pablo Neira Ayuso
maintainerPATCH 1/8 tests: py: adapt this to new add element command semantics
Since fd33d96 ("src: create element command"), add element doesn't fail anymore if the element exists, you have to use create instead in case you want to check if the element already exists.
--- tests/py/ip/sets.t | 6 +++--- tests/py/ip6/sets.t | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-)
tests/py/ip/sets.t +3 -3
@@ -13,9 +13,9 @@ !set1 type ipv4_addr;ok ?set1 192.168.3.4;ok -?set1 192.168.3.4;fail +?set1 192.168.3.4;ok +?set1 192.168.3.5 192.168.3.6;ok ?set1 192.168.3.5 192.168.3.6;ok -?set1 192.168.3.5 192.168.3.6;fail ?set1 192.168.3.8 192.168.3.9;ok ?set1 192.168.3.10 192.168.3.11;ok ?set1 1234:1234:1234:1234:1234:1234:1234:1234;fail
@@ -24,7 +24,7 @@ !set2 type ipv4_addr;ok ?set2 192.168.3.4;ok ?set2 192.168.3.5 192.168.3.6;ok -?set2 192.168.3.5 192.168.3.6;fail +?set2 192.168.3.5 192.168.3.6;ok ?set2 192.168.3.8 192.168.3.9;ok ?set2 192.168.3.10 192.168.3.11;ok
tests/py/ip6/sets.t +2 -2
@@ -13,12 +13,12 @@ ?set2 192.168.3.4;fail !set2 type ipv6_addr;ok ?set2 1234:1234::1234:1234:1234:1234:1234;ok -?set2 1234:1234::1234:1234:1234:1234:1234;fail +?set2 1234:1234::1234:1234:1234:1234:1234;ok ?set2 1234::1234:1234:1234;ok ?set2 1234:1234:1234:1234:1234::1234:1234 1234:1234::123;ok ?set2 192.168.3.8 192.168.3.9;fail ?set2 1234:1234::1234:1234:1234:1234;ok -?set2 1234:1234::1234:1234:1234:1234;fail +?set2 1234:1234::1234:1234:1234:1234;ok ?set2 1234:1234:1234::1234;ok ip6 saddr @set2 drop;ok
Signature · 1 line
2.1.4
Pablo Neira Ayuso
maintainerPATCH 2/8 src: add quota statement
This new statement is stateful, so it can be used from flow tables, eg.
# nft add rule filter input \
flow table http { ip saddr timeout 60s quota over 50 mbytes } dropThis basically sets a quota per source IP address of 50 mbytes after which packets are dropped. Note that the timeout releases the entry if no traffic is seen from this IP after 60 seconds.
--- include/statement.h | 10 +++++++++ src/evaluate.c | 1 + src/netlink_delinearize.c | 14 ++++++++++++ src/netlink_linearize.c | 16 ++++++++++++++ src/parser_bison.y | 38 +++++++++++++++++++++++++++----- src/scanner.l | 2 ++ src/statement.c | 26 ++++++++++++++++++++++ tests/py/any/quota.t | 24 ++++++++++++++++++++ tests/py/any/quota.t.payload | 52 ++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 178 insertions(+), 5 deletions(-) create mode 100644 tests/py/any/quota.t create mode 100644 tests/py/any/quota.t.payload
include/statement.h +10 -0
@@ -105,6 +105,13 @@ struct queue_stmt {extern struct stmt *queue_stmt_alloc (const struct location *loc); +struct quota_stmt { +uint64_t bytes; +uint32_t flags; +}; + +struct stmt *quota_stmt_alloc (const struct location *loc); +#include <ct.h> struct ct_stmt {enum nft_ct_keys key;
@@ -200,6 +207,7 @@ extern struct stmt *xt_stmt_alloc(const struct location *loc);* @STMT_DUP: dup statement * @STMT_FWD: forward statement * @STMT_XT: XT statement +* @STMT_QUOTA: quota statement */ enum stmt_types {STMT_INVALID ,
@@ -221,6 +229,7 @@ enum stmt_types {STMT_DUP ,STMT_FWD ,STMT_XT , + STMT_QUOTA, };/**
@@ -272,6 +281,7 @@ struct stmt {struct masq_stmt masq;struct redir_stmt redir;struct queue_stmt queue; +struct quota_stmt quota;struct ct_stmt ct;struct set_stmt set;struct dup_stmt dup;
src/evaluate.c +1 -0
@@ -2366,6 +2366,7 @@ int stmt_evaluate(struct eval_ctx *ctx, struct stmt *stmt)switch (stmt->ops->type) {case STMT_COUNTER :case STMT_LIMIT : +case STMT_QUOTA :return 0 ;case STMT_EXPRESSION :return stmt_evaluate_expr (ctx, stmt);
src/netlink_delinearize.c +14 -0
@@ -620,6 +620,19 @@ static void netlink_parse_limit(struct netlink_parse_ctx *ctx, ctx->stmt = stmt; } +static void netlink_parse_quota (struct netlink_parse_ctx *ctx, +const struct location *loc, +const struct nftnl_expr *nle) +{ +struct stmt *stmt; + + stmt =quota_stmt_alloc (loc); + stmt->quota.bytes =nftnl_expr_get_u64 (nle,NFTNL_EXPR_QUOTA_BYTES ); + stmt->quota.flags =nftnl_expr_get_u32 (nle,NFTNL_EXPR_QUOTA_FLAGS ); + + ctx->stmt = stmt; +} +static void netlink_parse_reject (struct netlink_parse_ctx *ctx,const struct location *loc,const struct nftnl_expr *expr)
@@ -989,6 +1002,7 @@ static const struct { { .name = "fwd", .parse = netlink_parse_fwd }, { .name = "target", .parse = netlink_parse_target }, { .name = "match", .parse = netlink_parse_match }, + { .name = "quota", .parse = netlink_parse_quota }, }; static int netlink_parse_expr(const struct nftnl_expr *nle,
src/netlink_linearize.c +16 -0
@@ -657,6 +657,19 @@ netlink_gen_limit_stmt(struct netlink_linearize_ctx *ctx, }static struct nftnl_expr * +netlink_gen_quota_stmt (struct netlink_linearize_ctx *ctx, +const struct stmt *stmt) +{ +struct nftnl_expr *nle; + + nle =alloc_nft_expr ("quota" ); +nftnl_expr_set_u64 (nle,NFTNL_EXPR_QUOTA_BYTES , stmt->quota.bytes); +nftnl_expr_set_u32 (nle,NFTNL_EXPR_QUOTA_FLAGS , stmt->quota.flags); + +return nle; +} + +static struct nftnl_expr *netlink_gen_stmt_stateful (struct netlink_linearize_ctx *ctx,const struct stmt *stmt) {
@@ -665,6 +678,8 @@ netlink_gen_stmt_stateful(struct netlink_linearize_ctx *ctx,return netlink_gen_counter_stmt (ctx, stmt);case STMT_LIMIT :return netlink_gen_limit_stmt (ctx, stmt); +case STMT_QUOTA : +return netlink_gen_quota_stmt (ctx, stmt);default :BUG ("unknown stateful statement type %s\n" , stmt->ops->name); }
@@ -1105,6 +1120,7 @@ static void netlink_gen_stmt(struct netlink_linearize_ctx *ctx,return netlink_gen_fwd_stmt (ctx, stmt);case STMT_COUNTER :case STMT_LIMIT : +case STMT_QUOTA : nle =netlink_gen_stmt_stateful (ctx, stmt);nftnl_rule_add_expr (ctx->nlr, nle);break ;
src/parser_bison.y +33 -5
@@ -376,6 +376,8 @@ static void location_update(struct location *loc, struct location *rhs, int n) %token OVER "over" %token UNTIL "until" +%token QUOTA "quota" + %token NANOSECOND "nanosecond" %token MICROSECOND "microsecond" %token MILLISECOND "millisecond"
@@ -431,8 +433,8 @@ static void location_update(struct location *loc, struct location *rhs, int n) %destructor { handle_free(&$$); } set_spec set_identifier %type <val> family_spec family_spec_explicit chain_policy prio_spec -%type <string> dev_spec -%destructor { xfree($$); } dev_spec +%type <string> dev_spec quota_unit +%destructor { xfree($$); } dev_spec quota_unit %type <table> table_block_alloc table_block %destructor { close_scope(state); table_free($$); } table_block_alloc
@@ -466,9 +468,9 @@ static void location_update(struct location *loc, struct location *rhs, int n) %type <stmt> log_stmt log_stmt_alloc %destructor { stmt_free($$); } log_stmt log_stmt_alloc %type <val> level_type -%type <stmt> limit_stmt -%destructor { stmt_free($$); } limit_stmt -%type <val> limit_burst limit_mode time_unit +%type <stmt> limit_stmt quota_stmt +%destructor { stmt_free($$); } limit_stmt quota_stmt +%type <val> limit_burst limit_mode time_unit quota_mode %type <stmt> reject_stmt reject_stmt_alloc %destructor { stmt_free($$); } reject_stmt reject_stmt_alloc %type <stmt> nat_stmt nat_stmt_alloc masq_stmt masq_stmt_alloc redir_stmt redir_stmt_alloc
@@ -1373,6 +1375,7 @@ stmt : verdict_stmt | meta_stmt | log_stmt | limit_stmt + | quota_stmt | reject_stmt | nat_stmt | queue_stmt
@@ -1542,6 +1545,31 @@ limit_stmt : LIMIT RATE limit_mode NUM SLASH time_unit limit_burst } ; +quota_mode : OVER { $$ = NFT_QUOTA_F_INV; } + | UNTIL { $$ = 0; } + | /* empty */ { $$ = 0; } + ; + +quota_unit : BYTES { $$ = xstrdup("bytes"); } + | STRING { $$ = $1; } + ; + +quota_stmt : QUOTA quota_mode NUM quota_unit + { + struct error_record *erec; + uint64_t rate; + + erec = data_unit_parse(&@$, $4, &rate); + if (erec != NULL) { + erec_queue(erec, state->msgs); + YYERROR; + } + $$ = quota_stmt_alloc(&@$); + $$->quota.bytes = $3 * rate; + $$->quota.flags = $2; + } + ; + limit_mode : OVER { $$ = NFT_LIMIT_F_INV; } | UNTIL { $$ = 0; } | /* empty */ { $$ = 0; }
src/scanner.l +2 -0
@@ -319,6 +319,8 @@ addrstring ({macaddr}|{ip4addr}|{ip6addr}) "until" { return UNTIL; } "over" { return OVER; } +"quota" { return QUOTA; } + "nanosecond" { return NANOSECOND; } "microsecond" { return MICROSECOND; } "millisecond" { return MILLISECOND; }
src/statement.c +26 -0
@@ -325,6 +325,32 @@ struct stmt *queue_stmt_alloc(const struct location *loc)return stmt_alloc (loc, &queue_stmt_ops); } +static void quota_stmt_print (const struct stmt *stmt) +{ +bool inv = stmt->quota.flags &NFT_QUOTA_F_INV ; +const char *data_unit; +uint64_t bytes; + + data_unit =get_rate (stmt->quota.bytes, &bytes); +printf ("quota %s%" PRIu64" %s" , + inv ?"over " :"" , bytes, data_unit); +} + +static const struct stmt_ops quota_stmt_ops = { + .type =STMT_QUOTA , + .name ="quota" , + .print = quota_stmt_print, +}; + +struct stmt *quota_stmt_alloc (const struct location *loc) +{ +struct stmt *stmt; + + stmt =stmt_alloc (loc, "a_stmt_ops); + stmt->flags |=STMT_F_STATEFUL ; +return stmt; +} +static void reject_stmt_print (const struct stmt *stmt) {printf ("reject" );
tests/py/any/quota.t new file +24 -0
@@ -0,0 +1,24 @@ +:output;type filter hook output priority 0 +:ingress;type filter hook ingress device lo priority 0 + +*ip;test-ip4;output +*ip6;test-ip6;output +*inet;test-inet;output +*arp;test-arp;output +*bridge;test-bridge;output +*netdev;test-netdev;ingress + +quota 1025 bytes;ok +quota 1 kbytes;ok +quota 2 kbytes;ok +quota 1025 kbytes;ok +quota 1023 mbytes;ok +quota 10230 mbytes;ok +quota 1023000 mbytes;ok + +quota over 1 kbytes;ok +quota over 2 kbytes;ok +quota over 1025 kbytes;ok +quota over 1023 mbytes;ok +quota over 10230 mbytes;ok +quota over 1023000 mbytes;ok
tests/py/any/quota.t.payload new file +52 -0
52 lines not on this page · raw message
Signature · 1 line
2.1.4
Pablo Neira Ayuso
maintainerPATCH 3/8 src: add numgen expression
In reply to Pablo Neira AyusoThis new expression allows us to generate incremental and random numbers bound to a specified modulus value.
The following rule sets the conntrack mark of 0 to the first packet seen, then 1 to second packet, then 0 again to the third packet and so on:
# nft add rule x y ct mark set numgen inc mod 2
A more useful example is a simple load balancing scenario, where you can also use maps to set the destination NAT address based on this new numgen expression:
# nft add rule nat prerouting \
dnat to numgen inc mod 2 map { 0 : 192.168.10.100, 1 : 192.168.20.200 }So this is distributing new connections in a round-robin fashion between 192.168.10.100 and 192.168.20.200. Don't forget the special NAT chain
semantics: Only the first packet evaluates the rule, follow up packets rely on conntrack to apply the NAT information.
You can also emulate flow distribution with different backend weights using intervals:
# nft add rule nat prerouting \
dnat to numgen inc mod 10 map { 0-5 : 192.168.10.100, 6-9 : 192.168.20.200 }So 192.168.10.100 gets 60% of the workload, while 192.168.20.200 gets 40%.
We can also be mixed with dynamic sets, thus weight can be updated in runtime.
--- include/Makefile.am | 1 + include/expression.h | 8 ++++++ include/numgen.h | 7 +++++ src/Makefile.am | 1 + src/evaluate.c | 27 ++++++++++++++++++ src/netlink_delinearize.c | 18 ++++++++++++ src/netlink_linearize.c | 15 ++++++++++ src/numgen.c | 68 ++++++++++++++++++++++++++++++++++++++++++++ src/parser_bison.y | 21 ++++++++++++-- src/scanner.l | 4 +++ tests/py/ip/numgen.t | 6 ++++ tests/py/ip/numgen.t.payload | 24 ++++++++++++++++ 12 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 include/numgen.h create mode 100644 src/numgen.c create mode 100644 tests/py/ip/numgen.t create mode 100644 tests/py/ip/numgen.t.payload
include/Makefile.am +1 -0
@@ -17,6 +17,7 @@ noinst_HEADERS = cli.h \ headers.h \ list.h \ meta.h \ + numgen.h \ netlink.h \ parser.h \ proto.h \
include/expression.h +8 -0
@@ -33,6 +33,7 @@* @EXPR_UNARY: byteorder conversion, generated during evaluation * @EXPR_BINOP: binary operations (bitwise, shifts) * @EXPR_RELATIONAL: equality and relational expressions +* @EXPR_NUMGEN: number generation expression */ enum expr_types {EXPR_INVALID ,
@@ -55,6 +56,7 @@ enum expr_types {EXPR_UNARY ,EXPR_BINOP ,EXPR_RELATIONAL , + EXPR_NUMGEN, }; enum ops {
@@ -170,6 +172,7 @@ enum expr_flags {#include <payload.h> #include <exthdr.h> +#include <numgen.h> #include <meta.h> #include <ct.h>
@@ -277,6 +280,11 @@ struct expr {enum nft_ct_keys key;int8_t direction; } ct; +struct { +/* EXPR_NUMGEN */ +enum nft_ng_types type; +uint32_t mod; + } numgen; }; };
include/numgen.h new file +7 -0
@@ -0,0 +1,7 @@ +#ifndef NFTABLES_NUMGEN_H +#define NFTABLES_NUMGEN_H + +extern struct expr *numgen_expr_alloc (const struct location *loc, +enum nft_ng_types type,uint32_t until); + +#endif /* NFTABLES_NUMGEN_H */
src/Makefile.am +1 -0
@@ -37,6 +37,7 @@ nft_SOURCES = main.c \ payload.c \ exthdr.c \ meta.c \ + numgen.c \ ct.c \ netlink.c \ netlink_linearize.c \
src/evaluate.c +27 -0
@@ -1158,6 +1158,31 @@ static int expr_evaluate_mapping(struct eval_ctx *ctx, struct expr **expr)return 0 ; } +/* We got datatype context via statement. If the basetype is compatible, set +* this expression datatype to the one of the statement to make it datatype +* compatible. This is a more conservative approach than enabling datatype +* compatibility between two different datatypes whose basetype is the same, +* let's revisit this later once users come with valid usecases to generalize +* this. +*/ +static void expr_dtype_integer_compatible (struct eval_ctx *ctx, +struct expr *expr) +{ +if (ctx->ectx.dtype && + ctx->ectx.dtype->basetype == &integer_type && + ctx->ectx.len ==4 *BITS_PER_BYTE ) { + expr->dtype = ctx->ectx.dtype; + expr->len = ctx->ectx.len; + } +} + +static int expr_evaluate_numgen (struct eval_ctx *ctx,struct expr **exprp) +{ +expr_dtype_integer_compatible (ctx, *exprp); + +return expr_evaluate_primary (ctx, exprp); +} +/* * Transfer the invertible binops to the constant side of an equality * expression. A left shift is only invertible if the low n bits are
@@ -1560,6 +1585,8 @@ static int expr_evaluate(struct eval_ctx *ctx, struct expr **expr)return expr_evaluate_mapping (ctx, expr);case EXPR_RELATIONAL :return expr_evaluate_relational (ctx, expr); +case EXPR_NUMGEN : +return expr_evaluate_numgen (ctx, expr);default :BUG ("unknown expression type %s\n" , (*expr)->ops->name); }
src/netlink_delinearize.c +18 -0
@@ -507,6 +507,22 @@ static void netlink_parse_meta(struct netlink_parse_ctx *ctx,netlink_parse_meta_stmt (ctx, loc, nle); } +static void netlink_parse_numgen (struct netlink_parse_ctx *ctx, +const struct location *loc, +const struct nftnl_expr *nle) +{ +enum nft_registers dreg; +uint32_t type, until; +struct expr *expr; + + type =nftnl_expr_get_u32 (nle,NFTNL_EXPR_NG_TYPE ); + until =nftnl_expr_get_u32 (nle,NFTNL_EXPR_NG_UNTIL ); + + expr =numgen_expr_alloc (loc, type, until); + dreg =netlink_parse_register (nle,NFTNL_EXPR_NG_DREG ); +netlink_set_register (ctx, dreg, expr); +} +static void netlink_parse_ct_stmt (struct netlink_parse_ctx *ctx,const struct location *loc,const struct nftnl_expr *nle)
@@ -1003,6 +1019,7 @@ static const struct { { .name = "target", .parse = netlink_parse_target }, { .name = "match", .parse = netlink_parse_match }, { .name = "quota", .parse = netlink_parse_quota }, + { .name = "numgen", .parse = netlink_parse_numgen }, }; static int netlink_parse_expr(const struct nftnl_expr *nle,
@@ -1622,6 +1639,7 @@ static void expr_postprocess(struct rule_pp_ctx *ctx, struct expr **exprp)case EXPR_SET_REF :case EXPR_META :case EXPR_VERDICT : +case EXPR_NUMGEN :break ;case EXPR_CT :ct_expr_update_type (&ctx->pctx, expr);
src/netlink_linearize.c +15 -0
@@ -151,6 +151,19 @@ static void netlink_gen_meta(struct netlink_linearize_ctx *ctx,nftnl_rule_add_expr (ctx->nlr, nle); } +static void netlink_gen_numgen (struct netlink_linearize_ctx *ctx, +const struct expr *expr, +enum nft_registers dreg) +{ +struct nftnl_expr *nle; + + nle =alloc_nft_expr ("numgen" ); +netlink_put_register (nle,NFTNL_EXPR_NG_DREG , dreg); +netlink_put_register (nle,NFTNL_EXPR_NG_TYPE , expr->numgen.type); +nftnl_expr_set_u32 (nle,NFTNL_EXPR_NG_UNTIL , expr->numgen.mod); +nftnl_rule_add_expr (ctx->nlr, nle); +} +static void netlink_gen_ct (struct netlink_linearize_ctx *ctx,const struct expr *expr,enum nft_registers dreg)
@@ -614,6 +627,8 @@ static void netlink_gen_expr(struct netlink_linearize_ctx *ctx,return netlink_gen_ct (ctx, expr, dreg);case EXPR_SET_ELEM :return netlink_gen_expr (ctx, expr->key, dreg); +case EXPR_NUMGEN : +return netlink_gen_numgen (ctx, expr, dreg);default :BUG ("unknown expression type %s\n" , expr->ops->name); }
src/numgen.c new file +68 -0
68 lines not on this page · raw message
src/parser_bison.y +18 -3
@@ -407,6 +407,10 @@ static void location_update(struct location *loc, struct location *rhs, int n) %token DUP "dup" %token FWD "fwd" +%token NUMGEN "numgen" +%token INC "inc" +%token MOD "mod" + %token POSITION "position" %token COMMENT "comment"
@@ -552,8 +556,8 @@ static void location_update(struct location *loc, struct location *rhs, int n) %type <expr> arp_hdr_expr %destructor { expr_free($$); } arp_hdr_expr %type <val> arp_hdr_field -%type <expr> ip_hdr_expr icmp_hdr_expr -%destructor { expr_free($$); } ip_hdr_expr icmp_hdr_expr +%type <expr> ip_hdr_expr icmp_hdr_expr numgen_expr +%destructor { expr_free($$); } ip_hdr_expr icmp_hdr_expr numgen_expr %type <val> ip_hdr_field icmp_hdr_field %type <expr> ip6_hdr_expr icmp6_hdr_expr %destructor { expr_free($$); } ip6_hdr_expr icmp6_hdr_expr
@@ -582,7 +586,7 @@ static void location_update(struct location *loc, struct location *rhs, int n) %type <expr> meta_expr %destructor { expr_free($$); } meta_expr -%type <val> meta_key meta_key_qualified meta_key_unqualified +%type <val> meta_key meta_key_qualified meta_key_unqualified numgen_type %type <expr> ct_expr %destructor { expr_free($$); } ct_expr
@@ -1967,6 +1971,7 @@ primary_expr : symbol_expr { $$ = $1; } | exthdr_expr { $$ = $1; } | meta_expr { $$ = $1; } | ct_expr { $$ = $1; } + | numgen_expr { $$ = $1; } | '(' basic_expr ')' { $$ = $2; } ;
@@ -2454,6 +2459,16 @@ meta_stmt : META meta_key SET expr } ; +numgen_type : INC { $$ = NFT_NG_INCREMENTAL; } + | RANDOM { $$ = NFT_NG_RANDOM; } + ; + +numgen_expr : NUMGEN numgen_type MOD NUM + { + $$ = numgen_expr_alloc(&@$, $2, $4); + } + ; + ct_expr : CT ct_key { $$ = ct_expr_alloc(&@$, $2, -1);
src/scanner.l +4 -0
@@ -467,6 +467,10 @@ addrstring ({macaddr}|{ip4addr}|{ip6addr}) "proto-dst" { return PROTO_DST; } "label" { return LABEL; } +"numgen" { return NUMGEN; } +"inc" { return INC; } +"mod" { return MOD; } + "dup" { return DUP; } "fwd" { return FWD; }
tests/py/ip/numgen.t new file +6 -0
@@ -0,0 +1,6 @@ +:pre;type nat hook prerouting priority 0 +*ip;test-ip4;pre + +ct mark set numgen inc mod 2;ok +dnat to numgen inc mod 2 map { 0 : 192.168.10.100, 1 : 192.168.20.200 };ok +dnat to numgen inc mod 10 map { 0-5 : 192.168.10.100, 6-9 : 192.168.20.200};ok
tests/py/ip/numgen.t.payload new file +24 -0
@@ -0,0 +1,24 @@ +# ct mark set numgen inc mod 2 +ip test-ip4 pre + [ numgen reg 1 = inc(2)] + [ ct set mark with reg 1 ] + +# dnat to numgen inc mod 2 map { 0 : 192.168.10.100, 1 : 192.168.20.200 } +__map%d x b +__map%d x 0 + element 00000000 : 640aa8c0 0 [end] element 01000000 : c814a8c0 0 [end] +ip test-ip4 pre + [ numgen reg 1 = inc(2)] + [ lookup reg 1 set __map%d dreg 1 ] + [ nat dnat ip addr_min reg 1 addr_max reg 0 ] + +# dnat to numgen inc mod 10 map { 0-5 : 192.168.10.100, 6-9 : 192.168.20.200} +__map%d test-ip4 f +__map%d test-ip4 0 + element 00000000 : 640aa8c0 0 [end] element 06000000 : c814a8c0 0 [end] element 0a000000 : 1 [end] +ip test-ip4 pre + [ numgen reg 1 = inc(10)] + [ byteorder reg 1 = hton(reg 1, 4, 4) ] + [ lookup reg 1 set __map%d dreg 1 ] + [ nat dnat ip addr_min reg 1 addr_max reg 0 ] +
Signature · 1 line
2.1.4
Pablo Neira Ayuso
maintainerPATCH 4/8 src: add hash expression
In reply to Pablo Neira AyusoThis is special expression that transforms an input expression into a 32-bit unsigned integer. This expression takes a modulus parameter to scale the result and the random seed so the hash result becomes harder to predict.
You can use it to set the packet mark, eg.
# nft add rule x y meta mark set jhash ip saddr . ip daddr mod 2 seed 0xdeadbeef
You can combine this with maps too, eg.
# nft add rule x y dnat to jhash ip saddr mod 2 seed 0xdeadbeef map { \
0 : 192.168.20.100, \
1 : 192.168.30.100 \
}Currently, this expression implements the jenkins hash implementation available in the Linux kernel:
http://lxr.free-electrons.com/source/include/linux/jhash.h
But it should be possible to extend it to support any other hash function type.
--- include/expression.h | 9 +++++++ include/hash.h | 7 ++++++ src/Makefile.am | 1 + src/evaluate.c | 21 ++++++++++++++++ src/hash.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++ src/netlink_delinearize.c | 32 +++++++++++++++++++++++++ src/netlink_linearize.c | 23 ++++++++++++++++++ src/parser_bison.y | 15 ++++++++++-- src/scanner.l | 3 +++ tests/py/ip/hash.t | 5 ++++ tests/py/ip/hash.t.payload | 17 +++++++++++++ 11 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 include/hash.h create mode 100644 src/hash.c create mode 100644 tests/py/ip/hash.t create mode 100644 tests/py/ip/hash.t.payload
include/expression.h +9 -0
@@ -34,6 +34,7 @@* @EXPR_BINOP: binary operations (bitwise, shifts) * @EXPR_RELATIONAL: equality and relational expressions * @EXPR_NUMGEN: number generation expression +* @EXPR_HASH: hash expression */ enum expr_types {EXPR_INVALID ,
@@ -57,6 +58,7 @@ enum expr_types {EXPR_BINOP ,EXPR_RELATIONAL ,EXPR_NUMGEN , + EXPR_HASH, }; enum ops {
@@ -174,6 +176,7 @@ enum expr_flags {#include <exthdr.h> #include <numgen.h> #include <meta.h> +#include <hash.h> #include <ct.h> /**
@@ -285,6 +288,12 @@ struct expr {enum nft_ng_types type;uint32_t mod; } numgen; +struct { +/* EXPR_HASH */ +struct expr *expr; +uint32_t mod; +uint32_t seed; + } hash; }; };
include/hash.h new file +7 -0
@@ -0,0 +1,7 @@ +#ifndef NFTABLES_HASH_H +#define NFTABLES_HASH_H + +extern struct expr *hash_expr_alloc (const struct location *loc, +uint32_t modulus,uint32_t seed); + +#endif /* NFTABLES_HASH_H */
src/Makefile.am +1 -0
@@ -36,6 +36,7 @@ nft_SOURCES = main.c \ proto.c \ payload.c \ exthdr.c \ + hash.c \ meta.c \ numgen.c \ ct.c \
src/evaluate.c +21 -0
@@ -1183,6 +1183,25 @@ static int expr_evaluate_numgen(struct eval_ctx *ctx, struct expr **exprp)return expr_evaluate_primary (ctx, exprp); } +static int expr_evaluate_hash (struct eval_ctx *ctx,struct expr **exprp) +{ +struct expr *expr = *exprp; + +expr_dtype_integer_compatible (ctx, expr); + +expr_set_context (&ctx->ectx,NULL ,0 ); +if (expr_evaluate (ctx, &expr->hash.expr) <0 ) +return -1 ; + +/* expr_evaluate_primary() sets the context to what to the input +* expression to be hashed. Since this input is transformed to a 4 bytes +* integer, restore context to the datatype that results from hashing. +*/ +expr_set_context (&ctx->ectx, expr->dtype, expr->len); + +return 0 ; +} +/* * Transfer the invertible binops to the constant side of an equality * expression. A left shift is only invertible if the low n bits are
@@ -1587,6 +1606,8 @@ static int expr_evaluate(struct eval_ctx *ctx, struct expr **expr)return expr_evaluate_relational (ctx, expr);case EXPR_NUMGEN :return expr_evaluate_numgen (ctx, expr); +case EXPR_HASH : +return expr_evaluate_hash (ctx, expr);default :BUG ("unknown expression type %s\n" , (*expr)->ops->name); }
src/hash.c new file +60 -0
60 lines not on this page · raw message
src/netlink_delinearize.c +32 -0
@@ -463,6 +463,34 @@ static void netlink_parse_exthdr(struct netlink_parse_ctx *ctx,netlink_set_register (ctx, dreg, expr); } +static void netlink_parse_hash (struct netlink_parse_ctx *ctx, +const struct location *loc, +const struct nftnl_expr *nle) +{ +enum nft_registers sreg, dreg; +struct expr *expr, *hexpr; +uint32_t mod, seed, len; + + sreg =netlink_parse_register (nle,NFTNL_EXPR_HASH_SREG ); + hexpr =netlink_get_register (ctx, loc, sreg); + + seed =nftnl_expr_get_u32 (nle,NFTNL_EXPR_HASH_SEED ); + mod =nftnl_expr_get_u32 (nle,NFTNL_EXPR_HASH_MODULUS ); + len =nftnl_expr_get_u32 (nle,NFTNL_EXPR_HASH_LEN ) *BITS_PER_BYTE ; + +if (hexpr->len < len) { + hexpr =netlink_parse_concat_expr (ctx, loc, sreg, len); +if (hexpr ==NULL ) +return ; + } + + expr =hash_expr_alloc (loc, mod, seed); + expr->hash.expr = hexpr; + + dreg =netlink_parse_register (nle,NFTNL_EXPR_HASH_DREG ); +netlink_set_register (ctx, dreg, expr); +} +static void netlink_parse_meta_expr (struct netlink_parse_ctx *ctx,const struct location *loc,const struct nftnl_expr *nle)
@@ -1020,6 +1048,7 @@ static const struct { { .name = "match", .parse = netlink_parse_match }, { .name = "quota", .parse = netlink_parse_quota }, { .name = "numgen", .parse = netlink_parse_numgen }, + { .name = "hash", .parse = netlink_parse_hash }, }; static int netlink_parse_expr(const struct nftnl_expr *nle,
@@ -1641,6 +1670,9 @@ static void expr_postprocess(struct rule_pp_ctx *ctx, struct expr **exprp)case EXPR_VERDICT :case EXPR_NUMGEN :break ; +case EXPR_HASH : +expr_postprocess (ctx, &expr->hash.expr); +break ;case EXPR_CT :ct_expr_update_type (&ctx->pctx, expr);break ;
src/netlink_linearize.c +23 -0
@@ -104,6 +104,27 @@ static void netlink_gen_concat(struct netlink_linearize_ctx *ctx, } } +static void netlink_gen_hash (struct netlink_linearize_ctx *ctx, +const struct expr *expr, +enum nft_registers dreg) +{ +enum nft_registers sreg; +struct nftnl_expr *nle; + + sreg =get_register (ctx, expr->hash.expr); +netlink_gen_expr (ctx, expr->hash.expr, sreg); +release_register (ctx, expr->hash.expr); + + nle =alloc_nft_expr ("hash" ); +netlink_put_register (nle,NFTNL_EXPR_HASH_SREG , sreg); +netlink_put_register (nle,NFTNL_EXPR_HASH_DREG , dreg); +nftnl_expr_set_u32 (nle,NFTNL_EXPR_HASH_LEN , +div_round_up (expr->hash.expr->len,BITS_PER_BYTE )); +nftnl_expr_set_u32 (nle,NFTNL_EXPR_HASH_MODULUS , expr->hash.mod); +nftnl_expr_set_u32 (nle,NFTNL_EXPR_HASH_SEED , expr->hash.seed); +nftnl_rule_add_expr (ctx->nlr, nle); +} +static void netlink_gen_payload (struct netlink_linearize_ctx *ctx,const struct expr *expr,enum nft_registers dreg)
@@ -629,6 +650,8 @@ static void netlink_gen_expr(struct netlink_linearize_ctx *ctx,return netlink_gen_expr (ctx, expr->key, dreg);case EXPR_NUMGEN :return netlink_gen_numgen (ctx, expr, dreg); +case EXPR_HASH : +return netlink_gen_hash (ctx, expr, dreg);default :BUG ("unknown expression type %s\n" , expr->ops->name); }
src/parser_bison.y +13 -2
@@ -411,6 +411,9 @@ static void location_update(struct location *loc, struct location *rhs, int n) %token INC "inc" %token MOD "mod" +%token JHASH "jhash" +%token SEED "seed" + %token POSITION "position" %token COMMENT "comment"
@@ -556,8 +559,8 @@ static void location_update(struct location *loc, struct location *rhs, int n) %type <expr> arp_hdr_expr %destructor { expr_free($$); } arp_hdr_expr %type <val> arp_hdr_field -%type <expr> ip_hdr_expr icmp_hdr_expr numgen_expr -%destructor { expr_free($$); } ip_hdr_expr icmp_hdr_expr numgen_expr +%type <expr> ip_hdr_expr icmp_hdr_expr numgen_expr hash_expr +%destructor { expr_free($$); } ip_hdr_expr icmp_hdr_expr numgen_expr hash_expr %type <val> ip_hdr_field icmp_hdr_field %type <expr> ip6_hdr_expr icmp6_hdr_expr %destructor { expr_free($$); } ip6_hdr_expr icmp6_hdr_expr
@@ -1972,6 +1975,7 @@ primary_expr : symbol_expr { $$ = $1; } | meta_expr { $$ = $1; } | ct_expr { $$ = $1; } | numgen_expr { $$ = $1; } + | hash_expr { $$ = $1; } | '(' basic_expr ')' { $$ = $2; } ;
@@ -2469,6 +2473,13 @@ numgen_expr : NUMGEN numgen_type MOD NUM } ; +hash_expr : JHASH expr MOD NUM SEED NUM + { + $$ = hash_expr_alloc(&@$, $4, $6); + $$->hash.expr = $2; + } + ; + ct_expr : CT ct_key { $$ = ct_expr_alloc(&@$, $2, -1);
src/scanner.l +3 -0
@@ -471,6 +471,9 @@ addrstring ({macaddr}|{ip4addr}|{ip6addr}) "inc" { return INC; } "mod" { return MOD; } +"jhash" { return JHASH; } +"seed" { return SEED; } + "dup" { return DUP; } "fwd" { return FWD; }
tests/py/ip/hash.t new file +5 -0
@@ -0,0 +1,5 @@ +:pre;type nat hook prerouting priority 0 +*ip;test-ip4;pre + +ct mark set jhash ip saddr . ip daddr mod 2 seed 0xdeadbeef;ok +dnat to jhash ip saddr mod 2 seed 0xdeadbeef map { 0 : 192.168.20.100, 1 : 192.168.30.100 };ok
tests/py/ip/hash.t.payload new file +17 -0
@@ -0,0 +1,17 @@ +# ct mark set jhash ip saddr . ip daddr mod 2 seed 0xdeadbeef +ip test-ip4 pre + [ payload load 4b @ network header + 12 => reg 2 ] + [ payload load 4b @ network header + 16 => reg 13 ] + [ hash reg 1 = jhash(reg 2, 8, 3735928559) % modulus 2] + [ ct set mark with reg 1 ] + +# dnat to jhash ip saddr mod 2 seed 0xdeadbeef map { 0 : 192.168.20.100, 1 : 192.168.30.100 } +__map%d test-ip4 b +__map%d test-ip4 0 + element 00000000 : 6414a8c0 0 [end] element 01000000 : 641ea8c0 0 [end] +ip test-ip4 pre + [ payload load 4b @ network header + 12 => reg 2 ] + [ hash reg 1 = jhash(reg 2, 4, 3735928559) % modulus 2] + [ lookup reg 1 set __map%d dreg 1 ] + [ nat dnat ip addr_min reg 1 addr_max reg 0 ] +
Signature · 1 line
2.1.4
Pablo Neira Ayuso
maintainerPATCH 5/8 evaluate: add expr_evaluate_integer()
In reply to Pablo Neira AyusoAdd a helper function to wrap the integer evaluation code.
--- src/evaluate.c | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-)
src/evaluate.c +23 -15
@@ -263,27 +263,35 @@ static int expr_evaluate_string(struct eval_ctx *ctx, struct expr **exprp)return 0 ; } -static int expr_evaluate_value (struct eval_ctx *ctx,struct expr **expr) +static int expr_evaluate_integer (struct eval_ctx *ctx,struct expr **exprp) { +struct expr *expr = *exprp;mpz_t mask; +mpz_init_bitmask (mask, ctx->ectx.len); +if (mpz_cmp (expr->value, mask) >0 ) { +char *valstr =mpz_get_str (NULL ,10 , expr->value); +char *rangestr =mpz_get_str (NULL ,10 , mask); +expr_error (ctx->msgs, expr, +"Value %s exceeds valid range 0-%s" , + valstr, rangestr); +free (valstr); +free (rangestr); +mpz_clear (mask); +return -1 ; + } + expr->byteorder = ctx->ectx.byteorder; + expr->len = ctx->ectx.len; +mpz_clear (mask); +return 0 ; +} + +static int expr_evaluate_value (struct eval_ctx *ctx,struct expr **expr) +{switch (expr_basetype (*expr)->type) {case TYPE_INTEGER : -mpz_init_bitmask (mask, ctx->ectx.len); -if (mpz_cmp ((*expr)->value, mask) >0 ) { -char *valstr =mpz_get_str (NULL ,10 , (*expr)->value); -char *rangestr =mpz_get_str (NULL ,10 , mask); -expr_error (ctx->msgs, *expr, -"Value %s exceeds valid range 0-%s" , - valstr, rangestr); -free (valstr); -free (rangestr); -mpz_clear (mask); +if (expr_evaluate_integer (ctx, expr) <0 )return -1 ; - } - (*expr)->byteorder = ctx->ectx.byteorder; - (*expr)->len = ctx->ectx.len; -mpz_clear (mask);break ;case TYPE_STRING :if (expr_evaluate_string (ctx, expr) <0 )
Signature · 1 line
2.1.4
Pablo Neira Ayuso
maintainerPATCH 6/8 evaluate: validate maximum hash and numgen value
In reply to Pablo Neira AyusoWe can validate that values don't get over the maximum datatype length, this is expressed in number of bits, so the maximum value is always power of 2.
However, since we got the hash and numgen expressions, the user should not set a value higher that what the specified modulus option, which may not be power of 2. This patch extends the expression context with a new optional field to store the maximum value.
After this patch, nft bails out if the user specifies non-sense rules like those below:
# nft add rule x y jhash ip saddr mod 10 seed 0xa 10
<cmdline>:1:45-46: Error: Value 10 exceeds valid range 0-9
add rule x y jhash ip saddr mod 10 seed 0xa 10
^^The modulus sets a valid value range of [0, n), so n is out of the valid value range.
# nft add rule x y numgen inc mod 10 eq 12
<cmdline>:1:35-36: Error: Value 12 exceeds valid range 0-9
add rule x y numgen inc mod 10 eq 12
^^--- include/expression.h | 8 ++++++-- src/evaluate.c | 28 ++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 8 deletions(-)
include/expression.h +6 -2
@@ -104,21 +104,24 @@ enum symbol_types {* @dtype: expected datatype * @byteorder: expected byteorder * @len: expected len +* @maxval: expected maximum value */ struct expr_ctx {const struct datatype *dtype;enum byteorder byteorder;unsigned int len; +unsigned int maxval; };static inline void __expr_set_context (struct expr_ctx *ctx,const struct datatype *dtype,enum byteorder byteorder, -unsigned int len) +unsigned int len,unsigned int maxval) { ctx->dtype = dtype; ctx->byteorder = byteorder; ctx->len = len; + ctx->maxval = maxval; } static inline void expr_set_context(struct expr_ctx *ctx,
@@ -126,7 +129,8 @@ static inline void expr_set_context(struct expr_ctx *ctx, unsigned int len) {__expr_set_context (ctx, dtype, - dtype ? dtype->byteorder :BYTEORDER_INVALID , len); + dtype ? dtype->byteorder :BYTEORDER_INVALID , + len,0 ); }/**
src/evaluate.c +22 -6
@@ -266,12 +266,23 @@ static int expr_evaluate_string(struct eval_ctx *ctx, struct expr **exprp)static int expr_evaluate_integer (struct eval_ctx *ctx,struct expr **exprp) {struct expr *expr = *exprp; +char *valstr, *rangestr;mpz_t mask; +if (ctx->ectx.maxval >0 && +mpz_cmp_ui (expr->value, ctx->ectx.maxval) >0 ) { + valstr =mpz_get_str (NULL ,10 , expr->value); +expr_error (ctx->msgs, expr, +"Value %s exceeds valid range 0-%u" , + valstr, ctx->ectx.maxval); +free (valstr); +return -1 ; + } +mpz_init_bitmask (mask, ctx->ectx.len);if (mpz_cmp (expr->value, mask) >0 ) { -char *valstr =mpz_get_str (NULL ,10 , expr->value); -char *rangestr =mpz_get_str (NULL ,10 , mask); + valstr =mpz_get_str (NULL ,10 , expr->value); + rangestr =mpz_get_str (NULL ,10 , mask);expr_error (ctx->msgs, expr,"Value %s exceeds valid range 0-%s" , valstr, rangestr);
@@ -309,7 +320,7 @@ static int expr_evaluate_value(struct eval_ctx *ctx, struct expr **expr)static int expr_evaluate_primary (struct eval_ctx *ctx,struct expr **expr) {__expr_set_context (&ctx->ectx, (*expr)->dtype, (*expr)->byteorder, - (*expr)->len); + (*expr)->len,0 );return 0 ; }
@@ -1186,9 +1197,13 @@ static void expr_dtype_integer_compatible(struct eval_ctx *ctx,static int expr_evaluate_numgen (struct eval_ctx *ctx,struct expr **exprp) { -expr_dtype_integer_compatible (ctx, *exprp); +struct expr *expr = *exprp; + +expr_dtype_integer_compatible (ctx, expr); -return expr_evaluate_primary (ctx, exprp); +__expr_set_context (&ctx->ectx, expr->dtype, expr->byteorder, expr->len, + expr->numgen.mod -1 ); +return 0 ; }static int expr_evaluate_hash (struct eval_ctx *ctx,struct expr **exprp)
@@ -1205,7 +1220,8 @@ static int expr_evaluate_hash(struct eval_ctx *ctx, struct expr **exprp)* expression to be hashed. Since this input is transformed to a 4 bytes * integer, restore context to the datatype that results from hashing. */ -expr_set_context (&ctx->ectx, expr->dtype, expr->len); +__expr_set_context (&ctx->ectx, expr->dtype, expr->byteorder, expr->len, + expr->hash.mod -1 );return 0 ; }
Signature · 1 line
2.1.4
Pablo Neira Ayuso
maintainerPATCH 7/8 parser_bison: add variable_expr rule
In reply to Pablo Neira AyusoThis patch adds a rule for variable expression so we can reuse it in a follow up patch to allow set element initialization from variable.
--- src/parser_bison.y | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-)
src/parser_bison.y +13 -10
@@ -496,8 +496,8 @@ static void location_update(struct location *loc, struct location *rhs, int n) %type <stmt> flow_stmt flow_stmt_alloc %destructor { stmt_free($$); } flow_stmt flow_stmt_alloc -%type <expr> symbol_expr verdict_expr integer_expr -%destructor { expr_free($$); } symbol_expr verdict_expr integer_expr +%type <expr> symbol_expr verdict_expr integer_expr variable_expr +%destructor { expr_free($$); } symbol_expr verdict_expr integer_expr variable_expr %type <expr> primary_expr shift_expr and_expr %destructor { expr_free($$); } primary_expr shift_expr and_expr %type <expr> exclusive_or_expr inclusive_or_expr
@@ -1927,14 +1927,7 @@ match_stmt : relational_expr } ; -symbol_expr : string - { - $$ = symbol_expr_alloc(&@$, SYMBOL_VALUE, - current_scope(state), - $1); - xfree($1); - } - | '$' identifier +variable_expr : '$' identifier { struct scope *scope = current_scope(state);
@@ -1948,6 +1941,16 @@ symbol_expr : string scope, $2); xfree($2); } + ; + +symbol_expr : variable_expr + | string + { + $$ = symbol_expr_alloc(&@$, SYMBOL_VALUE, + current_scope(state), + $1); + xfree($1); + } | AT identifier { $$ = symbol_expr_alloc(&@$, SYMBOL_SET,
Signature · 1 line
2.1.4
Pablo Neira Ayuso
maintainerPATCH 8/8 parser_bison: allow variable references in set elements definition
In reply to Pablo Neira AyusoAndreas reports that he cannot use variables in set definitions:
define s-ext-2-int = 10.10.10.10 . 25, 10.10.10.10 . 143
set s-ext-2-int {
type ipv4_addr . inet_service
elements = { $s-ext-2-int }
}This syntax is not correct though, since the curly braces should be placed in the variable definition itself, so we have context to handle this variable as a list of set elements.
The correct syntax that works after this patch is:
define s-ext-2-int = { 10.10.10.10 . 25, 10.10.10.10 . 143 }
table inet forward {
set s-ext-2-int {
type ipv4_addr . inet_service
elements = $s-ext-2-int
}
}--- src/parser_bison.y | 12 ++++++++---- tests/shell/testcases/nft-f/0009variable_0 | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) create mode 100755 tests/shell/testcases/nft-f/0009variable_0
src/parser_bison.y +8 -4
@@ -525,8 +525,8 @@ static void location_update(struct location *loc, struct location *rhs, int n) %type <expr> verdict_map_expr verdict_map_list_expr verdict_map_list_member_expr %destructor { expr_free($$); } verdict_map_expr verdict_map_list_expr verdict_map_list_member_expr -%type <expr> set_expr set_list_expr set_list_member_expr -%destructor { expr_free($$); } set_expr set_list_expr set_list_member_expr +%type <expr> set_expr set_block_expr set_list_expr set_list_member_expr +%destructor { expr_free($$); } set_expr set_block_expr set_list_expr set_list_member_expr %type <expr> set_elem_expr set_elem_expr_alloc set_lhs_expr set_rhs_expr %destructor { expr_free($$); } set_elem_expr set_elem_expr_alloc set_lhs_expr set_rhs_expr %type <expr> set_elem_expr_stmt set_elem_expr_stmt_alloc
@@ -1061,7 +1061,7 @@ set_block : /* empty */ { $$ = $<set>-1; } $1->gc_int = $3 * 1000; $$ = $1; } - | set_block ELEMENTS '=' set_expr + | set_block ELEMENTS '=' set_block_expr { $1->init = $4; $$ = $1;
@@ -1069,6 +1069,10 @@ set_block : /* empty */ { $$ = $<set>-1; } | set_block set_mechanism stmt_seperator ; +set_block_expr : set_expr + | variable_expr + ; + set_flag_list : set_flag_list COMMA set_flag { $$ = $1 | $3;
@@ -1104,7 +1108,7 @@ map_block : /* empty */ { $$ = $<set>-1; } $1->flags |= $3; $$ = $1; } - | map_block ELEMENTS '=' set_expr + | map_block ELEMENTS '=' set_block_expr { $1->init = $4; $$ = $1;
tests/shell/testcases/nft-f/0009variable_0 new file +23 -0
@@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +tmpfile=$(mktemp) +if [ ! -w $tmpfile ] ; then + echo "Failed to create tmp file" >&2 + exit 0 +fi + +trap "rm -rf $tmpfile" EXIT # cleanup if aborted + +RULESET="define concat-set-variable = { 10.10.10.10 . 25, 10.10.10.10 . 143 } + +table inet forward { + set concat-set-variable { + type ipv4_addr . inet_service + elements = \$concat-set-variable + } +}" + +echo "$RULESET" > $tmpfile +$NFT -f $tmpfile
Signature · 1 line
2.1.4