net/sched/cls_bpf.c

   1 /*
   2  * Berkeley Packet Filter based traffic classifier
   3  *
   4  * Might be used to classify traffic through flexible, user-defined and
   5  * possibly JIT-ed BPF filters for traffic control as an alternative to
   6  * ematches.
   7  *
   8  * (C) 2013 Daniel Borkmann <dborkman@redhat.com>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License version 2 as
  12  * published by the Free Software Foundation.
  13  */
  14
  15 #include <linux/module.h>
  16 #include <linux/types.h>
  17 #include <linux/skbuff.h>
  18 #include <linux/filter.h>
  19 #include <linux/bpf.h>
  20
  21 #include <net/rtnetlink.h>
  22 #include <net/pkt_cls.h>
  23 #include <net/sock.h>
  24
  25 MODULE_LICENSE("GPL");
  26 MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>");
  27 MODULE_DESCRIPTION("TC BPF based classifier");
  28
  29 #define CLS_BPF_NAME_LEN        256
  30
  31 struct cls_bpf_head {
  32         struct list_head plist;
  33         u32 hgen;
  34         struct rcu_head rcu;
  35 };
  36
  37 struct cls_bpf_prog {
  38         struct bpf_prog *filter;
  39         struct list_head link;
  40         struct tcf_result res;
  41         struct tcf_exts exts;
  42         u32 handle;
  43         union {
  44                 u32 bpf_fd;
  45                 u16 bpf_num_ops;
  46         };
  47         struct sock_filter *bpf_ops;
  48         const char *bpf_name;
  49         struct tcf_proto *tp;
  50         struct rcu_head rcu;
  51 };
  52
  53 static const struct nla_policy bpf_policy[TCA_BPF_MAX + 1] = {
  54         [TCA_BPF_CLASSID]       = { .type = NLA_U32 },
  55         [TCA_BPF_FD]            = { .type = NLA_U32 },
  56         [TCA_BPF_NAME]          = { .type = NLA_NUL_STRING, .len = CLS_BPF_NAME_LEN },
  57         [TCA_BPF_OPS_LEN]       = { .type = NLA_U16 },
  58         [TCA_BPF_OPS]           = { .type = NLA_BINARY,
  59                                     .len = sizeof(struct sock_filter) * BPF_MAXINSNS },
  60 };
  61
  62 static int cls_bpf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
  63                             struct tcf_result *res)
  64 {
  65         struct cls_bpf_head *head = rcu_dereference_bh(tp->root);
  66         struct cls_bpf_prog *prog;
  67         int ret = -1;
  68
  69         /* Needed here for accessing maps. */
  70         rcu_read_lock();
  71         list_for_each_entry_rcu(prog, &head->plist, link) {
  72                 int filter_res = BPF_PROG_RUN(prog->filter, skb);
  73
  74                 if (filter_res == 0)
  75                         continue;
  76
  77                 *res = prog->res;
  78                 if (filter_res != -1)
  79                         res->classid = filter_res;
  80
  81                 ret = tcf_exts_exec(skb, &prog->exts, res);
  82                 if (ret < 0)
  83                         continue;
  84
  85                 break;
  86         }
  87         rcu_read_unlock();
  88
  89         return ret;
  90 }
  91
  92 static bool cls_bpf_is_ebpf(const struct cls_bpf_prog *prog)
  93 {
  94         return !prog->bpf_ops;
  95 }
  96
  97 static int cls_bpf_init(struct tcf_proto *tp)
  98 {
  99         struct cls_bpf_head *head;
 100
 101         head = kzalloc(sizeof(*head), GFP_KERNEL);
 102         if (head == NULL)
 103                 return -ENOBUFS;
 104
 105         INIT_LIST_HEAD_RCU(&head->plist);
 106         rcu_assign_pointer(tp->root, head);
 107
 108         return 0;
 109 }
 110
 111 static void cls_bpf_delete_prog(struct tcf_proto *tp, struct cls_bpf_prog *prog)
 112 {
 113         tcf_exts_destroy(&prog->exts);
 114
 115         if (cls_bpf_is_ebpf(prog))
 116                 bpf_prog_put(prog->filter);
 117         else
 118                 bpf_prog_destroy(prog->filter);
 119
 120         kfree(prog->bpf_name);
 121         kfree(prog->bpf_ops);
 122         kfree(prog);
 123 }
 124
 125 static void __cls_bpf_delete_prog(struct rcu_head *rcu)
 126 {
 127         struct cls_bpf_prog *prog = container_of(rcu, struct cls_bpf_prog, rcu);
 128
 129         cls_bpf_delete_prog(prog->tp, prog);
 130 }
 131
 132 static int cls_bpf_delete(struct tcf_proto *tp, unsigned long arg)
 133 {
 134         struct cls_bpf_prog *prog = (struct cls_bpf_prog *) arg;
 135
 136         list_del_rcu(&prog->link);
 137         tcf_unbind_filter(tp, &prog->res);
 138         call_rcu(&prog->rcu, __cls_bpf_delete_prog);
 139
 140         return 0;
 141 }
 142
 143 static bool cls_bpf_destroy(struct tcf_proto *tp, bool force)
 144 {
 145         struct cls_bpf_head *head = rtnl_dereference(tp->root);
 146         struct cls_bpf_prog *prog, *tmp;
 147
 148         if (!force && !list_empty(&head->plist))
 149                 return false;
 150
 151         list_for_each_entry_safe(prog, tmp, &head->plist, link) {
 152                 list_del_rcu(&prog->link);
 153                 tcf_unbind_filter(tp, &prog->res);
 154                 call_rcu(&prog->rcu, __cls_bpf_delete_prog);
 155         }
 156
 157         RCU_INIT_POINTER(tp->root, NULL);
 158         kfree_rcu(head, rcu);
 159         return true;
 160 }
 161
 162 static unsigned long cls_bpf_get(struct tcf_proto *tp, u32 handle)
 163 {
 164         struct cls_bpf_head *head = rtnl_dereference(tp->root);
 165         struct cls_bpf_prog *prog;
 166         unsigned long ret = 0UL;
 167
 168         if (head == NULL)
 169                 return 0UL;
 170
 171         list_for_each_entry(prog, &head->plist, link) {
 172                 if (prog->handle == handle) {
 173                         ret = (unsigned long) prog;
 174                         break;
 175                 }
 176         }
 177
 178         return ret;
 179 }
 180
 181 static int cls_bpf_prog_from_ops(struct nlattr **tb,
 182                                  struct cls_bpf_prog *prog, u32 classid)
 183 {
 184         struct sock_filter *bpf_ops;
 185         struct sock_fprog_kern fprog_tmp;
 186         struct bpf_prog *fp;
 187         u16 bpf_size, bpf_num_ops;
 188         int ret;
 189
 190         bpf_num_ops = nla_get_u16(tb[TCA_BPF_OPS_LEN]);
 191         if (bpf_num_ops > BPF_MAXINSNS || bpf_num_ops == 0)
 192                 return -EINVAL;
 193
 194         bpf_size = bpf_num_ops * sizeof(*bpf_ops);
 195         if (bpf_size != nla_len(tb[TCA_BPF_OPS]))
 196                 return -EINVAL;
 197
 198         bpf_ops = kzalloc(bpf_size, GFP_KERNEL);
 199         if (bpf_ops == NULL)
 200                 return -ENOMEM;
 201
 202         memcpy(bpf_ops, nla_data(tb[TCA_BPF_OPS]), bpf_size);
 203
 204         fprog_tmp.len = bpf_num_ops;
 205         fprog_tmp.filter = bpf_ops;
 206
 207         ret = bpf_prog_create(&fp, &fprog_tmp);
 208         if (ret < 0) {
 209                 kfree(bpf_ops);
 210                 return ret;
 211         }
 212
 213         prog->bpf_ops = bpf_ops;
 214         prog->bpf_num_ops = bpf_num_ops;
 215         prog->bpf_name = NULL;
 216
 217         prog->filter = fp;
 218         prog->res.classid = classid;
 219
 220         return 0;
 221 }
 222
 223 static int cls_bpf_prog_from_efd(struct nlattr **tb,
 224                                  struct cls_bpf_prog *prog, u32 classid)
 225 {
 226         struct bpf_prog *fp;
 227         char *name = NULL;
 228         u32 bpf_fd;
 229
 230         bpf_fd = nla_get_u32(tb[TCA_BPF_FD]);
 231
 232         fp = bpf_prog_get(bpf_fd);
 233         if (IS_ERR(fp))
 234                 return PTR_ERR(fp);
 235
 236         if (fp->type != BPF_PROG_TYPE_SCHED_CLS) {
 237                 bpf_prog_put(fp);
 238                 return -EINVAL;
 239         }
 240
 241         if (tb[TCA_BPF_NAME]) {
 242                 name = kmemdup(nla_data(tb[TCA_BPF_NAME]),
 243                                nla_len(tb[TCA_BPF_NAME]),
 244                                GFP_KERNEL);
 245                 if (!name) {
 246                         bpf_prog_put(fp);
 247                         return -ENOMEM;
 248                 }
 249         }
 250
 251         prog->bpf_ops = NULL;
 252         prog->bpf_fd = bpf_fd;
 253         prog->bpf_name = name;
 254
 255         prog->filter = fp;
 256         prog->res.classid = classid;
 257
 258         return 0;
 259 }
 260
 261 static int cls_bpf_modify_existing(struct net *net, struct tcf_proto *tp,
 262                                    struct cls_bpf_prog *prog,
 263                                    unsigned long base, struct nlattr **tb,
 264                                    struct nlattr *est, bool ovr)
 265 {
 266         struct tcf_exts exts;
 267         bool is_bpf, is_ebpf;
 268         u32 classid;
 269         int ret;
 270
 271         is_bpf = tb[TCA_BPF_OPS_LEN] && tb[TCA_BPF_OPS];
 272         is_ebpf = tb[TCA_BPF_FD];
 273
 274         if ((!is_bpf && !is_ebpf) || (is_bpf && is_ebpf) ||
 275             !tb[TCA_BPF_CLASSID])
 276                 return -EINVAL;
 277
 278         tcf_exts_init(&exts, TCA_BPF_ACT, TCA_BPF_POLICE);
 279         ret = tcf_exts_validate(net, tp, tb, est, &exts, ovr);
 280         if (ret < 0)
 281                 return ret;
 282
 283         classid = nla_get_u32(tb[TCA_BPF_CLASSID]);
 284
 285         ret = is_bpf ? cls_bpf_prog_from_ops(tb, prog, classid) :
 286                        cls_bpf_prog_from_efd(tb, prog, classid);
 287         if (ret < 0) {
 288                 tcf_exts_destroy(&exts);
 289                 return ret;
 290         }
 291
 292         tcf_bind_filter(tp, &prog->res, base);
 293         tcf_exts_change(tp, &prog->exts, &exts);
 294
 295         return 0;
 296 }
 297
 298 static u32 cls_bpf_grab_new_handle(struct tcf_proto *tp,
 299                                    struct cls_bpf_head *head)
 300 {
 301         unsigned int i = 0x80000000;
 302         u32 handle;
 303
 304         do {
 305                 if (++head->hgen == 0x7FFFFFFF)
 306                         head->hgen = 1;
 307         } while (--i > 0 && cls_bpf_get(tp, head->hgen));
 308
 309         if (unlikely(i == 0)) {
 310                 pr_err("Insufficient number of handles\n");
 311                 handle = 0;
 312         } else {
 313                 handle = head->hgen;
 314         }
 315
 316         return handle;
 317 }
 318
 319 static int cls_bpf_change(struct net *net, struct sk_buff *in_skb,
 320                           struct tcf_proto *tp, unsigned long base,
 321                           u32 handle, struct nlattr **tca,
 322                           unsigned long *arg, bool ovr)
 323 {
 324         struct cls_bpf_head *head = rtnl_dereference(tp->root);
 325         struct cls_bpf_prog *oldprog = (struct cls_bpf_prog *) *arg;
 326         struct nlattr *tb[TCA_BPF_MAX + 1];
 327         struct cls_bpf_prog *prog;
 328         int ret;
 329
 330         if (tca[TCA_OPTIONS] == NULL)
 331                 return -EINVAL;
 332
 333         ret = nla_parse_nested(tb, TCA_BPF_MAX, tca[TCA_OPTIONS], bpf_policy);
 334         if (ret < 0)
 335                 return ret;
 336
 337         prog = kzalloc(sizeof(*prog), GFP_KERNEL);
 338         if (!prog)
 339                 return -ENOBUFS;
 340
 341         tcf_exts_init(&prog->exts, TCA_BPF_ACT, TCA_BPF_POLICE);
 342
 343         if (oldprog) {
 344                 if (handle && oldprog->handle != handle) {
 345                         ret = -EINVAL;
 346                         goto errout;
 347                 }
 348         }
 349
 350         if (handle == 0)
 351                 prog->handle = cls_bpf_grab_new_handle(tp, head);
 352         else
 353                 prog->handle = handle;
 354         if (prog->handle == 0) {
 355                 ret = -EINVAL;
 356                 goto errout;
 357         }
 358
 359         ret = cls_bpf_modify_existing(net, tp, prog, base, tb, tca[TCA_RATE], ovr);
 360         if (ret < 0)
 361                 goto errout;
 362
 363         if (oldprog) {
 364                 list_replace_rcu(&prog->link, &oldprog->link);
 365                 tcf_unbind_filter(tp, &oldprog->res);
 366                 call_rcu(&oldprog->rcu, __cls_bpf_delete_prog);
 367         } else {
 368                 list_add_rcu(&prog->link, &head->plist);
 369         }
 370
 371         *arg = (unsigned long) prog;
 372         return 0;
 373 errout:
 374         kfree(prog);
 375
 376         return ret;
 377 }
 378
 379 static int cls_bpf_dump_bpf_info(const struct cls_bpf_prog *prog,
 380                                  struct sk_buff *skb)
 381 {
 382         struct nlattr *nla;
 383
 384         if (nla_put_u16(skb, TCA_BPF_OPS_LEN, prog->bpf_num_ops))
 385                 return -EMSGSIZE;
 386
 387         nla = nla_reserve(skb, TCA_BPF_OPS, prog->bpf_num_ops *
 388                           sizeof(struct sock_filter));
 389         if (nla == NULL)
 390                 return -EMSGSIZE;
 391
 392         memcpy(nla_data(nla), prog->bpf_ops, nla_len(nla));
 393
 394         return 0;
 395 }
 396
 397 static int cls_bpf_dump_ebpf_info(const struct cls_bpf_prog *prog,
 398                                   struct sk_buff *skb)
 399 {
 400         if (nla_put_u32(skb, TCA_BPF_FD, prog->bpf_fd))
 401                 return -EMSGSIZE;
 402
 403         if (prog->bpf_name &&
 404             nla_put_string(skb, TCA_BPF_NAME, prog->bpf_name))
 405                 return -EMSGSIZE;
 406
 407         return 0;
 408 }
 409
 410 static int cls_bpf_dump(struct net *net, struct tcf_proto *tp, unsigned long fh,
 411                         struct sk_buff *skb, struct tcmsg *tm)
 412 {
 413         struct cls_bpf_prog *prog = (struct cls_bpf_prog *) fh;
 414         struct nlattr *nest;
 415         int ret;
 416
 417         if (prog == NULL)
 418                 return skb->len;
 419
 420         tm->tcm_handle = prog->handle;
 421
 422         nest = nla_nest_start(skb, TCA_OPTIONS);
 423         if (nest == NULL)
 424                 goto nla_put_failure;
 425
 426         if (nla_put_u32(skb, TCA_BPF_CLASSID, prog->res.classid))
 427                 goto nla_put_failure;
 428
 429         if (cls_bpf_is_ebpf(prog))
 430                 ret = cls_bpf_dump_ebpf_info(prog, skb);
 431         else
 432                 ret = cls_bpf_dump_bpf_info(prog, skb);
 433         if (ret)
 434                 goto nla_put_failure;
 435
 436         if (tcf_exts_dump(skb, &prog->exts) < 0)
 437                 goto nla_put_failure;
 438
 439         nla_nest_end(skb, nest);
 440
 441         if (tcf_exts_dump_stats(skb, &prog->exts) < 0)
 442                 goto nla_put_failure;
 443
 444         return skb->len;
 445
 446 nla_put_failure:
 447         nla_nest_cancel(skb, nest);
 448         return -1;
 449 }
 450
 451 static void cls_bpf_walk(struct tcf_proto *tp, struct tcf_walker *arg)
 452 {
 453         struct cls_bpf_head *head = rtnl_dereference(tp->root);
 454         struct cls_bpf_prog *prog;
 455
 456         list_for_each_entry(prog, &head->plist, link) {
 457                 if (arg->count < arg->skip)
 458                         goto skip;
 459                 if (arg->fn(tp, (unsigned long) prog, arg) < 0) {
 460                         arg->stop = 1;
 461                         break;
 462                 }
 463 skip:
 464                 arg->count++;
 465         }
 466 }
 467
 468 static struct tcf_proto_ops cls_bpf_ops __read_mostly = {
 469         .kind           =       "bpf",
 470         .owner          =       THIS_MODULE,
 471         .classify       =       cls_bpf_classify,
 472         .init           =       cls_bpf_init,
 473         .destroy        =       cls_bpf_destroy,
 474         .get            =       cls_bpf_get,
 475         .change         =       cls_bpf_change,
 476         .delete         =       cls_bpf_delete,
 477         .walk           =       cls_bpf_walk,
 478         .dump           =       cls_bpf_dump,
 479 };
 480
 481 static int __init cls_bpf_init_mod(void)
 482 {
 483         return register_tcf_proto_ops(&cls_bpf_ops);
 484 }
 485
 486 static void __exit cls_bpf_exit_mod(void)
 487 {
 488         unregister_tcf_proto_ops(&cls_bpf_ops);
 489 }
 490
 491 module_init(cls_bpf_init_mod);
 492 module_exit(cls_bpf_exit_mod);