4 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
5 * Copyright (C) 2015 Huawei Inc.
9 #include <bpf/libbpf.h>
10 #include <linux/err.h>
11 #include <linux/string.h>
14 #include "bpf-loader.h"
15 #include "bpf-prologue.h"
16 #include "llvm-utils.h"
17 #include "probe-event.h"
18 #include "probe-finder.h" // for MAX_PROBES
19 #include "llvm-utils.h"
21 #define DEFINE_PRINT_FN(name, level) \
22 static int libbpf_##name(const char *fmt, ...) \
27 va_start(args, fmt); \
28 ret = veprintf(level, verbose, pr_fmt(fmt), args);\
33 DEFINE_PRINT_FN(warning, 1)
34 DEFINE_PRINT_FN(info, 1)
35 DEFINE_PRINT_FN(debug, 1)
37 struct bpf_prog_priv {
38 struct perf_probe_event pev;
40 struct bpf_insn *insns_buf;
43 static bool libbpf_initialized;
46 bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name)
48 struct bpf_object *obj;
50 if (!libbpf_initialized) {
51 libbpf_set_print(libbpf_warning,
54 libbpf_initialized = true;
57 obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, name);
59 pr_debug("bpf: failed to load buffer\n");
60 return ERR_PTR(-EINVAL);
66 struct bpf_object *bpf__prepare_load(const char *filename, bool source)
68 struct bpf_object *obj;
70 if (!libbpf_initialized) {
71 libbpf_set_print(libbpf_warning,
74 libbpf_initialized = true;
82 err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz);
84 return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE);
85 obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, filename);
88 obj = bpf_object__open(filename);
91 pr_debug("bpf: failed to load %s\n", filename);
100 struct bpf_object *obj, *tmp;
102 bpf_object__for_each_safe(obj, tmp) {
104 bpf_object__close(obj);
109 bpf_prog_priv__clear(struct bpf_program *prog __maybe_unused,
112 struct bpf_prog_priv *priv = _priv;
114 cleanup_perf_probe_events(&priv->pev, 1);
115 zfree(&priv->insns_buf);
120 config__exec(const char *value, struct perf_probe_event *pev)
123 pev->target = strdup(value);
130 config__module(const char *value, struct perf_probe_event *pev)
132 pev->uprobes = false;
133 pev->target = strdup(value);
140 config__bool(const char *value,
141 bool *pbool, bool invert)
149 err = strtobool(value, &bool_value);
153 *pbool = invert ? !bool_value : bool_value;
158 config__inlines(const char *value,
159 struct perf_probe_event *pev __maybe_unused)
161 return config__bool(value, &probe_conf.no_inlines, true);
165 config__force(const char *value,
166 struct perf_probe_event *pev __maybe_unused)
168 return config__bool(value, &probe_conf.force_add, false);
175 int (*func)(const char *, struct perf_probe_event *);
176 } bpf_config_terms[] = {
179 .usage = "exec=<full path of file>",
180 .desc = "Set uprobe target",
181 .func = config__exec,
185 .usage = "module=<module name> ",
186 .desc = "Set kprobe module",
187 .func = config__module,
191 .usage = "inlines=[yes|no] ",
192 .desc = "Probe at inline symbol",
193 .func = config__inlines,
197 .usage = "force=[yes|no] ",
198 .desc = "Forcibly add events with existing name",
199 .func = config__force,
204 do_config(const char *key, const char *value,
205 struct perf_probe_event *pev)
209 pr_debug("config bpf program: %s=%s\n", key, value);
210 for (i = 0; i < ARRAY_SIZE(bpf_config_terms); i++)
211 if (strcmp(key, bpf_config_terms[i].key) == 0)
212 return bpf_config_terms[i].func(value, pev);
214 pr_debug("BPF: ERROR: invalid config option in object: %s=%s\n",
217 pr_debug("\nHint: Currently valid options are:\n");
218 for (i = 0; i < ARRAY_SIZE(bpf_config_terms); i++)
219 pr_debug("\t%s:\t%s\n", bpf_config_terms[i].usage,
220 bpf_config_terms[i].desc);
223 return -BPF_LOADER_ERRNO__CONFIG_TERM;
227 parse_config_kvpair(const char *config_str, struct perf_probe_event *pev)
229 char *text = strdup(config_str);
231 const char *main_str = NULL;
235 pr_debug("No enough memory: dup config_str failed\n");
236 return ERR_PTR(-ENOMEM);
240 while ((sep = strchr(line, ';'))) {
244 equ = strchr(line, '=');
246 pr_warning("WARNING: invalid config in BPF object: %s\n",
248 pr_warning("\tShould be 'key=value'.\n");
253 err = do_config(line, equ + 1, pev);
261 main_str = config_str + (line - text);
264 return err ? ERR_PTR(err) : main_str;
268 parse_config(const char *config_str, struct perf_probe_event *pev)
271 const char *main_str = parse_config_kvpair(config_str, pev);
273 if (IS_ERR(main_str))
274 return PTR_ERR(main_str);
276 err = parse_perf_probe_command(main_str, pev);
278 pr_debug("bpf: '%s' is not a valid config string\n",
280 /* parse failed, don't need clear pev. */
281 return -BPF_LOADER_ERRNO__CONFIG;
287 config_bpf_program(struct bpf_program *prog)
289 struct perf_probe_event *pev = NULL;
290 struct bpf_prog_priv *priv = NULL;
291 const char *config_str;
294 /* Initialize per-program probing setting */
295 probe_conf.no_inlines = false;
296 probe_conf.force_add = false;
298 config_str = bpf_program__title(prog, false);
299 if (IS_ERR(config_str)) {
300 pr_debug("bpf: unable to get title for program\n");
301 return PTR_ERR(config_str);
304 priv = calloc(sizeof(*priv), 1);
306 pr_debug("bpf: failed to alloc priv\n");
311 pr_debug("bpf: config program '%s'\n", config_str);
312 err = parse_config(config_str, pev);
316 if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) {
317 pr_debug("bpf: '%s': group for event is set and not '%s'.\n",
318 config_str, PERF_BPF_PROBE_GROUP);
319 err = -BPF_LOADER_ERRNO__GROUP;
321 } else if (!pev->group)
322 pev->group = strdup(PERF_BPF_PROBE_GROUP);
325 pr_debug("bpf: strdup failed\n");
331 pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n",
333 err = -BPF_LOADER_ERRNO__EVENTNAME;
336 pr_debug("bpf: config '%s' is ok\n", config_str);
338 err = bpf_program__set_private(prog, priv, bpf_prog_priv__clear);
340 pr_debug("Failed to set priv for program '%s'\n", config_str);
348 clear_perf_probe_event(pev);
353 static int bpf__prepare_probe(void)
356 static bool initialized = false;
359 * Make err static, so if init failed the first, bpf__prepare_probe()
360 * fails each time without calling init_probe_symbol_maps multiple
367 err = init_probe_symbol_maps(false);
369 pr_debug("Failed to init_probe_symbol_maps\n");
370 probe_conf.max_probes = MAX_PROBES;
375 preproc_gen_prologue(struct bpf_program *prog, int n,
376 struct bpf_insn *orig_insns, int orig_insns_cnt,
377 struct bpf_prog_prep_result *res)
379 struct probe_trace_event *tev;
380 struct perf_probe_event *pev;
381 struct bpf_prog_priv *priv;
382 struct bpf_insn *buf;
383 size_t prologue_cnt = 0;
386 err = bpf_program__get_private(prog, (void **)&priv);
392 if (n < 0 || n >= pev->ntevs)
397 buf = priv->insns_buf;
398 err = bpf__gen_prologue(tev->args, tev->nargs,
400 BPF_MAXINSNS - orig_insns_cnt);
404 title = bpf_program__title(prog, false);
408 pr_debug("Failed to generate prologue for program %s\n",
413 memcpy(&buf[prologue_cnt], orig_insns,
414 sizeof(struct bpf_insn) * orig_insns_cnt);
416 res->new_insn_ptr = buf;
417 res->new_insn_cnt = prologue_cnt + orig_insns_cnt;
422 pr_debug("Internal error in preproc_gen_prologue\n");
423 return -BPF_LOADER_ERRNO__PROLOGUE;
426 static int hook_load_preprocessor(struct bpf_program *prog)
428 struct perf_probe_event *pev;
429 struct bpf_prog_priv *priv;
430 bool need_prologue = false;
433 err = bpf_program__get_private(prog, (void **)&priv);
435 pr_debug("Internal error when hook preprocessor\n");
436 return -BPF_LOADER_ERRNO__INTERNAL;
440 for (i = 0; i < pev->ntevs; i++) {
441 struct probe_trace_event *tev = &pev->tevs[i];
443 if (tev->nargs > 0) {
444 need_prologue = true;
450 * Since all tevs don't have argument, we don't need generate
453 if (!need_prologue) {
454 priv->need_prologue = false;
458 priv->need_prologue = true;
459 priv->insns_buf = malloc(sizeof(struct bpf_insn) * BPF_MAXINSNS);
460 if (!priv->insns_buf) {
461 pr_debug("No enough memory: alloc insns_buf failed\n");
465 err = bpf_program__set_prep(prog, pev->ntevs,
466 preproc_gen_prologue);
470 int bpf__probe(struct bpf_object *obj)
473 struct bpf_program *prog;
474 struct bpf_prog_priv *priv;
475 struct perf_probe_event *pev;
477 err = bpf__prepare_probe();
479 pr_debug("bpf__prepare_probe failed\n");
483 bpf_object__for_each_program(prog, obj) {
484 err = config_bpf_program(prog);
488 err = bpf_program__get_private(prog, (void **)&priv);
493 err = convert_perf_probe_events(pev, 1);
495 pr_debug("bpf_probe: failed to convert perf probe events");
499 err = apply_perf_probe_events(pev, 1);
501 pr_debug("bpf_probe: failed to apply perf probe events");
506 * After probing, let's consider prologue, which
507 * adds program fetcher to BPF programs.
509 * hook_load_preprocessorr() hooks pre-processor
510 * to bpf_program, let it generate prologue
511 * dynamically during loading.
513 err = hook_load_preprocessor(prog);
518 return err < 0 ? err : 0;
521 #define EVENTS_WRITE_BUFSIZE 4096
522 int bpf__unprobe(struct bpf_object *obj)
525 struct bpf_program *prog;
526 struct bpf_prog_priv *priv;
528 bpf_object__for_each_program(prog, obj) {
531 err = bpf_program__get_private(prog, (void **)&priv);
535 for (i = 0; i < priv->pev.ntevs; i++) {
536 struct probe_trace_event *tev = &priv->pev.tevs[i];
537 char name_buf[EVENTS_WRITE_BUFSIZE];
538 struct strfilter *delfilter;
540 snprintf(name_buf, EVENTS_WRITE_BUFSIZE,
541 "%s:%s", tev->group, tev->event);
542 name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0';
544 delfilter = strfilter__new(name_buf, NULL);
546 pr_debug("Failed to create filter for unprobing\n");
551 err = del_perf_probe_events(delfilter);
552 strfilter__delete(delfilter);
554 pr_debug("Failed to delete %s\n", name_buf);
563 int bpf__load(struct bpf_object *obj)
567 err = bpf_object__load(obj);
569 pr_debug("bpf: load objects failed\n");
575 int bpf__foreach_tev(struct bpf_object *obj,
576 bpf_prog_iter_callback_t func,
579 struct bpf_program *prog;
582 bpf_object__for_each_program(prog, obj) {
583 struct probe_trace_event *tev;
584 struct perf_probe_event *pev;
585 struct bpf_prog_priv *priv;
588 err = bpf_program__get_private(prog,
591 pr_debug("bpf: failed to get private field\n");
592 return -BPF_LOADER_ERRNO__INTERNAL;
596 for (i = 0; i < pev->ntevs; i++) {
599 if (priv->need_prologue)
600 fd = bpf_program__nth_fd(prog, i);
602 fd = bpf_program__fd(prog);
605 pr_debug("bpf: failed to get file descriptor\n");
609 err = (*func)(tev, fd, arg);
611 pr_debug("bpf: call back failed, stop iterate\n");
619 #define ERRNO_OFFSET(e) ((e) - __BPF_LOADER_ERRNO__START)
620 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(BPF_LOADER_ERRNO__##c)
621 #define NR_ERRNO (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
623 static const char *bpf_loader_strerror_table[NR_ERRNO] = {
624 [ERRCODE_OFFSET(CONFIG)] = "Invalid config string",
625 [ERRCODE_OFFSET(GROUP)] = "Invalid group name",
626 [ERRCODE_OFFSET(EVENTNAME)] = "No event name found in config string",
627 [ERRCODE_OFFSET(INTERNAL)] = "BPF loader internal error",
628 [ERRCODE_OFFSET(COMPILE)] = "Error when compiling BPF scriptlet",
629 [ERRCODE_OFFSET(CONFIG_TERM)] = "Invalid config term in config string",
630 [ERRCODE_OFFSET(PROLOGUE)] = "Failed to generate prologue",
631 [ERRCODE_OFFSET(PROLOGUE2BIG)] = "Prologue too big for program",
632 [ERRCODE_OFFSET(PROLOGUEOOB)] = "Offset out of bound for prologue",
636 bpf_loader_strerror(int err, char *buf, size_t size)
638 char sbuf[STRERR_BUFSIZE];
644 err = err > 0 ? err : -err;
646 if (err >= __LIBBPF_ERRNO__START)
647 return libbpf_strerror(err, buf, size);
649 if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) {
650 msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)];
651 snprintf(buf, size, "%s", msg);
652 buf[size - 1] = '\0';
656 if (err >= __BPF_LOADER_ERRNO__END)
657 snprintf(buf, size, "Unknown bpf loader error %d", err);
659 snprintf(buf, size, "%s",
660 strerror_r(err, sbuf, sizeof(sbuf)));
662 buf[size - 1] = '\0';
666 #define bpf__strerror_head(err, buf, size) \
667 char sbuf[STRERR_BUFSIZE], *emsg;\
672 bpf_loader_strerror(err, sbuf, sizeof(sbuf));\
676 scnprintf(buf, size, "%s", emsg);\
679 #define bpf__strerror_entry(val, fmt...)\
681 scnprintf(buf, size, fmt);\
685 #define bpf__strerror_end(buf, size)\
687 buf[size - 1] = '\0';
689 int bpf__strerror_prepare_load(const char *filename, bool source,
690 int err, char *buf, size_t size)
695 n = snprintf(buf, size, "Failed to load %s%s: ",
696 filename, source ? " from source" : "");
698 buf[size - 1] = '\0';
704 ret = bpf_loader_strerror(err, buf, size);
705 buf[size - 1] = '\0';
709 int bpf__strerror_probe(struct bpf_object *obj __maybe_unused,
710 int err, char *buf, size_t size)
712 bpf__strerror_head(err, buf, size);
713 case BPF_LOADER_ERRNO__CONFIG_TERM: {
714 scnprintf(buf, size, "%s (add -v to see detail)", emsg);
717 bpf__strerror_entry(EEXIST, "Probe point exist. Try 'perf probe -d \"*\"' and set 'force=yes'");
718 bpf__strerror_entry(EACCES, "You need to be root");
719 bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0");
720 bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file");
721 bpf__strerror_end(buf, size);
725 int bpf__strerror_load(struct bpf_object *obj,
726 int err, char *buf, size_t size)
728 bpf__strerror_head(err, buf, size);
729 case LIBBPF_ERRNO__KVER: {
730 unsigned int obj_kver = bpf_object__get_kversion(obj);
731 unsigned int real_kver;
733 if (fetch_kernel_version(&real_kver, NULL, 0)) {
734 scnprintf(buf, size, "Unable to fetch kernel version");
738 if (obj_kver != real_kver) {
740 "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")",
741 KVER_PARAM(obj_kver),
742 KVER_PARAM(real_kver));
746 scnprintf(buf, size, "Failed to load program for unknown reason");
749 bpf__strerror_end(buf, size);