diff --git a/src/bpf_gen_internal.h b/src/bpf_gen_internal.h index 6c5ad6c..206adf2 100644 --- a/src/bpf_gen_internal.h +++ b/src/bpf_gen_internal.h @@ -51,9 +51,17 @@ struct bpf_gen { __u32 nr_ksyms; int fd_array; int nr_fd_array; + /* + * Maps with pointers to functions, that programs get their own copies + * of, take slots in fd_array after nr_obj_maps maps of the object. + */ + __u32 nr_obj_maps; + __u32 nr_func_ptr_maps; + __u32 max_func_ptr_maps; }; -void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps); +void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps, + int max_func_ptr_maps); int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps); void bpf_gen__free(struct bpf_gen *gen); void bpf_gen__load_btf(struct bpf_gen *gen, const void *raw_data, __u32 raw_size); @@ -68,6 +76,9 @@ void bpf_gen__prog_load(struct bpf_gen *gen, void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size, __u64 flags); void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx); +int bpf_gen__func_ptr_map_create(struct bpf_gen *gen, const char *map_name, int obj_map_idx, + void *value, __u32 value_size, const __u32 *ptr_offs, + const __u64 *ptr_vals, int ptr_cnt); void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type); void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, bool is_typeless, bool is_ld64, int kind, int insn_idx); diff --git a/src/gen_loader.c b/src/gen_loader.c index af3a04f..251392a 100644 --- a/src/gen_loader.c +++ b/src/gen_loader.c @@ -112,13 +112,23 @@ static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn in static int add_data(struct bpf_gen *gen, const void *data, __u32 size); static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off); -void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) +void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps, + int max_func_ptr_maps) { size_t stack_sz = sizeof(struct loader_stack), nr_progs_sz; int i; gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int)); gen->log_level = log_level; + gen->nr_obj_maps = nr_maps; + gen->max_func_ptr_maps = max_func_ptr_maps; + if (nr_maps + max_func_ptr_maps > MAX_USED_MAPS) { + pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS); + gen->error = -E2BIG; + return; + } + /* their fds are closed like the fds of the maps of the object when loading fails */ + nr_maps += max_func_ptr_maps; /* save ctx pointer into R6 */ emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); @@ -385,6 +395,9 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) return gen->error; } emit_sys_close_stack(gen, stack_off(btf_fd)); + /* programs hold their maps with pointers to functions, nothing else needs them */ + for (i = 0; i < gen->nr_func_ptr_maps; i++) + emit_sys_close_blob(gen, blob_fd_array_off(gen, gen->nr_obj_maps + i)); for (i = 0; i < gen->nr_progs; i++) move_stack2ctx(gen, sizeof(struct bpf_loader_ctx) + @@ -1127,52 +1140,41 @@ void bpf_gen__prog_load(struct bpf_gen *gen, gen->nr_progs++; } -void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, - __u32 value_size, __u64 flags) +/* + * if (map_desc[map_idx].initial_value) { + * if (ctx->flags & BPF_SKEL_KERNEL) + * bpf_probe_read_kernel(value, value_size, initial_value); + * else + * bpf_copy_from_user(value, value_size, initial_value); + * nr_more_insns that the caller emits + * } + */ +static void emit_copy_initial_value(struct bpf_gen *gen, int map_idx, int value, + __u32 value_size, int nr_more_insns) { - int attr_size = offsetofend(union bpf_attr, flags); - int map_update_attr, value, key; - union bpf_attr attr; - int zero = 0; + emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, + sizeof(struct bpf_loader_ctx) + + sizeof(struct bpf_map_desc) * map_idx + + offsetof(struct bpf_map_desc, initial_value))); + emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8 + nr_more_insns)); + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, value)); + emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); + emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, + offsetof(struct bpf_loader_ctx, flags))); + emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); + emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); + emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); + emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); +} - memset(&attr, 0, attr_size); - attr.flags = tgt_endian(flags); +/* Update the element of the map whose fd is in the slot map_idx of fd_array */ +static void emit_map_update_elem(struct bpf_gen *gen, int map_idx, union bpf_attr *attr, + int attr_size, int key, int value, __u32 value_size) +{ + int map_update_attr; - value = add_data(gen, pvalue, value_size); - key = add_data(gen, &zero, sizeof(zero)); - - /* - * if (map_desc[map_idx].initial_value) { - * if (ctx->flags & BPF_SKEL_KERNEL) - * bpf_probe_read_kernel(value, value_size, initial_value); - * else - * bpf_copy_from_user(value, value_size, initial_value); - * } - * - * The runtime initial_value comes from the host-supplied loader - * ctx and would overwrite the blob value that the program signature - * covers and the kernel verifies at load time. For a signed loader - * (gen_hash) the attested blob value must be authoritative, so skip - * the override and leave the signed value in place. - */ - if (!OPTS_GET(gen->opts, gen_hash, false)) { - emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, - sizeof(struct bpf_loader_ctx) + - sizeof(struct bpf_map_desc) * map_idx + - offsetof(struct bpf_map_desc, initial_value))); - emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, value)); - emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, - offsetof(struct bpf_loader_ctx, flags))); - emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); - } - - map_update_attr = add_data(gen, &attr, attr_size); + map_update_attr = add_data(gen, attr, attr_size); pr_debug("gen: map_update_elem: idx %d, value: off %d size %u, attr: off %d size %d\n", map_idx, value, value_size, map_update_attr, attr_size); move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, @@ -1185,6 +1187,33 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, emit_check_err(gen); } +void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, + __u32 value_size, __u64 flags) +{ + int attr_size = offsetofend(union bpf_attr, flags); + union bpf_attr attr; + int value, key; + int zero = 0; + + memset(&attr, 0, attr_size); + attr.flags = tgt_endian(flags); + + value = add_data(gen, pvalue, value_size); + key = add_data(gen, &zero, sizeof(zero)); + + /* + * The runtime initial_value comes from the host-supplied loader + * ctx and would overwrite the blob value that the program signature + * covers and the kernel verifies at load time. For a signed loader + * (gen_hash) the attested blob value must be authoritative, so skip + * the override and leave the signed value in place. + */ + if (!OPTS_GET(gen->opts, gen_hash, false)) + emit_copy_initial_value(gen, map_idx, value, value_size, 0); + + emit_map_update_elem(gen, map_idx, &attr, attr_size, key, value, value_size); +} + void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot, int inner_map_idx) { @@ -1214,6 +1243,77 @@ void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slo emit_check_err(gen); } +/* + * A copy of the read-only data map obj_map_idx for a program, with the offsets + * of its functions in it, see create_func_ptr_map() in libbpf.c. It's not + * a map of the object: it's not in the loader ctx. Its content is the content + * of obj_map_idx, that the host may supply when the skeleton is loaded, with + * ptr_cnt 64-bit ptr_vals at ptr_offs. + * Return the index of the map in fd_array for instructions to refer to. + */ +int bpf_gen__func_ptr_map_create(struct bpf_gen *gen, const char *map_name, int obj_map_idx, + void *pvalue, __u32 value_size, const __u32 *ptr_offs, + const __u64 *ptr_vals, int ptr_cnt) +{ + int attr_size = offsetofend(union bpf_attr, map_extra); + int map_create_attr, map_idx, key, value, zero = 0, i; + union bpf_attr attr; + + if (gen->nr_func_ptr_maps == gen->max_func_ptr_maps) { + gen->error = -EDOM; /* internal bug */ + return 0; + } + map_idx = gen->nr_obj_maps + gen->nr_func_ptr_maps++; + + memset(&attr, 0, attr_size); + attr.map_type = tgt_endian(BPF_MAP_TYPE_ARRAY); + attr.key_size = tgt_endian((__u32)sizeof(int)); + attr.value_size = tgt_endian(value_size); + attr.max_entries = tgt_endian((__u32)1); + attr.map_flags = tgt_endian((__u32)BPF_F_RDONLY_PROG); + if (map_name) + libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); + + map_create_attr = add_data(gen, &attr, attr_size); + pr_debug("gen: func_ptr_map_create: %s idx %d value_size %u, attr: off %d size %d\n", + map_name, map_idx, value_size, map_create_attr, attr_size); + emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); + debug_ret(gen, "func_ptr_map_create %s idx %d value_size %d", map_name, map_idx, + value_size); + emit_check_err(gen); + /* remember map_fd in fd_array */ + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, blob_fd_array_off(gen, map_idx))); + emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0)); + + /* pvalue has the pointers already */ + value = add_data(gen, pvalue, value_size); + key = add_data(gen, &zero, sizeof(zero)); + + /* see bpf_gen__map_update_elem() */ + if (!OPTS_GET(gen->opts, gen_hash, false)) { + /* the jump over these instructions has 16-bit offset */ + if (ptr_cnt > 10000) { + gen->error = -E2BIG; + return 0; + } + emit_copy_initial_value(gen, obj_map_idx, value, value_size, 3 * ptr_cnt); + /* the content that the host supplied doesn't have them */ + for (i = 0; i < ptr_cnt; i++) { + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, value + ptr_offs[i])); + emit(gen, BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, ptr_vals[i])); + } + } + + attr_size = offsetofend(union bpf_attr, flags); + memset(&attr, 0, attr_size); + emit_map_update_elem(gen, map_idx, &attr, attr_size, key, value, value_size); + + bpf_gen__map_freeze(gen, map_idx); + return map_idx; +} + void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) { int attr_size = offsetofend(union bpf_attr, map_fd); diff --git a/src/libbpf.c b/src/libbpf.c index 2e0cf7c..39fc3c5 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -6554,12 +6554,6 @@ static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, obj->func_ptr_maps[i].map_idx == map_idx) return obj->func_ptr_maps[i].fd; - if (obj->gen_loader) { - pr_warn("prog '%s': map '%s': pointers to functions in data are not supported by light skeleton\n", - prog->name, map->name); - return -ENOTSUP; - } - /* the map is not created yet, what it's going to have is in mmaped */ if (!map->mmaped) return -EINVAL; @@ -6591,6 +6585,9 @@ static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, } else { val = (__u64)prog->subprogs[j].sub_insn_off * BPF_INSN_SZ; } + /* light skeleton can be generated for a target of another endianness */ + if (!is_native_endianness(obj)) + val = bswap_64(val); memcpy(data + ptrs[i].sec_off, &val, sizeof(val)); } @@ -6606,6 +6603,8 @@ static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, if (j < cnt && ptrs[j].sec_off == i) continue; memcpy(&val, data + i, sizeof(val)); + if (!is_native_endianness(obj)) + val = bswap_64(val); if (!val || val % BPF_INSN_SZ) continue; for (k = 0; k < prog->subprog_cnt; k++) { @@ -6617,6 +6616,30 @@ static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, } } + if (obj->gen_loader) { + __u32 *ptr_offs = calloc(cnt, sizeof(*ptr_offs)); + __u64 *ptr_vals = calloc(cnt, sizeof(*ptr_vals)); + + if (!ptr_offs || !ptr_vals) { + free(ptr_offs); + free(ptr_vals); + err = -ENOMEM; + goto err_free; + } + for (i = 0; i < cnt; i++) { + ptr_offs[i] = ptrs[i].sec_off; + memcpy(&ptr_vals[i], data + ptrs[i].sec_off, sizeof(val)); + if (!is_native_endianness(obj)) + ptr_vals[i] = bswap_64(ptr_vals[i]); + } + /* it's an index in fd_array of the loader, not an fd */ + map_fd = bpf_gen__func_ptr_map_create(obj->gen_loader, map->name, map_idx, data, + value_size, ptr_offs, ptr_vals, cnt); + free(ptr_offs); + free(ptr_vals); + goto done; + } + opts.token_fd = obj->token_fd; if (obj->token_fd) opts.map_flags |= BPF_F_TOKEN_FD; @@ -6634,6 +6657,7 @@ static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, err = -errno; goto err_close; } +done: tmp = libbpf_reallocarray(obj->func_ptr_maps, obj->func_ptr_map_cnt + 1, sizeof(*obj->func_ptr_maps)); @@ -6653,7 +6677,8 @@ static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, return map_fd; err_close: - close(map_fd); + if (!obj->gen_loader) + close(map_fd); err_free: free(data); return err; @@ -6706,7 +6731,8 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) prog->name, i, map->name); return map_fd; } - insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; + insn[0].src_reg = obj->gen_loader ? BPF_PSEUDO_MAP_IDX_VALUE : + BPF_PSEUDO_MAP_VALUE; insn[0].imm = map_fd; } else if (obj->gen_loader) { insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; @@ -9590,8 +9616,27 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch * permit cross-endian creation of "light skeleton". */ if (obj->gen_loader) { + int nr_func_ptr_maps = 0, nr_progs = 0, i; + bool text_has_callx = false; + + /* + * Every program that has callx may get a copy of every map with + * pointers to functions. Which functions a program calls is not + * known yet. Any of them may call the ones that have callx. + */ + for (i = 0; i < obj->nr_maps; i++) + if (obj->maps[i].autocreate && obj->maps[i].func_ptr_cnt) + nr_func_ptr_maps++; + for (i = 0; i < obj->nr_programs; i++) + if (prog_is_subprog(obj, &obj->programs[i]) && + prog_has_callx(&obj->programs[i])) + text_has_callx = true; + for (i = 0; i < obj->nr_programs; i++) + if (obj->programs[i].autoload && !prog_is_subprog(obj, &obj->programs[i]) && + (text_has_callx || prog_has_callx(&obj->programs[i]))) + nr_progs++; bpf_gen__init(obj->gen_loader, obj->log_level | extra_log_level, - obj->nr_programs, obj->nr_maps); + obj->nr_programs, obj->nr_maps, nr_func_ptr_maps * nr_progs); } else if (!is_native_endianness(obj)) { pr_warn("object '%s': loading non-native endianness is unsupported\n", obj->name); return libbpf_err(-LIBBPF_ERRNO__ENDIAN); @@ -10132,7 +10177,8 @@ void bpf_object__close(struct bpf_object *obj) zfree(&obj->jumptable_maps); for (i = 0; i < obj->func_ptr_map_cnt; i++) - close(obj->func_ptr_maps[i].fd); + if (!obj->gen_loader) + close(obj->func_ptr_maps[i].fd); zfree(&obj->func_ptr_maps); zfree(&obj->func_ptrs);