diff --git a/kmod/patch/kpatch-syscall.h b/kmod/patch/kpatch-syscall.h index 16c5a0de3..a23f38e91 100644 --- a/kmod/patch/kpatch-syscall.h +++ b/kmod/patch/kpatch-syscall.h @@ -236,6 +236,27 @@ } \ static inline long __kpatch_do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) +#elif defined(CONFIG_RISCV) + +/* arch/riscv/include/asm/syscall_wrapper.h versions */ + +#define __KPATCH_SYSCALL_DEFINEx(x, name, ...) \ + asmlinkage long __riscv_sys##name(const struct pt_regs *regs); \ + ALLOW_ERROR_INJECTION(__riscv_sys##name, ERRNO); \ + static inline long __kpatch_do_sys##name(__MAP(x, __SC_DECL, __VA_ARGS__));\ + __SYSCALL_SE_DEFINEx(x, sys, name, __VA_ARGS__) \ + { \ + long ret = __kpatch_do_sys##name(__MAP(x, __SC_CAST, __VA_ARGS__));\ + __MAP(x, __SC_TEST, __VA_ARGS__); \ + __PROTECT(x, ret, __MAP(x, __SC_ARGS, __VA_ARGS__)); \ + return ret; \ + } \ + asmlinkage long __riscv_sys##name(const struct pt_regs *regs) \ + { \ + return __se_sys##name(SC_RISCV_REGS_TO_ARGS(x, __VA_ARGS__)); \ + } \ + static inline long __kpatch_do_sys##name(__MAP(x, __SC_DECL, __VA_ARGS__)) + #endif /* which arch */ diff --git a/kpatch-build/Makefile b/kpatch-build/Makefile index c8f3bbaff..872f99167 100644 --- a/kpatch-build/Makefile +++ b/kpatch-build/Makefile @@ -21,7 +21,7 @@ PLUGIN_CFLAGS := $(filter-out -std=gnu11 -Wconversion, $(CFLAGS)) PLUGIN_CFLAGS += -shared -I$(GCC_PLUGINS_DIR)/include \ -Igcc-plugins -fPIC -fno-rtti -O2 -Wall endif -ifeq ($(filter $(ARCH),s390x x86_64 ppc64le aarch64 loongarch64),) +ifeq ($(filter $(ARCH),s390x x86_64 ppc64le aarch64 loongarch64 riscv64),) $(error Unsupported architecture ${ARCH}, check https://github.com/dynup/kpatch/#supported-architectures) endif diff --git a/kpatch-build/create-diff-object.c b/kpatch-build/create-diff-object.c index 3d01e71b9..080b4fecf 100644 --- a/kpatch-build/create-diff-object.c +++ b/kpatch-build/create-diff-object.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -85,6 +86,9 @@ struct special_section { unsigned int size); }; +#define GENERIC_SPECIAL_SECTION_ARCHS \ + (AARCH64 | X86_64 | PPC64 | S390 | LOONGARCH64 | RISCV64) + /************* * Functions * **********/ @@ -152,6 +156,54 @@ static bool is_bundleable(struct symbol *sym) #define PPC64_LOCAL_ENTRY_OFFSET(other) \ (((1 << (((other) & STO_PPC64_LOCAL_MASK) >> STO_PPC64_LOCAL_BIT)) >> 2) << 2) +/* + * RISC-V NOP instruction detection. + * RISC-V has two NOP encodings depending on whether the C (compressed) + * extension is enabled: + * - Regular NOP (4 bytes): 0x00000013, little-endian: 0x13 0x00 0x00 0x00 + * - C.NOP (2 bytes): 0x0001, little-endian: 0x01 0x00 + * + * The -fpatchable-function-entry=N[,M] option decides how many NOPs + * are emitted and where the function label lands within that NOP window. + * The compiler's __patchable_function_entries relocation gives us the + * ftrace patchsite; derive the actual byte length from the instructions. + */ +static bool riscv_is_4byte_nop(const unsigned char *insn) +{ + return insn[0] == 0x13 && insn[1] == 0x00 && + insn[2] == 0x00 && insn[3] == 0x00; +} + +static bool riscv_is_2byte_nop(const unsigned char *insn) +{ + return insn[0] == 0x01 && insn[1] == 0x00; +} + +static int riscv_nop_len(const unsigned char *insn, const unsigned char *end) +{ + if (insn + 2 <= end && riscv_is_2byte_nop(insn)) + return 2; + if (insn + 4 <= end && riscv_is_4byte_nop(insn)) + return 4; + return 0; +} + +static unsigned int riscv_nop_window_size(const unsigned char *insn, + const unsigned char *end) +{ + unsigned int size = 0; + int nop_size; + + while (insn + size < end) { + nop_size = riscv_nop_len(insn + size, end); + if (!nop_size) + break; + size += nop_size; + } + + return size; +} + /* * On ppc64le, the function prologue generated by GCC 6+ has the sequence: * @@ -183,6 +235,7 @@ static bool is_gcc6_localentry_bundled_sym(struct kpatch_elf *kelf, case S390: return false; case LOONGARCH64: + case RISCV64: return false; default: ERROR("unsupported arch"); @@ -232,6 +285,14 @@ static struct rela *toc_rela(const struct rela *rela) (unsigned int)rela->addend); } +static bool kpatch_is_local_notype_symbol(struct symbol *sym) +{ + return sym->name && + sym->type == STT_NOTYPE && + sym->bind == STB_LOCAL && + sym->sym.st_size == 0; +} + /* * Mapping symbols are used to mark and label the transitions between code and * data in elf files. They begin with a "$" dollar symbol. Don't correlate them @@ -240,12 +301,43 @@ static struct rela *toc_rela(const struct rela *rela) */ static bool kpatch_is_mapping_symbol(struct kpatch_elf *kelf, struct symbol *sym) { + if (!kpatch_is_local_notype_symbol(sym)) + return false; + switch (kelf->arch) { case AARCH64: - if (sym->name && sym->name[0] == '$' - && sym->type == STT_NOTYPE \ - && sym->bind == STB_LOCAL) - return true; + case RISCV64: + return sym->name[0] == '$'; + case X86_64: + case PPC64: + case S390: + case LOONGARCH64: + return false; + default: + ERROR("unsupported arch"); + } + + return false; +} + +/* + * GCC generates local labels like .L0, .LVL133, etc. for DWARF info. + * They have zero size and can change sections between compilations. Do + * not include .LC* here; those symbols point to string literals and are + * handled by kpatch_include_standard_elements(). + */ +static bool kpatch_is_unstable_local_label(struct kpatch_elf *kelf, + struct symbol *sym) +{ + if (!kpatch_is_local_notype_symbol(sym)) + return false; + + switch (kelf->arch) { + case AARCH64: + case RISCV64: + return sym->name[0] == '.' && + sym->name[1] == 'L' && + strncmp(sym->name, ".LC", 3); case X86_64: case PPC64: case S390: @@ -258,6 +350,13 @@ static bool kpatch_is_mapping_symbol(struct kpatch_elf *kelf, struct symbol *sym return false; } +static bool kpatch_is_ignorable_local_symbol(struct kpatch_elf *kelf, + struct symbol *sym) +{ + return kpatch_is_mapping_symbol(kelf, sym) || + kpatch_is_unstable_local_label(kelf, sym); +} + static unsigned int function_padding_size(struct kpatch_elf *kelf, struct symbol *sym) { unsigned int size = 0; @@ -287,6 +386,31 @@ static unsigned int function_padding_size(struct kpatch_elf *kelf, struct symbol break; } + case RISCV64: + { + uint8_t *insn = sym->sec->data->d_buf; + uint8_t *insn_end = sym->sec->data->d_buf + sym->sym.st_value; + int nop_size; + + /* + * Count NOP padding bytes before the function entry. + * RISC-V may mix 2-byte C.NOP and 4-byte regular NOP + * instructions, so check each position individually. + */ + while (insn < insn_end) { + nop_size = riscv_nop_len(insn, insn_end); + if (!nop_size) + break; + size += nop_size; + insn += nop_size; + } + + if (insn < insn_end && size != 0) + log_error("function %s within section %s has invalid padding\n", + sym->name, sym->sec->name); + + break; + } default: break; } @@ -660,6 +784,238 @@ static bool rela_equal(struct rela *rela1, struct rela *rela2) return !kpatch_mangled_strcmp(rela_toc1->sym->name, rela_toc2->sym->name); } +/* + * _error_injection_whitelist contains entries generated by + * ALLOW_ERROR_INJECTION(): + * + * struct error_injection_entry { + * unsigned long addr; + * int etype; + * }; + * + * The address field is relocated, while etype is stored directly in the + * section data. Do not include the kernel header here: create-diff-object + * operates on ELF objects and only needs the layout of this metadata. + */ +struct kpatch_error_injection_entry { + unsigned long addr; + int etype; +}; + +struct kpatch_error_injection_key { + const char *target; + long target_offset; + int etype; +}; + +/* + * Return a stable description of the relocation target. Symbol indices and + * relocation ordering are deliberately not part of the description. + */ +static bool kpatch_error_injection_target(struct rela *rela, + const char **target, + long *target_offset) +{ + struct symbol *sym = rela->sym; + + if (!sym || !sym->name) + return false; + + if (sym->type == STT_SECTION) { + if (!sym->sec || !sym->sec->name) + return false; + + *target = sym->sec->name; + *target_offset = (long)sym->sym.st_value + rela->addend; + return true; + } + + /* Use the correlated name when available, independent of the side. */ + *target = sym->twin ? sym->twin->name : sym->name; + *target_offset = rela->addend; + + return *target != NULL; +} + +static bool kpatch_error_injection_etype(struct kpatch_elf *kelf, + const unsigned char *data, + int *etype) +{ + GElf_Ehdr ehdr; + uint32_t value; + + if (!gelf_getehdr(kelf->elf, &ehdr)) + ERROR("gelf_getehdr"); + + switch (ehdr.e_ident[EI_DATA]) { + case ELFDATA2LSB: + value = (uint32_t)data[0] | + ((uint32_t)data[1] << 8) | + ((uint32_t)data[2] << 16) | + ((uint32_t)data[3] << 24); + break; + case ELFDATA2MSB: + value = ((uint32_t)data[0] << 24) | + ((uint32_t)data[1] << 16) | + ((uint32_t)data[2] << 8) | + (uint32_t)data[3]; + break; + default: + return false; + } + + *etype = (int)value; + return true; +} + +/* + * Convert the whitelist into an unordered list of semantic keys. + * If the section does not have the expected shape, leave it unchanged + * instead of ignoring it. + */ +static bool kpatch_collect_error_injection_keys( + struct kpatch_elf *kelf, struct section *sec, + struct kpatch_error_injection_key **keys, size_t *nr_keys) +{ + struct kpatch_error_injection_key *result = NULL; + struct section *relasec = sec->rela; + struct rela *rela; + unsigned char *data; + bool *seen = NULL; + size_t entry_size = sizeof(struct kpatch_error_injection_entry); + size_t nr_entries, nr_relas = 0; + size_t index; + + *keys = NULL; + *nr_keys = 0; + + if (!sec->data || !relasec) + return false; + if (sec->data->d_size && !sec->data->d_buf) + return false; + + if (sec->data->d_size % entry_size) + return false; + + nr_entries = sec->data->d_size / entry_size; + list_for_each_entry(rela, &relasec->relas, list) + nr_relas++; + + if (nr_entries != nr_relas) + return false; + + if (nr_entries) { + result = calloc(nr_entries, sizeof(*result)); + seen = calloc(nr_entries, sizeof(*seen)); + if (!result || !seen) + ERROR("calloc"); + } + + data = sec->data->d_buf; + list_for_each_entry(rela, &relasec->relas, list) { + const char *target; + long target_offset; + int etype; + + /* The relocation must point to addr at the start of an entry. */ + if (rela->offset % entry_size != + offsetof(struct kpatch_error_injection_entry, addr)) + goto fail; + + index = rela->offset / entry_size; + if (index >= nr_entries || seen[index]) + goto fail; + + if (!kpatch_error_injection_target(rela, &target, &target_offset)) + goto fail; + + if (!kpatch_error_injection_etype( + kelf, + data + index * entry_size + + offsetof(struct kpatch_error_injection_entry, etype), + &etype)) + goto fail; + + result[index].target = target; + result[index].target_offset = target_offset; + result[index].etype = etype; + seen[index] = true; + } + + free(seen); + *keys = result; + *nr_keys = nr_entries; + return true; + +fail: + free(seen); + free(result); + return false; +} + +static bool kpatch_error_injection_key_equal( + const struct kpatch_error_injection_key *key1, + const struct kpatch_error_injection_key *key2) +{ + return key1->etype == key2->etype && + key1->target_offset == key2->target_offset && + !strcmp(key1->target, key2->target); +} + +/* Compare the whitelist as a semantic multiset. */ +static bool kpatch_error_injection_whitelist_equal( + struct kpatch_elf *kelf, struct section *sec) +{ + struct kpatch_error_injection_key *keys1 = NULL, *keys2 = NULL; + bool *matched = NULL; + size_t nr_keys1, nr_keys2; + size_t i, j; + bool found; + bool equal = false; + + if (!sec->twin || !sec->rela || !sec->twin->rela) + return false; + + if (!kpatch_collect_error_injection_keys( + kelf, sec->twin, &keys1, &nr_keys1) || + !kpatch_collect_error_injection_keys( + kelf, sec, &keys2, &nr_keys2)) + goto out; + + if (nr_keys1 != nr_keys2) + goto out; + + if (nr_keys2) { + matched = calloc(nr_keys2, sizeof(*matched)); + if (!matched) + ERROR("calloc"); + } + + for (i = 0; i < nr_keys1; i++) { + found = false; + for (j = 0; j < nr_keys2; j++) { + if (matched[j] || + !kpatch_error_injection_key_equal(&keys1[i], + &keys2[j])) + continue; + + matched[j] = true; + found = true; + break; + } + if (!found) + goto out; + } + + equal = true; + +out: + free(matched); + free(keys1); + free(keys2); + return equal; +} + static void kpatch_compare_correlated_rela_section(struct section *relasec) { struct rela *rela1, *rela2 = NULL; @@ -795,6 +1151,49 @@ static bool insn_is_load_immediate(struct kpatch_elf *kelf, void *addr) break; + case RISCV64: + /* + * RISC-V li a1, imm = addi a1, x0, imm: + * + * Verify the complete I-type encoding fields. The immediate + * field is intentionally ignored because it contains __LINE__. + */ + { + unsigned int insn32; + unsigned int opcode, funct3, rd, rs1; + + insn32 = (unsigned int)insn[0] | + ((unsigned int)insn[1] << 8) | + ((unsigned int)insn[2] << 16) | + ((unsigned int)insn[3] << 24); + + opcode = insn32 & 0x7f; + funct3 = (insn32 >> 12) & 0x7; + rd = (insn32 >> 7) & 0x1f; + rs1 = (insn32 >> 15) & 0x1f; + + if (opcode == 0x13 && /* OP-IMM */ + funct3 == 0x0 && /* ADDI */ + rs1 == 0 && /* x0 */ + rd == 11) /* a1 */ + return true; + } + + /* + * C-ext: C.li rd, imm (16-bit compressed): + * byte0 & 0x03 == 0x01 (quadrant 1) + * byte1 >> 5 == 0x02 (funct3 = 010 for C.li) + * rd = ((byte1 & 0x0f) << 1) | ((byte0 >> 7) & 1) + */ + if ((insn[0] & 0x03) == 0x01 && (insn[1] >> 5) == 0x02) { + int rd = ((insn[1] & 0x0f) << 1) | ((insn[0] >> 7) & 1); + + if (rd == 11) /* a1 */ + return true; + } + + break; + case S390: /* arg2: lghi %r3, imm */ if (insn[0] == 0xa7 && insn[1] == 0x39) @@ -1222,7 +1621,7 @@ static void kpatch_correlate_symbols(struct kpatch_elf *kelf_orig, !strncmp(sym_orig->name, ".LC", 3)) continue; - if (kpatch_is_mapping_symbol(kelf_orig, sym_orig)) + if (kpatch_is_ignorable_local_symbol(kelf_orig, sym_orig)) continue; /* group section symbols must have correlated sections */ @@ -1791,7 +2190,7 @@ static void kpatch_replace_sections_syms(struct kpatch_elf *kelf) */ } else if (target_off == start && target_off == end) { - if(kpatch_is_mapping_symbol(kelf, sym)) + if (kpatch_is_ignorable_local_symbol(kelf, sym)) continue; /* @@ -2486,33 +2885,126 @@ static int fixup_group_size(struct kpatch_elf *kelf, int offset) return (int)(rela->addend - offset); } +static bool is_riscv_jump_table_rela(const struct rela *rela) +{ + return rela->type == R_RISCV_ADD32 || + rela->type == R_RISCV_SUB32 || + rela->type == R_RISCV_ADD64 || + rela->type == R_RISCV_SUB64; +} + static bool jump_table_group_filter(struct lookup_table *lookup, struct section *relasec, unsigned int group_offset, unsigned int group_size) { struct rela *code = NULL, *key = NULL, *rela; + struct rela *riscv_field[3][2] = {}; bool tracepoint = false, dynamic_debug = false; struct lookup_result symbol; + bool riscv_relative = false; int i = 0; /* * Here we hard-code knowledge about the contents of the jump_entry * struct. It has three fields: code, target, and key. Each field has * a relocation associated with it. + * + * On RISC-V, each field uses paired ADD/SUB relocations, so there + * are 6 relocations per entry instead of 3. The ADD relocation + * (first of each pair) is the one that references the actual symbol. */ list_for_each_entry(rela, &relasec->relas, list) { - if (rela->offset >= group_offset && - rela->offset < group_offset + group_size) { + if (rela->offset < group_offset || + rela->offset >= group_offset + group_size) + continue; + + i++; + } + + if (i == 6) { + /* + * RISC-V RV64 jump_entry is: s32 code, s32 target, + * long key. Each field is represented by an ADD/SUB + * pair. Identify entries by type and offset rather than + * relocation list order. + */ + list_for_each_entry(rela, &relasec->relas, list) { + unsigned int offset, field; + int pair; + + if (rela->offset < group_offset || + rela->offset >= group_offset + group_size) + continue; + + if (!is_riscv_jump_table_rela(rela)) + ERROR("BUG: mixed RISC-V __jump_table relocation group"); + + offset = rela->offset - group_offset; + switch (offset) { + case 0: + field = 0; + break; + case 4: + field = 1; + break; + case 8: + field = 2; + break; + default: + ERROR("BUG: invalid RISC-V __jump_table relocation offset"); + } + + if (field == 2) { + if (rela->type == R_RISCV_ADD64) + pair = 0; + else if (rela->type == R_RISCV_SUB64) + pair = 1; + else + ERROR("BUG: invalid RISC-V __jump_table key relocation"); + } else { + if (rela->type == R_RISCV_ADD32) + pair = 0; + else if (rela->type == R_RISCV_SUB32) + pair = 1; + else + ERROR("BUG: invalid RISC-V __jump_table 32-bit relocation"); + } + + if (riscv_field[field][pair]) + ERROR("BUG: duplicate RISC-V __jump_table relocation"); + + riscv_field[field][pair] = rela; + } + + if (!riscv_field[0][0] || !riscv_field[0][1] || + !riscv_field[1][0] || !riscv_field[1][1] || + !riscv_field[2][0] || !riscv_field[2][1]) + ERROR("BUG: incomplete RISC-V __jump_table relocation pair"); + + code = riscv_field[0][0]; + key = riscv_field[2][0]; + riscv_relative = true; + } else if (i == 3) { + /* Generic architectures use one relocation per field. */ + i = 0; + list_for_each_entry(rela, &relasec->relas, list) { + if (rela->offset < group_offset || + rela->offset >= group_offset + group_size) + continue; + if (i == 0) code = rela; else if (i == 2) key = rela; + i++; } + } else { + ERROR("BUG: __jump_table has an unexpected format"); } - if (i != 3 || !key || !code) + if (!key || !code) ERROR("BUG: __jump_table has an unexpected format"); if (!strncmp(key->sym->name, "__tracepoint_", 13)) @@ -2521,6 +3013,28 @@ static bool jump_table_group_filter(struct lookup_table *lookup, if (is_dynamic_debug_symbol(key->sym)) dynamic_debug = true; + if (riscv_relative && + lookup_symbol(lookup, key->sym, &symbol) && + !strcmp(symbol.objname, "vmlinux") && + !symbol.exported) { + /* + * RISC-V relative jump entries are made from ADD/SUB rela + * pairs. If the ADD side is converted to a livepatch rela + * while the SUB side remains a normal module rela, module jump + * label init can see an invalid key pointer. Tracepoints and + * dynamic debug entries are inert when omitted; for other keys, + * fail the build instead of producing a module which can crash + * during MODULE_STATE_COMING. + */ + if (tracepoint || dynamic_debug) + return false; + + log_error("RISC-V jump label at %s()+0x%lx uses unexported key %s\n", + code->sym->name, code->addend, key->sym->name); + jump_label_errors++; + return false; + } + if (KLP_ARCH) { /* * On older kernels (with .klp.arch support), jump labels @@ -2652,22 +3166,22 @@ static bool static_call_sites_group_filter(struct lookup_table *lookup, static struct special_section special_sections[] = { { .name = "__bug_table", - .arch = AARCH64 | X86_64 | PPC64 | S390 | LOONGARCH64, + .arch = GENERIC_SPECIAL_SECTION_ARCHS, .group_size = bug_table_group_size, }, { .name = ".fixup", - .arch = AARCH64 | X86_64 | PPC64 | S390 | LOONGARCH64, + .arch = GENERIC_SPECIAL_SECTION_ARCHS, .group_size = fixup_group_size, }, { .name = "__ex_table", /* must come after .fixup */ - .arch = AARCH64 | X86_64 | PPC64 | S390 | LOONGARCH64, + .arch = GENERIC_SPECIAL_SECTION_ARCHS, .group_size = ex_table_group_size, }, { .name = "__jump_table", - .arch = AARCH64 | X86_64 | PPC64 | S390 | LOONGARCH64, + .arch = GENERIC_SPECIAL_SECTION_ARCHS, .group_size = jump_table_group_size, .group_filter = jump_table_group_filter, }, @@ -2691,6 +3205,11 @@ static struct special_section special_sections[] = { .arch = AARCH64 | X86_64 | S390 | LOONGARCH64, .group_size = altinstructions_group_size, }, + { + .name = ".alternative", + .arch = RISCV64, + .group_size = altinstructions_group_size, + }, { .name = ".static_call_sites", .arch = X86_64, @@ -2937,7 +3456,40 @@ static void kpatch_regenerate_special_section(struct kpatch_elf *kelf, rela->offset -= src_offset - dest_offset; rela->rela.r_offset = rela->offset; - kpatch_include_symbol(rela->sym); + /* + * On RISC-V the fields in .alternative, + * __jump_table, __bug_table and __ex_table are + * encoded with paired R_RISCV_ADD/SUB + * relocations: the SUB half references a local + * anchor symbol whose st_value equals the + * field's own offset in the base section. When + * a group is compacted to a new offset, move these + * anchors together with it; otherwise the resolved + * field value keeps referencing the group's old + * offset, which makes the kernel patch arbitrary + * code at module load time (e.g. cpufeature + * alternatives can overwrite unrelated functions). + * + * The range check makes the adjustment idempotent + * in case an anchor is shared. + */ + if (kelf->arch == RISCV64 && + rela->sym->sec == relasec->base) { + if (rela->sym->sym.st_value >= src_offset && + rela->sym->sym.st_value < src_offset + group_size) + rela->sym->sym.st_value += + (long)dest_offset - (long)src_offset; + + /* + * The regenerated base section is included below. + * Including it recursively here would walk the old, + * unfiltered relocation list and pull references from + * discarded groups into the output. + */ + rela->sym->include = 1; + } else { + kpatch_include_symbol(rela->sym); + } if (!strcmp(special->name, ".fixup")) kpatch_update_ex_table_addend(kelf, special, @@ -2971,6 +3523,22 @@ static void kpatch_regenerate_special_section(struct kpatch_elf *kelf, /* overwrite with new relas list */ list_replace(&newrelas, &relasec->relas); + /* + * On RISC-V the paired R_RISCV_SUB* relocations reference local + * anchor symbols inside the base section itself. After compaction + * every such anchor must lie within the regenerated section; an + * out-of-bounds value means the fields would resolve to garbage and + * the kernel could patch arbitrary code at module load time. + */ + if (kelf->arch == RISCV64) { + list_for_each_entry(rela, &relasec->relas, list) { + if (rela->sym->sec == relasec->base && + rela->sym->sym.st_value >= dest_offset) + ERROR("special section %s: unresolved in-section anchor", + relasec->base->name); + } + } + /* include both rela and base sections */ relasec->include = 1; relasec->base->include = 1; @@ -3109,6 +3677,14 @@ static void kpatch_check_relocations(struct kpatch_elf *kelf) if (is_ftr_alt_fixup_reloc(relasec, rela)) continue; + /* + * Sections without allocated data (e.g. NOBITS/.bss + * or sections not materialized for the output) have no + * size to check against; skip the range validation. + */ + if (!rela->sym->sec->data) + continue; + sec_size = rela->sym->sec->data->d_size; sec_off = (long)rela->sym->sym.st_value + rela_target_offset(kelf, relasec, rela); @@ -3151,9 +3727,13 @@ static void kpatch_include_debug_sections(struct kpatch_elf *kelf) list_for_each_entry(sec, &kelf->sections, list) { if (!is_rela_section(sec) || !is_debug_section(sec)) continue; - list_for_each_entry_safe(rela, saferela, &sec->relas, list) - if (!rela->sym->sec->include) + list_for_each_entry_safe(rela, saferela, &sec->relas, list) { + if (!rela->sym->sec->include) { list_del(&rela->list); + continue; + } + kpatch_include_symbol(rela->sym); + } } } @@ -3180,7 +3760,7 @@ static void kpatch_mark_ignored_sections(struct kpatch_elf *kelf) sec->ignore = 1; } - if (kelf->arch == X86_64) { + if (kelf->arch == X86_64 || kelf->arch == RISCV64) { if (!strcmp(sec->name, ".rela__patchable_function_entries") || !strcmp(sec->name, "__patchable_function_entries")) sec->ignore = 1; @@ -3214,7 +3794,7 @@ static void kpatch_mark_ignored_sections(struct kpatch_elf *kelf) if (strsec->secsym) strsec->secsym->include = 1; - name = strsec->data->d_buf + rela->addend; + name = strsec->data->d_buf + rela->sym->sym.st_value + rela->addend; ignoresec = find_section_by_name(&kelf->sections, name); if (!ignoresec) ERROR("KPATCH_IGNORE_SECTION: can't find %s", name); @@ -3227,6 +3807,24 @@ static void kpatch_mark_ignored_sections(struct kpatch_elf *kelf) } } +/* Ignore the whitelist only when its semantic contents are unchanged. */ +static void kpatch_mark_equal_error_injection_whitelist_same( + struct kpatch_elf *kelf) +{ + struct section *sec; + + list_for_each_entry(sec, &kelf->sections, list) { + if (strcmp(sec->name, "_error_injection_whitelist")) + continue; + + if (!kpatch_error_injection_whitelist_equal(kelf, sec)) + continue; + + sec->ignore = 1; + sec->rela->ignore = 1; + } +} + static void kpatch_mark_ignored_sections_same(struct kpatch_elf *kelf) { struct section *sec; @@ -3585,6 +4183,117 @@ static int function_ptr_rela(const struct rela *rela, struct kpatch_elf *kelf) return funcptr; } +static struct symbol *find_or_create_section_symbol(struct kpatch_elf *kelf, + struct section *sec) +{ + struct symbol *sym; + + if (sec->secsym) + return sec->secsym; + + /* + * Newer toolchains are stingy with their section symbols, create one + * if it doesn't exist already. + */ + ALLOC_LINK(sym, &kelf->symbols); + sym->sec = sec; + sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION); + sym->type = STT_SECTION; + sym->bind = STB_LOCAL; + sym->name = sec->name; + sec->secsym = sym; + + return sym; +} + +static void add_intermediate_rela(struct kpatch_elf *kelf, + struct section *relasec, + struct symbol *sym, long addend, + unsigned int offset) +{ + struct rela *rela; + + ALLOC_LINK(rela, &relasec->relas); + rela->sym = sym; + rela->type = absolute_rela_type(kelf); + rela->addend = addend; + rela->offset = offset; +} + +static unsigned int kpatch_relocation_field_offset(unsigned int index, + size_t field) +{ + return (unsigned int)(index * sizeof(struct kpatch_relocation) + field); +} + +static void add_kpatch_relocation_relas(struct kpatch_elf *kelf, + struct section *krela_sec, + struct section *dest_sec, + struct symbol *strsym, char *objname, + unsigned int index, unsigned int dest_off, + struct symbol *ksym, long ksym_addend) +{ + struct symbol *dest_sym; + + dest_sym = find_or_create_section_symbol(kelf, dest_sec); + add_intermediate_rela(kelf, krela_sec->rela, dest_sym, dest_off, + kpatch_relocation_field_offset(index, + offsetof(struct kpatch_relocation, dest))); + + add_intermediate_rela(kelf, krela_sec->rela, strsym, + offset_of_string(&kelf->strings, objname), + kpatch_relocation_field_offset(index, + offsetof(struct kpatch_relocation, objname))); + + add_intermediate_rela(kelf, krela_sec->rela, ksym, ksym_addend, + kpatch_relocation_field_offset(index, + offsetof(struct kpatch_relocation, ksym))); +} + +static struct symbol *create_riscv_hi20_anchor(struct kpatch_elf *kelf, + struct section *sec, + unsigned long value, + unsigned int index) +{ + struct symbol *sym; + char buf[64]; + + ALLOC_LINK(sym, &kelf->symbols); + snprintf(buf, sizeof(buf), KPATCH_RISCV_HI20_PREFIX "%u", index); + sym->name = strdup(buf); + if (!sym->name) + ERROR("strdup"); + sym->sec = sec; + sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_NOTYPE); + sym->sym.st_value = value; + sym->type = STT_NOTYPE; + sym->bind = STB_LOCAL; + sym->strip = SYMBOL_USED; + + return sym; +} + +static void mark_riscv_paired_lo12_relas(struct section *relasec, + const struct rela *hi20) +{ + struct rela *rela; + + if (!is_riscv_hi20_rela(hi20)) + return; + + list_for_each_entry(rela, &relasec->relas, list) { + if (!is_riscv_lo12_rela(rela)) + continue; + if (!rela->sym) + continue; + if ((unsigned int)(rela->sym->sym.st_value + rela->addend) != hi20->offset) + continue; + + rela->need_klp_reloc = true; + rela->riscv_paired_lo12 = true; + } +} + static bool need_klp_reloc(struct kpatch_elf *kelf, struct lookup_table *table, struct section *relasec, const struct rela *rela) { @@ -3800,8 +4509,12 @@ static void kpatch_create_intermediate_sections(struct kpatch_elf *kelf, * internal symbol function pointer check which is done * via .toc indirection in need_klp_reloc(). */ - if (need_klp_reloc(kelf, table, relasec, rela)) + if (need_klp_reloc(kelf, table, relasec, rela)) { toc_rela(rela)->need_klp_reloc = true; + if (kelf->arch == RISCV64) + mark_riscv_paired_lo12_relas(relasec, + toc_rela(rela)); + } } } @@ -3868,6 +4581,34 @@ static void kpatch_create_intermediate_sections(struct kpatch_elf *kelf, ERROR("unsupported klp relocation reference to symbol '%s' in module-specific special section '%s'", rela->sym->name, relasec->base->name); + if (rela->riscv_paired_lo12) { + struct symbol *lo12_sym; + + /* + * The RISC-V module loader requires a PCREL_LO12 + * relocation to live in the same relsec as its + * matching HI20 relocation, but the LO12 still + * points at the local text address of that HI20. + */ + krelas[index].addend = rela->addend; + krelas[index].type = rela->type; + krelas[index].external = 0; + krelas[index].local = 1; + + lo12_sym = create_riscv_hi20_anchor(kelf, relasec->base, + rela->sym->sym.st_value + rela->addend, index); + add_kpatch_relocation_relas(kelf, krela_sec, + relasec->base, strsym, + objname, index, + rela->offset, lo12_sym, 0); + + rela->sym->strip = SYMBOL_USED; + list_del(&rela->list); + free(rela); + index++; + continue; + } + if (!lookup_symbol(table, rela->sym, &symbol)) ERROR("can't find symbol '%s' in symbol table", rela->sym->name); @@ -3909,46 +4650,12 @@ static void kpatch_create_intermediate_sections(struct kpatch_elf *kelf, krelas[index].addend = rela->addend; krelas[index].type = rela->type; krelas[index].external = !vmlinux && symbol.exported; + krelas[index].local = 0; - /* add rela to fill in krelas[index].dest field */ - ALLOC_LINK(rela2, &krela_sec->rela->relas); - if (!relasec->base->secsym) { - struct symbol *sym; - - /* - * Newer toolchains are stingy with their - * section symbols, create one if it doesn't - * exist already. - */ - ALLOC_LINK(sym, &kelf->symbols); - sym->sec = relasec->base; - sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION); - sym->type = STT_SECTION; - sym->bind = STB_LOCAL; - sym->name = relasec->base->name; - relasec->base->secsym = sym; - } - rela2->sym = relasec->base->secsym; - rela2->type = absolute_rela_type(kelf); - rela2->addend = rela->offset; - rela2->offset = (unsigned int)(index * sizeof(*krelas) + \ - offsetof(struct kpatch_relocation, dest)); - - /* add rela to fill in krelas[index].objname field */ - ALLOC_LINK(rela2, &krela_sec->rela->relas); - rela2->sym = strsym; - rela2->type = absolute_rela_type(kelf); - rela2->addend = offset_of_string(&kelf->strings, objname); - rela2->offset = (unsigned int)(index * sizeof(*krelas) + \ - offsetof(struct kpatch_relocation, objname)); - - /* add rela to fill in krelas[index].ksym field */ - ALLOC_LINK(rela2, &krela_sec->rela->relas); - rela2->sym = ksym_sec_sym; - rela2->type = absolute_rela_type(kelf); - rela2->addend = (unsigned int)(index * sizeof(*ksyms)); - rela2->offset = (unsigned int)(index * sizeof(*krelas) + \ - offsetof(struct kpatch_relocation, ksym)); + add_kpatch_relocation_relas(kelf, krela_sec, relasec->base, + strsym, objname, index, + rela->offset, ksym_sec_sym, + (long)(index * sizeof(*ksyms))); /* * Mark the referred to symbol for removal but @@ -4042,6 +4749,50 @@ static void kpatch_set_pfe_link(struct kpatch_elf *kelf) } } +static struct rela *kpatch_find_pfe_rela(struct kpatch_elf *kelf, + struct symbol *sym) +{ + struct section *sec; + struct rela *rela; + + if (!kelf->has_pfe) + return NULL; + + list_for_each_entry(sec, &kelf->sections, list) { + if (strcmp(sec->name, "__patchable_function_entries")) + continue; + if (!sec->rela) + continue; + + list_for_each_entry(rela, &sec->rela->relas, list) { + if (rela->sym->sec && sym->sec == rela->sym->sec && + rela->sym->pfe == sec) + return rela; + } + } + + return NULL; +} + +/* + * A patchable function entry can legitimately be at section offset 0, so use a + * negative sentinel for functions whose callsite was not described by a PFE rela. + */ +#define FUNC_PROFILING_CALLSITE_NONE (-1L) + +static bool kpatch_has_func_profiling_callsite(const struct symbol *sym) +{ + return sym->func_profiling_callsite != FUNC_PROFILING_CALLSITE_NONE; +} + +static void kpatch_set_func_profiling_callsite(struct symbol *sym, + struct rela *rela) +{ + sym->has_func_profiling = 1; + sym->func_profiling_callsite = (long)rela->sym->sym.st_value + + rela->addend; +} + /* * This function basically reimplements the functionality of the Linux * recordmcount script, so that patched functions can be recognized by ftrace. @@ -4056,7 +4807,7 @@ static void kpatch_create_ftrace_callsite_sections(struct kpatch_elf *kelf) struct symbol *sym, *rela_sym; struct rela *rela; void **funcs; - unsigned long insn_offset = 0; + long insn_offset = 0; unsigned int rela_offset; nr = 0; @@ -4207,6 +4958,35 @@ static void kpatch_create_ftrace_callsite_sections(struct kpatch_elf *kelf) insn_offset = 0; break; } + case RISCV64: { + long patchsite; + unsigned char *insn = sym->sec->data->d_buf; + unsigned char *insn_end = sym->sec->data->d_buf + sym->sec->data->d_size; + unsigned int nop_window_size; + + if (!kpatch_has_func_profiling_callsite(sym)) + ERROR("%s: missing __patchable_function_entries callsite", + sym->name); + + patchsite = sym->func_profiling_callsite; + if (patchsite < 0 || + patchsite >= (long)sym->sec->data->d_size) + ERROR("%s: RISC-V patchable function entry is out of range", + sym->name); + nop_window_size = riscv_nop_window_size(insn + patchsite, + insn_end); + if (!nop_window_size) + ERROR("%s: unexpected instruction in RISC-V patchable function entry", + sym->name); + if (patchsite > (long)sym->sym.st_value || + (long)sym->sym.st_value >= + patchsite + nop_window_size) + ERROR("%s: function entry is outside the RISC-V patchable function entry", + sym->name); + + insn_offset = patchsite; + break; + } default: ERROR("unsupported arch"); } @@ -4226,7 +5006,22 @@ static void kpatch_create_ftrace_callsite_sections(struct kpatch_elf *kelf) */ if (kelf->pfe_ordered) sec->sh.sh_flags |= SHF_LINK_ORDER; - rela_sym = sym->sec->secsym; + if (!sym->sec->secsym) { + /* + * Newer toolchains are stingy with their + * section symbols, create one if it doesn't + * exist already. + */ + ALLOC_LINK(rela_sym, &kelf->symbols); + rela_sym->sec = sym->sec; + rela_sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION); + rela_sym->type = STT_SECTION; + rela_sym->bind = STB_LOCAL; + rela_sym->name = sym->sec->name; + sym->sec->secsym = rela_sym; + } else { + rela_sym = sym->sec->secsym; + } rela_offset = 0; rela_sym->pfe = sec; } else { @@ -4241,7 +5036,7 @@ static void kpatch_create_ftrace_callsite_sections(struct kpatch_elf *kelf) ALLOC_LINK(rela, &sec->rela->relas); rela->sym = rela_sym; rela->type = absolute_rela_type(kelf); - rela->addend = insn_offset - rela->sym->sym.st_value; + rela->addend = insn_offset - (long)rela->sym->sym.st_value; rela->offset = rela_offset; index++; @@ -4395,45 +5190,24 @@ static void kpatch_no_sibling_calls_ppc64le(struct kpatch_elf *kelf) sibling_call_errors); } -static bool kpatch_symbol_has_pfe_entry(struct kpatch_elf *kelf, struct symbol *sym) -{ - struct section *sec; - struct rela *rela; - - if (!kelf->has_pfe) - return false; - - list_for_each_entry(sec, &kelf->sections, list) { - if (strcmp(sec->name, "__patchable_function_entries")) - continue; - if (!sec->rela) - continue; - - list_for_each_entry(rela, &sec->rela->relas, list) { - if (rela->sym->sec && sym->sec == rela->sym->sec && - rela->sym->pfe == sec) { - return true; - } - } - } - - return false; -} - /* Check which functions have fentry/mcount calls; save this info for later use. */ static void kpatch_find_func_profiling_calls(struct kpatch_elf *kelf) { struct symbol *sym; struct rela *rela; + struct rela *pfe_rela; unsigned char *insn; list_for_each_entry(sym, &kelf->symbols, list) { if (sym->type != STT_FUNC || sym->is_pfx || !sym->sec) continue; + sym->func_profiling_callsite = FUNC_PROFILING_CALLSITE_NONE; + switch(kelf->arch) { case PPC64: - if (kpatch_symbol_has_pfe_entry(kelf, sym)) { - sym->has_func_profiling = 1; + pfe_rela = kpatch_find_pfe_rela(kelf, sym); + if (pfe_rela) { + kpatch_set_func_profiling_callsite(sym, pfe_rela); } else if (sym->sec->rela) { list_for_each_entry(rela, &sym->sec->rela->relas, list) { if (!strcmp(rela->sym->name, "_mcount")) { @@ -4466,8 +5240,10 @@ static void kpatch_find_func_profiling_calls(struct kpatch_elf *kelf) break; case AARCH64: case LOONGARCH64: - if (kpatch_symbol_has_pfe_entry(kelf, sym)) - sym->has_func_profiling = 1; + case RISCV64: + pfe_rela = kpatch_find_pfe_rela(kelf, sym); + if (pfe_rela) + kpatch_set_func_profiling_callsite(sym, pfe_rela); break; default: ERROR("unsupported arch"); @@ -4584,6 +5360,7 @@ int main(int argc, char *argv[]) * section, symbol, and rela lists of kelf_patched. */ kpatch_mark_ignored_sections(kelf_patched); + kpatch_mark_equal_error_injection_whitelist_same(kelf_patched); kpatch_compare_correlated_elements(kelf_patched); kpatch_mark_ignored_functions_same(kelf_patched); kpatch_mark_ignored_sections_same(kelf_patched); diff --git a/kpatch-build/create-klp-module.c b/kpatch-build/create-klp-module.c index fef92c854..f1c5fa29e 100644 --- a/kpatch-build/create-klp-module.c +++ b/kpatch-build/create-klp-module.c @@ -25,11 +25,49 @@ #include "log.h" #include "kpatch-elf.h" #include "kpatch-intermediate.h" +#include "kpatch-riscv-insn.h" /* For log.h */ char *childobj; enum loglevel loglevel = NORMAL; +enum riscv_lo12_patch_result { + RISCV_LO12_KEEP_RELA, + RISCV_LO12_RELA_HANDLED, +}; + +struct riscv_pcrel_slot { + struct list_head list; + struct section *sec; + unsigned int offset; +}; + +struct writable_section { + struct list_head list; + struct section *sec; +}; + +static LIST_HEAD(riscv_pcrel_slots); +static LIST_HEAD(writable_sections); + +static struct section *find_or_add_klp_relasec(struct kpatch_elf *kelf, + struct section *base, + char *objname); + +static void link_local_symbol(struct kpatch_elf *kelf, struct symbol *sym) +{ + struct list_head *head; + struct symbol *s; + + head = &kelf->symbols; + list_for_each_entry(s, &kelf->symbols, list) { + if (!is_local_sym(s)) + break; + head = &s->list; + } + list_add_tail(&sym->list, head); +} + /* * Add a symbol from .kpatch.symbols to the symbol table * @@ -94,24 +132,409 @@ static struct symbol *find_or_add_ksym_to_symbols(struct kpatch_elf *kelf, * a) locals need to be grouped together, before globals * b) globals can be tacked into the end of the list */ - if (is_local_sym(sym)) { - struct list_head *head; - struct symbol *s; - - head = &kelf->symbols; - list_for_each_entry(s, &kelf->symbols, list) { - if (!is_local_sym(s)) - break; - head = &s->list; - } - list_add_tail(&sym->list, head); - } else { + if (is_local_sym(sym)) + link_local_symbol(kelf, sym); + else list_add_tail(&sym->list, &kelf->symbols); + + return sym; +} + +static struct symbol *find_or_add_local_anchor(struct kpatch_elf *kelf, + struct symbol *src, + long addend, + unsigned int index) +{ + struct symbol *sym, *anchor; + char buf[64]; + unsigned long value; + + value = src->sym.st_value + addend; + + list_for_each_entry(sym, &kelf->symbols, list) { + if (sym->sec == src->sec && + sym->type == STT_NOTYPE && + sym->bind == STB_LOCAL && + sym->sym.st_value == value && + sym->name && + !strncmp(sym->name, KPATCH_RISCV_HI20_PREFIX, + sizeof(KPATCH_RISCV_HI20_PREFIX) - 1)) + return sym; + } + + ALLOC_LINK(anchor, NULL); + snprintf(buf, sizeof(buf), KPATCH_RISCV_HI20_PREFIX "%u", index); + anchor->name = strdup(buf); + if (!anchor->name) + ERROR("strdup"); + anchor->sec = src->sec; + anchor->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_NOTYPE); + anchor->sym.st_value = value; + anchor->type = STT_NOTYPE; + anchor->bind = STB_LOCAL; + + link_local_symbol(kelf, anchor); + + return anchor; +} + +static size_t append_zeroed_section_data(struct section *sec, size_t len, + size_t align) +{ + size_t old_size, offset, new_size; + void *buf; + + old_size = sec->data->d_size; + offset = align ? (old_size + align - 1) & ~(align - 1) : old_size; + new_size = offset + len; + + buf = realloc(sec->data->d_buf, new_size); + if (!buf) + ERROR("realloc"); + + sec->data->d_buf = buf; + memset((char *)sec->data->d_buf + old_size, 0, new_size - old_size); + sec->data->d_size = new_size; + sec->sh.sh_size = new_size; + + return offset; +} + +static struct symbol *add_riscv_klp_data_slot(struct kpatch_elf *kelf) +{ + static unsigned int slot_index; + struct section *sec; + struct symbol *sym; + char buf[64]; + size_t offset; + + sec = find_section_by_name(&kelf->sections, KPATCH_RISCV_KLP_DATA_SEC); + if (!sec) { + sec = create_section_pair(kelf, KPATCH_RISCV_KLP_DATA_SEC, + sizeof(unsigned long), 1); + sec->sh.sh_flags = SHF_ALLOC | SHF_WRITE; + offset = 0; + } else { + offset = append_zeroed_section_data(sec, sizeof(unsigned long), 0); } + ALLOC_LINK(sym, NULL); + snprintf(buf, sizeof(buf), KPATCH_RISCV_SLOT_PREFIX "%u", slot_index++); + sym->name = strdup(buf); + if (!sym->name) + ERROR("strdup"); + sym->sec = sec; + sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_OBJECT); + sym->sym.st_value = offset; + sym->sym.st_size = sizeof(unsigned long); + sym->type = STT_OBJECT; + sym->bind = STB_LOCAL; + link_local_symbol(kelf, sym); + return sym; } +static void add_rela(struct section *relasec, unsigned int offset, + unsigned int type, struct symbol *sym, long addend) +{ + struct rela *rela; + + ALLOC_LINK(rela, &relasec->relas); + rela->offset = offset; + rela->type = type; + rela->sym = sym; + rela->addend = addend; +} + +static void mark_riscv_pcrel_slot(struct section *sec, unsigned int offset) +{ + struct riscv_pcrel_slot *slot; + + ALLOC_LINK(slot, NULL); + slot->sec = sec; + slot->offset = offset; + list_add_tail(&slot->list, &riscv_pcrel_slots); +} + +static bool is_riscv_pcrel_slot(struct section *sec, unsigned int offset) +{ + struct riscv_pcrel_slot *slot; + + list_for_each_entry(slot, &riscv_pcrel_slots, list) { + if (slot->sec == sec && slot->offset == offset) + return true; + } + + return false; +} + +static void ensure_section_writable(struct section *sec) +{ + struct writable_section *wsec; + void *buf; + + list_for_each_entry(wsec, &writable_sections, list) { + if (wsec->sec == sec) + return; + } + + buf = malloc(sec->data->d_size); + if (!buf) + ERROR("malloc"); + if (sec->data->d_buf) + memcpy(buf, sec->data->d_buf, sec->data->d_size); + else if (sec->data->d_size) + ERROR("section %s has size %zu but no data buffer", + sec->name, sec->data->d_size); + sec->data->d_buf = buf; + + ALLOC_LINK(wsec, NULL); + wsec->sec = sec; + list_add_tail(&wsec->list, &writable_sections); +} + +static struct rela *find_riscv_slot_hi20_rela(struct section *sec, + unsigned int offset) +{ + struct rela *rela; + + list_for_each_entry(rela, &sec->rela->relas, list) { + if (rela->offset == offset && rela->type == R_RISCV_PCREL_HI20 && + rela->sym && rela->sym->sec && + !strcmp(rela->sym->sec->name, KPATCH_RISCV_KLP_DATA_SEC)) + return rela; + } + + return NULL; +} + +static bool rewrite_riscv_addi_lo12_load(struct section *sec, + unsigned int offset, + unsigned int insn, + unsigned int rd, + unsigned int rs1) +{ + if (riscv_opcode(insn) != RISCV_OPCODE_OP_IMM || rd != rs1) + return false; + + /* Convert addi rd, rs1, imm to ld rd, imm(rs1). */ + insn = riscv_set_opcode_funct3(insn, RISCV_OPCODE_LOAD, + RISCV_FUNCT3_LD); + memcpy((char *)sec->data->d_buf + offset, &insn, sizeof(insn)); + + return true; +} + +static void emit_riscv_load_trampoline(struct kpatch_elf *kelf, + struct section *sec, + unsigned int offset, + struct symbol *hi20_anchor, + unsigned int insn, + unsigned int rd, + unsigned int rs1) +{ + static unsigned int trampoline_index; + struct rela *hi20_rela; + struct symbol *trampoline_anchor; + unsigned int hi20_insn, hi20_rd, call_jalr; + unsigned int trampoline[4], hi20_offset, trampoline_offset; + + /* + * A direct load consumes the resolved symbol value, while the KLP data + * slot contains its address. Preserve the extra dereference in an + * out-of-line sequence: + * + * auipc rd, slot_hi auipc rd, trampoline_hi + * load rd, slot_lo(rd) jalr x0, trampoline_lo(rd) + * ... + * trampoline: + * auipc rd, slot_hi + * ld rd, slot_lo(rd) + * load rd, 0(rd) + * jal x0, return + */ + if (riscv_opcode(insn) != RISCV_OPCODE_LOAD || rd != rs1 || rd == 0) + ERROR("unsupported RISC-V PCREL_LO12 instruction at %s+0x%x", + sec->name, offset); + + hi20_offset = (unsigned int)hi20_anchor->sym.st_value; + if (offset != hi20_offset + RISCV_INSN_SIZE || + hi20_offset + sizeof(hi20_insn) > sec->data->d_size) + ERROR("unsupported RISC-V PCREL_HI20/LO12 layout at %s+0x%x", + sec->name, offset); + + memcpy(&hi20_insn, (char *)sec->data->d_buf + hi20_offset, + sizeof(hi20_insn)); + hi20_rd = riscv_rd(hi20_insn); + if (riscv_opcode(hi20_insn) != RISCV_OPCODE_AUIPC || hi20_rd != rd) + ERROR("unsupported RISC-V PCREL_HI20 instruction at %s+0x%x", + sec->name, hi20_offset); + + hi20_rela = find_riscv_slot_hi20_rela(sec, hi20_offset); + if (!hi20_rela) + ERROR("missing RISC-V KLP slot relocation at %s+0x%x", + sec->name, hi20_offset); + + trampoline_offset = (unsigned int)append_zeroed_section_data(sec, + sizeof(trampoline), RISCV_INSN_SIZE); + + encode_riscv_auipc_jalr(rd, (long)trampoline_offset - hi20_offset, + &hi20_insn, &call_jalr); + memcpy((char *)sec->data->d_buf + hi20_offset, &hi20_insn, + sizeof(hi20_insn)); + memcpy((char *)sec->data->d_buf + offset, &call_jalr, + sizeof(call_jalr)); + + /* Move the normal HI20 relocation from the call site to the trampoline. */ + hi20_rela->offset = trampoline_offset; + trampoline[0] = riscv_auipc_insn(rd); /* auipc rd, slot_hi */ + trampoline[1] = riscv_ld_insn(rd, rd); /* ld rd, slot_lo(rd) */ + trampoline[2] = insn & ~RISCV_IMM_MASK; /* original load, imm=0 */ + trampoline[3] = encode_riscv_jal(0, + (long)(offset + RISCV_INSN_SIZE) - + (trampoline_offset + 3 * RISCV_INSN_SIZE)); + memcpy((char *)sec->data->d_buf + trampoline_offset, trampoline, + sizeof(trampoline)); + + trampoline_anchor = find_or_add_local_anchor(kelf, hi20_anchor, + (long)trampoline_offset - hi20_offset, + RISCV_TRAMPOLINE_INDEX_BASE + trampoline_index++); + add_rela(sec->rela, trampoline_offset + RISCV_INSN_SIZE, + R_RISCV_PCREL_LO12_I, trampoline_anchor, 0); +} + +static enum riscv_lo12_patch_result +patch_riscv_lo12_load(struct kpatch_elf *kelf, + struct section *sec, unsigned int offset, + struct symbol *hi20_anchor) +{ + unsigned int insn, rd, rs1; + + if (offset + sizeof(insn) > sec->data->d_size) + ERROR("RISC-V LO12 offset out of range for %s", sec->name); + + ensure_section_writable(sec); + memcpy(&insn, (char *)sec->data->d_buf + offset, sizeof(insn)); + rd = riscv_rd(insn); + rs1 = riscv_rs1(insn); + + if (rewrite_riscv_addi_lo12_load(sec, offset, insn, rd, rs1)) + return RISCV_LO12_KEEP_RELA; + + emit_riscv_load_trampoline(kelf, sec, offset, hi20_anchor, + insn, rd, rs1); + + return RISCV_LO12_RELA_HANDLED; +} + +static bool is_riscv_linker_rela(const struct rela *rela) +{ + return rela->type == R_RISCV_ALIGN || + rela->type == R_RISCV_RELAX; +} + +static void remove_riscv_linker_relas(struct kpatch_elf *kelf) +{ + struct section *relasec; + struct rela *rela, *safe; + + if (kelf->arch != RISCV64) + return; + + list_for_each_entry(relasec, &kelf->sections, list) { + if (!is_rela_section(relasec)) + continue; + + list_for_each_entry_safe(rela, safe, &relasec->relas, list) { + if (!is_riscv_linker_rela(rela)) + continue; + + list_del(&rela->list); + free(rela); + } + } +} + +static void normalize_riscv_lo12_relas(struct kpatch_elf *kelf) +{ + struct section *relasec; + struct rela *rela; + unsigned int index = 0; + + if (kelf->arch != RISCV64) + return; + + list_for_each_entry(relasec, &kelf->sections, list) { + if (!is_rela_section(relasec)) + continue; + + list_for_each_entry(rela, &relasec->relas, list) { + if (!is_riscv_lo12_rela(rela)) + continue; + + rela->sym = find_or_add_local_anchor(kelf, rela->sym, + rela->addend, + index++); + rela->addend = 0; + } + } +} + +static void create_riscv_klp_addr_relas(struct kpatch_elf *kelf, + struct symbol *dest, + unsigned int dest_off, + char *objname, + struct symbol *klp_sym, + long klp_addend, + bool patch_lo12) +{ + struct section *klp_relasec; + struct symbol *slot; + + if (!dest->sec->rela) + ERROR("missing rela section for %s", dest->sec->name); + + slot = add_riscv_klp_data_slot(kelf); + + add_rela(dest->sec->rela, + (unsigned int)(dest->sym.st_value + dest_off), + R_RISCV_PCREL_HI20, slot, 0); + if (patch_lo12) + mark_riscv_pcrel_slot(dest->sec, + (unsigned int)(dest->sym.st_value + dest_off)); + + klp_relasec = find_or_add_klp_relasec(kelf, slot->sec, objname); + if (!klp_relasec) + ERROR("error finding or adding .klp.rela section"); + + add_rela(klp_relasec, (unsigned int)slot->sym.st_value, + R_RISCV_64, klp_sym, klp_addend); +} + +static void create_riscv_local_rela(struct kpatch_elf *kelf, + struct symbol *dest, + unsigned int dest_off, + struct symbol *src, + unsigned int type) +{ + enum riscv_lo12_patch_result result; + + if (!dest->sec->rela) + ERROR("missing rela section for %s", dest->sec->name); + + if (is_riscv_pcrel_slot(dest->sec, (unsigned int)src->sym.st_value)) { + result = patch_riscv_lo12_load(kelf, dest->sec, + (unsigned int)(dest->sym.st_value + dest_off), + src); + if (result == RISCV_LO12_RELA_HANDLED) + return; + } + + add_rela(dest->sec->rela, + (unsigned int)(dest->sym.st_value + dest_off), + type, src, 0); +} + /* * Create a .klp.rela section given the base section and objname * @@ -176,6 +599,7 @@ static void create_klp_relasecs_and_syms(struct kpatch_elf *kelf, struct section struct rela *rela; char *objname; unsigned int nr, index, offset, dest_off; + long src_addend; krelas = krelasec->data->d_buf; nr = (unsigned int)(krelasec->data->d_size / sizeof(*krelas)); @@ -206,11 +630,31 @@ static void create_klp_relasecs_and_syms(struct kpatch_elf *kelf, struct section if (!rela) ERROR("find_rela_by_offset"); - /* Create (or find) a klp symbol from the rela src entry */ - sym = find_or_add_ksym_to_symbols(kelf, ksymsec, strings, - (unsigned int)rela->addend); - if (!sym) - ERROR("error finding or adding ksym to symtab"); + src_addend = krelas[index].addend; + if (krelas[index].local) { + sym = find_or_add_local_anchor(kelf, rela->sym, + rela->addend, index); + src_addend = 0; + } else { + /* Create (or find) a klp symbol from the rela src entry */ + sym = find_or_add_ksym_to_symbols(kelf, ksymsec, strings, + (unsigned int)rela->addend); + if (!sym) + ERROR("error finding or adding ksym to symtab"); + } + + if (kelf->arch == RISCV64 && krelas[index].local) { + create_riscv_local_rela(kelf, dest, dest_off, sym, + krelas[index].type); + continue; + } else if (kelf->arch == RISCV64 && + (krelas[index].type == R_RISCV_GOT_HI20 || + krelas[index].type == R_RISCV_PCREL_HI20)) { + create_riscv_klp_addr_relas(kelf, dest, dest_off, + objname, sym, src_addend, + krelas[index].type == R_RISCV_PCREL_HI20); + continue; + } /* Create (or find) the .klp.rela section for the dest sec and object */ klp_relasec = find_or_add_klp_relasec(kelf, dest->sec, objname); @@ -222,7 +666,7 @@ static void create_klp_relasecs_and_syms(struct kpatch_elf *kelf, struct section rela->offset = (unsigned int)(dest->sym.st_value + dest_off); rela->type = krelas[index].type; rela->sym = sym; - rela->addend = krelas[index].addend; + rela->addend = src_addend; } } @@ -486,6 +930,8 @@ int main(int argc, char *argv[]) } remove_intermediate_sections(kelf); + remove_riscv_linker_relas(kelf); + normalize_riscv_lo12_relas(kelf); kpatch_reindex_elements(kelf); /* Rebuild rela sections, new .klp.rela sections will be rebuilt too. */ diff --git a/kpatch-build/kpatch-build b/kpatch-build/kpatch-build index 67871a7ba..adc9529f7 100755 --- a/kpatch-build/kpatch-build +++ b/kpatch-build/kpatch-build @@ -53,6 +53,9 @@ RELEASE_FILE=/etc/os-release DEBUG=0 SKIPCLEANUP=0 SKIPCOMPILERCHECK=0 +SKIP_VMLINUX_FINAL_LINK=0 +CUSTOM_TARGETS=0 +MODULE_SYMVERS_WAS_ABSENT=0 ARCH_KCFLAGS="" DEBUG_KCFLAGS="" declare -a PATCH_LIST @@ -63,10 +66,18 @@ KLP_REPLACE=1 GCC="${CROSS_COMPILE:-}gcc" CLANG="${CROSS_COMPILE:-}clang" LD="${CROSS_COMPILE:-}ld" -LLD="${CROSS_COMPILE:-}ld.lld" READELF="${CROSS_COMPILE:-}readelf" OBJCOPY="${CROSS_COMPILE:-}objcopy" +if [[ -z "${LLD:-}" ]]; then + if [[ -n "${CROSS_COMPILE:-}" ]] && + command -v "${CROSS_COMPILE}ld.lld" >/dev/null 2>&1; then + LLD="${CROSS_COMPILE}ld.lld" + else + LLD="ld.lld" + fi +fi + declare -rA SUPPORTED_DEB_DISTROS=( ["debian"]="Debian OS" @@ -220,6 +231,10 @@ cleanup() { remove_patches restore_kernel_files + if [[ "$MODULE_SYMVERS_WAS_ABSENT" -eq 1 ]]; then + rm -f "$BUILDDIR/Module.symvers" + fi + [[ "$DEBUG" -eq 0 ]] && rm -rf "$TEMPDIR" rm -rf "$RPMTOPDIR" unset KCFLAGS @@ -440,6 +455,11 @@ find_special_section_data() { "loongarch64") check[a]=true # alt_instr ;; + "riscv64") + check[a]=true # alt_entry + check[b]=true # bug_entry + check[j]=true # jump_entry + ;; esac # Kernel CONFIG_ features @@ -460,7 +480,7 @@ find_special_section_data() { BEGIN { a = b = e = f = i = j = o = p = s = 0 } # Set state if name matches - check_a && a == 0 && /DW_AT_name.* alt_instr[[:space:]]*$/ {a = 1; next} + check_a && a == 0 && /DW_AT_name.* alt_(instr|entry)[[:space:]]*$/ {a = 1; next} check_b && b == 0 && /DW_AT_name.* bug_entry[[:space:]]*$/ {b = 1; next} check_e && e == 0 && /DW_AT_name.* exception_table_entry[[:space:]]*$/ {e = 1; next} check_f && f == 0 && /DW_AT_name.* fixup_entry[[:space:]]*$/ {f = 1; next} @@ -732,6 +752,77 @@ find_kobj() { done } +object_is_module() { + local obj="$1" + local cmdfile + local dry_run + + cmdfile="$(dirname "$obj")/.$(basename "$obj").cmd" + if [[ -e "$cmdfile" ]]; then + grep -Eq '(^|[[:space:]])-DMODULE([[:space:]]|$)' "$cmdfile" + return + fi + + # A clean tree may not have a .cmd file for a CONFIG=m object. + # Ask Kbuild to resolve the target without executing its recipes. + dry_run="$(make --no-print-directory -n \ + "${ARCH_MAKEVARS[@]}" "${MAKEVARS[@]}" "$obj" 2>&1)" || + die "--skip-vmlinux-final-link cannot determine the target for $obj" + + grep -Eq '(^|[[:space:]])-DMODULE([[:space:]]|$)' <<< "$dry_run" +} + +verify_skip_vmlinux_patch_targets() { + local patch + local path + local obj + local saved_pwd="$PWD" + + [[ "$SKIP_VMLINUX_FINAL_LINK" -eq 1 ]] || return + + cd "$BUILDDIR" || die + for patch in "${PATCH_LIST[@]}"; do + while IFS= read -r path; do + case "$path" in + *.c) + obj="${path%.c}.o" + if object_is_module "$obj"; then + die "--skip-vmlinux-final-link supports vmlinux objects only; $path is built as a module" + fi + ;; + *) + die "--skip-vmlinux-final-link cannot verify that $path affects vmlinux objects only" + ;; + esac + done < <(awk '($1 == "---" || $1 == "+++") && + NF >= 2 && $2 != "/dev/null" { + path = $2 + sub(/^[^/]*\//, "", path) + print path + }' "$patch" | sort -u) + done + cd "$saved_pwd" || die +} + +verify_skip_vmlinux_changed_objs() { + local obj + local saved_pwd="$PWD" + + [[ "$SKIP_VMLINUX_FINAL_LINK" -eq 1 ]] || return + + cd "$BUILDDIR" || die + while IFS= read -r obj; do + [[ -n "$obj" ]] || continue + if object_is_module "$obj"; then + die "--skip-vmlinux-final-link supports vmlinux objects only; $obj is built as a module" + fi + find_kobj "$obj" + [[ "$KOBJFILE" = vmlinux ]] || + die "--skip-vmlinux-final-link supports vmlinux objects only; $obj belongs to $KOBJFILE" + done < "$TEMPDIR/changed_objs" + cd "$saved_pwd" || die +} + # Only allow alphanumerics and '_' and '-' in the module name. Everything else # is replaced with '-'. Also truncate to 55 chars so the full name + NUL # terminator fits in the kernel's 56-byte module name array. @@ -769,6 +860,7 @@ kernel_src_arch() { ppc64le) echo "powerpc" ;; s390x) echo "s390" ;; loongarch64) echo "loongarch" ;; + riscv64) echo "riscv" ;; *) echo "$1" ;; esac } @@ -794,6 +886,9 @@ usage() { echo " --oot-module-src Specify out-of-tree module source directory" >&2 echo " -R, --non-replace Disable replace patch (replace is on by default)" >&2 echo " --skip-cleanup Skip post-build cleanup" >&2 + echo " --skip-vmlinux-final-link RISC-V vmlinux objects only: skip original and" >&2 + echo " patched final vmlinux links after building" >&2 + echo " vmlinux.o and running modpost" >&2 echo " --skip-compiler-check Skip compiler version matching check" >&2 echo " (not recommended)" >&2 echo " --version Version of kpatch-build" @@ -803,7 +898,7 @@ if ! command -v gawk &> /dev/null; then die "gawk not installed" fi -options="$(getopt -o ha:r:s:c:v:j:t:n:o:dR -l "help,archversion:,sourcerpm:,sourcedir:,config:,vmlinux:,jobs:,target:,name:,output:,oot-module:,oot-module-src:,debug,skip-gcc-check,skip-compiler-check,skip-cleanup,non-replace,version" -- "$@")" || die "getopt failed" +options="$(getopt -o ha:r:s:c:v:j:t:n:o:dR -l "help,archversion:,sourcerpm:,sourcedir:,config:,vmlinux:,jobs:,target:,name:,output:,oot-module:,oot-module-src:,debug,skip-gcc-check,skip-compiler-check,skip-cleanup,skip-vmlinux-final-link,non-replace,version" -- "$@")" || die "getopt failed" eval set -- "$options" @@ -844,6 +939,7 @@ while [[ $# -gt 0 ]]; do ;; -t|--target) TARGETS="$TARGETS $2" + CUSTOM_TARGETS=1 shift ;; -n|--name) @@ -878,6 +974,9 @@ while [[ $# -gt 0 ]]; do echo "Skipping cleanup" SKIPCLEANUP=1 ;; + --skip-vmlinux-final-link) + SKIP_VMLINUX_FINAL_LINK=1 + ;; --skip-gcc-check) echo "DEPRECATED: --skip-gcc-check is deprecated, use --skip-compiler-check instead" ;& @@ -904,6 +1003,15 @@ if [[ ${#PATCH_LIST[@]} -eq 0 ]]; then exit 1 fi +if [[ "$SKIP_VMLINUX_FINAL_LINK" -eq 1 ]]; then + [[ "$TARGET_ARCH" = riscv64 ]] || + die "--skip-vmlinux-final-link is only supported for RISC-V builds" + [[ "$CUSTOM_TARGETS" -eq 0 ]] || + die "--skip-vmlinux-final-link cannot be combined with --target" + [[ -z "$OOT_MODULE" ]] || + die "--skip-vmlinux-final-link cannot be used for out-of-tree modules" +fi + trace_on # Don't check external file. # shellcheck disable=SC1090 @@ -1314,6 +1422,10 @@ if [[ "$ARCH" = "loongarch64" ]]; then fi fi +if [[ "$TARGET_ARCH" = "riscv64" ]]; then + ARCH_KCFLAGS="-fPIC" +fi + export KCFLAGS="-I$DATADIR/patch -ffunction-sections -fdata-sections \ $ARCH_KCFLAGS $DEBUG_KCFLAGS" @@ -1325,6 +1437,17 @@ if [[ $DEBUG -ge 4 ]]; then fi save_env +if [[ "$SKIP_VMLINUX_FINAL_LINK" -eq 1 ]]; then + # Module.symvers is temporarily replaced with vmlinux.symvers below + # for the external livepatch module build. Preserve the original + # kernel tree state across kpatch-build. + if [[ -e "$BUILDDIR/Module.symvers" ]]; then + backup_kernel_file "Module.symvers" + else + MODULE_SYMVERS_WAS_ABSENT=1 + fi +fi + echo "Building original source" unset KPATCH_GCC_TEMPDIR @@ -1350,12 +1473,28 @@ if [[ "$ARCH" != "$TARGET_ARCH" ]]; then fi -# $TARGETS used as list, no quotes. +if [[ "$SKIP_VMLINUX_FINAL_LINK" -eq 1 ]]; then + echo "Skipping original vmlinux final link; building vmlinux.o and running modpost" + rm -f "$BUILDDIR/Module.symvers" + ORIGINAL_TARGETS="vmlinux.o modpost" +else + ORIGINAL_TARGETS="$TARGETS" +fi +# $ORIGINAL_TARGETS used as list, no quotes. # shellcheck disable=SC2086 -make "${ARCH_MAKEVARS[@]}" "${MAKEVARS[@]}" "-j$CPUS" $TARGETS 2>&1 | logger || die +make "${ARCH_MAKEVARS[@]}" "${MAKEVARS[@]}" "-j$CPUS" \ + $ORIGINAL_TARGETS 2>&1 | logger || die -# Save original module symvers -cp -f "$BUILDDIR/Module.symvers" "$TEMPDIR/Module.symvers" || die +# Save original symbol versions +if [[ "$SKIP_VMLINUX_FINAL_LINK" -eq 1 ]]; then + [[ -s "$BUILDDIR/vmlinux.symvers" ]] || die "original vmlinux.symvers was not generated" + cp -f "$BUILDDIR/vmlinux.symvers" "$TEMPDIR/Module.symvers" || die +else + [[ -s "$BUILDDIR/Module.symvers" ]] || die "original Module.symvers was not generated" + cp -f "$BUILDDIR/Module.symvers" "$TEMPDIR/Module.symvers" || die +fi + +verify_skip_vmlinux_patch_targets echo "Building patched source" apply_patches @@ -1363,9 +1502,17 @@ mkdir -p "$TEMPDIR/orig" "$TEMPDIR/patched" export KPATCH_GCC_TEMPDIR="$TEMPDIR" export KPATCH_GCC_SRCDIR="$BUILDDIR" save_env -# $TARGETS used as list, no quotes. +if [[ "$SKIP_VMLINUX_FINAL_LINK" -eq 1 ]]; then + echo "Skipping patched vmlinux final link; building vmlinux.o and running modpost" + rm -f "$BUILDDIR/Module.symvers" + PATCHED_TARGETS="vmlinux.o modpost" +else + PATCHED_TARGETS="$TARGETS" +fi +# $PATCHED_TARGETS used as list, no quotes. # shellcheck disable=SC2086 -KBUILD_MODPOST_WARN=1 make "${ARCH_MAKEVARS[@]}" "${MAKEVARS[@]}" "-j$CPUS" $TARGETS 2>&1 | logger || die +KBUILD_MODPOST_WARN=1 make "${ARCH_MAKEVARS[@]}" "${MAKEVARS[@]}" "-j$CPUS" \ + $PATCHED_TARGETS 2>&1 | logger || die # source.c:(.section+0xFF): undefined reference to `symbol' grep "undefined reference" "$LOGFILE" | sed -r "s/^.*\`(.*)'$/\\1/" \ @@ -1378,9 +1525,32 @@ if [[ ! -e "$TEMPDIR/changed_objs" ]]; then die "no changed objects found" fi -grep -q vmlinux "$KERNEL_SRCDIR/Module.symvers" || die "truncated $KERNEL_SRCDIR/Module.symvers file" +verify_skip_vmlinux_changed_objs -if [[ -n "$CONFIG_MODVERSIONS" ]]; then +if [[ "$SKIP_VMLINUX_FINAL_LINK" -eq 1 ]]; then + BUILD_SYMVERS_FILE="$BUILDDIR/vmlinux.symvers" +else + BUILD_SYMVERS_FILE="$BUILDDIR/Module.symvers" +fi + +grep -q vmlinux "$BUILD_SYMVERS_FILE" || die "truncated $BUILD_SYMVERS_FILE file" + +if [[ "$SKIP_VMLINUX_FINAL_LINK" -eq 1 ]]; then + ORIG_SYMVERS_SORTED="$TEMPDIR/Module.symvers.orig.sorted" + PATCHED_SYMVERS_SORTED="$TEMPDIR/Module.symvers.patched.sorted" + SYMVERS_DIFF="$CACHEDIR/Module.symvers.diff" + + [[ -s "$BUILDDIR/vmlinux.symvers" ]] || die "patched vmlinux.symvers was not generated" + LC_ALL=C sort "$TEMPDIR/Module.symvers" > "$ORIG_SYMVERS_SORTED" || die + LC_ALL=C sort "$BUILDDIR/vmlinux.symvers" > "$PATCHED_SYMVERS_SORTED" || die + if ! cmp -s "$ORIG_SYMVERS_SORTED" "$PATCHED_SYMVERS_SORTED"; then + diff -u "$ORIG_SYMVERS_SORTED" "$PATCHED_SYMVERS_SORTED" > "$SYMVERS_DIFF" || true + die "patched vmlinux.symvers differs from the original; see $SYMVERS_DIFF" + fi + + # External module builds consume Module.symvers from the kernel tree. + cp -f "$BUILDDIR/vmlinux.symvers" "$BUILDDIR/Module.symvers" || die +elif [[ -n "$CONFIG_MODVERSIONS" ]]; then trace_off "reading Module.symvers" while read -ra sym_line; do if [[ ${#sym_line[@]} -lt 4 ]]; then @@ -1473,13 +1643,13 @@ for i in $FILES; do KOBJFILE_NAME=vmlinux KOBJFILE_PATH="$VMLINUX" SYMTAB="${TEMPDIR}/${KOBJFILE_NAME}.symtab" - SYMVERS_FILE="$BUILDDIR/Module.symvers" + SYMVERS_FILE="$BUILD_SYMVERS_FILE" else KOBJFILE_NAME=$(basename "${KOBJFILE%.ko}") KOBJFILE_NAME="${KOBJFILE_NAME//-/_}" KOBJFILE_PATH="${TEMPDIR}/module/$KOBJFILE" SYMTAB="${KOBJFILE_PATH}.symtab" - SYMVERS_FILE="$BUILDDIR/Module.symvers" + SYMVERS_FILE="$BUILD_SYMVERS_FILE" fi "$READELF" -s --wide "$KOBJFILE_PATH" > "$SYMTAB" diff --git a/kpatch-build/kpatch-cc b/kpatch-build/kpatch-cc index ad23df932..50ecd0e06 100755 --- a/kpatch-build/kpatch-cc +++ b/kpatch-build/kpatch-cc @@ -44,6 +44,7 @@ if [[ "$TOOLCHAINCMD" =~ ^(.*-)?gcc$ || "$TOOLCHAINCMD" =~ ^(.*-)?clang$ ]] ; th arch/s390/purgatory/*|\ arch/s390/kernel/vdso64/*|\ arch/loongarch/vdso/*|\ + arch/riscv/vdso/*|\ drivers/firmware/efi/libstub/*|\ init/version.o|\ init/version-timestamp.o|\ diff --git a/kpatch-build/kpatch-elf.c b/kpatch-build/kpatch-elf.c index d418ecda5..e52dee04b 100755 --- a/kpatch-build/kpatch-elf.c +++ b/kpatch-build/kpatch-elf.c @@ -158,6 +158,8 @@ unsigned int absolute_rela_type(struct kpatch_elf *kelf) return R_AARCH64_ABS64; case LOONGARCH64: return R_LARCH_64; + case RISCV64: + return R_RISCV_64; default: ERROR("unsupported arch"); } @@ -224,6 +226,7 @@ long rela_target_offset(struct kpatch_elf *kelf, struct section *relasec, case PPC64: case AARCH64: case LOONGARCH64: + case RISCV64: add_off = 0; break; case X86_64: @@ -284,6 +287,15 @@ unsigned int insn_length(struct kpatch_elf *kelf, void *addr) case LOONGARCH64: return 4; + case RISCV64: + /* + * RISC-V instruction length is determined by the lowest 2 bits: + * 0b11 -> 32-bit (base) instruction + * 0b00, 0b01, 0b10 -> 16-bit compressed (C-ext) instruction + * This matches the kernel's GET_INSN_LENGTH() macro. + */ + return (*(unsigned char *)addr & 0x3) == 0x3 ? 4 : 2; + case S390: switch(insn[0] >> 6) { case 0: @@ -353,13 +365,33 @@ static void kpatch_create_rela_list(struct kpatch_elf *kelf, rela->sym->name, rela->addend); } - if (kelf->arch == LOONGARCH64) { + if (kelf->arch == LOONGARCH64 || kelf->arch == RISCV64) { /* - * LoongArch GCC creates local labels such as .LBB7266, - * replace them with section symbols. + * LoongArch and RISC-V GCC create local labels such as + * .LBB7266 / .L1245, replace them with section symbols + * to avoid false CHANGED detection due to label + * renumbering between compilations. + * Apply to text sections (branch target labels) and + * .rodata.* sections (function-specific read-only data + * like jump tables). Exclude special sections like + * __jump_table, __ex_table, .alternative which use + * local labels for specific purposes. */ if (rela->sym->sec && rela->sym->type == STT_NOTYPE && - rela->sym->bind == STB_LOCAL) { + rela->sym->bind == STB_LOCAL && + (is_text_section(rela->sym->sec) || + !strncmp(rela->sym->sec->name, ".rodata.", 8))) { + if (!rela->sym->sec->secsym) { + struct symbol *secsym; + + ALLOC_LINK(secsym, &kelf->symbols); + secsym->sec = rela->sym->sec; + secsym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION); + secsym->type = STT_SECTION; + secsym->bind = STB_LOCAL; + secsym->name = rela->sym->sec->name; + rela->sym->sec->secsym = secsym; + } log_debug("local label: %s -> ", rela->sym->name); rela->addend += rela->sym->sym.st_value; rela->sym = rela->sym->sec->secsym; @@ -636,6 +668,9 @@ struct kpatch_elf *kpatch_elf_open(const char *name) case EM_LOONGARCH: kelf->arch = LOONGARCH64; break; + case EM_RISCV: + kelf->arch = RISCV64; + break; default: ERROR("Unsupported target architecture"); } diff --git a/kpatch-build/kpatch-elf.h b/kpatch-build/kpatch-elf.h index 92e3bb68b..8256f3169 100644 --- a/kpatch-build/kpatch-elf.h +++ b/kpatch-build/kpatch-elf.h @@ -97,6 +97,7 @@ struct symbol { enum symbol_strip strip; /* used in the output elf */ }; int has_func_profiling; + long func_profiling_callsite; bool is_pfx; struct section *pfe; }; @@ -110,6 +111,7 @@ struct rela { long addend; char *string; bool need_klp_reloc; + bool riscv_paired_lo12; }; struct string { @@ -123,6 +125,7 @@ enum architecture { S390 = 0x1 << 2, AARCH64 = 0x1 << 3, LOONGARCH64 = 0x1 << 4, + RISCV64 = 0x1 << 5, }; struct kpatch_elf { @@ -187,6 +190,18 @@ bool is_local_func_sym(struct symbol *sym); bool is_local_sym(struct symbol *sym); bool is_ubsan_sec(const char *name); +static inline bool is_riscv_hi20_rela(const struct rela *rela) +{ + return rela->type == R_RISCV_GOT_HI20 || + rela->type == R_RISCV_PCREL_HI20; +} + +static inline bool is_riscv_lo12_rela(const struct rela *rela) +{ + return rela->type == R_RISCV_PCREL_LO12_I || + rela->type == R_RISCV_PCREL_LO12_S; +} + void print_strtab(char *buf, size_t size); void kpatch_create_shstrtab(struct kpatch_elf *kelf); void kpatch_create_strtab(struct kpatch_elf *kelf); diff --git a/kpatch-build/kpatch-intermediate.h b/kpatch-build/kpatch-intermediate.h index 2036cb3fd..45db35222 100644 --- a/kpatch-build/kpatch-intermediate.h +++ b/kpatch-build/kpatch-intermediate.h @@ -24,6 +24,10 @@ /* For .kpatch.{symbols,relocations,arch} sections */ +#define KPATCH_RISCV_HI20_PREFIX ".Lkpatch_riscv_hi20_" +#define KPATCH_RISCV_SLOT_PREFIX ".Lkpatch_riscv_slot_" +#define KPATCH_RISCV_KLP_DATA_SEC ".data.kpatch.riscv" + struct kpatch_symbol { unsigned long src; unsigned long sympos; @@ -36,6 +40,7 @@ struct kpatch_relocation { unsigned long dest; unsigned int type; int external; + int local; /* ksym points to a local anchor, not .kpatch.symbols */ long addend; char *objname; /* object to which this rela applies to */ struct kpatch_symbol *ksym; diff --git a/kpatch-build/kpatch-riscv-insn.h b/kpatch-build/kpatch-riscv-insn.h new file mode 100644 index 000000000..21a9595ef --- /dev/null +++ b/kpatch-build/kpatch-riscv-insn.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * RISC-V instruction encoding helpers for kpatch. + * + * Pure instruction field extraction and encoding functions with no + * dependency on kpatch data structures. + */ + +#ifndef _KPATCH_RISCV_INSN_H_ +#define _KPATCH_RISCV_INSN_H_ + +#include +#include "log.h" + +/* Instruction and field sizes */ +#define RISCV_INSN_SIZE 4 +#define RISCV_REG_MASK 0x1f +#define RISCV_OPCODE_MASK 0x7f + +/* Opcodes */ +#define RISCV_OPCODE_LOAD 0x03 +#define RISCV_OPCODE_OP_IMM 0x13 +#define RISCV_OPCODE_AUIPC 0x17 +#define RISCV_OPCODE_JALR 0x67 +#define RISCV_OPCODE_JAL 0x6f + +/* Funct3 */ +#define RISCV_FUNCT3_LD 0x3 + +/* Composite masks */ +#define RISCV_OPCODE_FUNCT3_MASK ((0x7u << 12) | RISCV_OPCODE_MASK) +#define RISCV_IMM_MASK 0xfff00000u /* I-type immediate, bits [31:20] */ + +/* Trampoline anchor index base */ +#define RISCV_TRAMPOLINE_INDEX_BASE 0x80000000u + +/* Register names */ +#define RISCV_REG_ZERO 0 + +/* Field extraction */ +static inline unsigned int riscv_opcode(unsigned int insn) +{ + return insn & RISCV_OPCODE_MASK; +} + +static inline unsigned int riscv_rd(unsigned int insn) +{ + return (insn >> 7) & RISCV_REG_MASK; +} + +static inline unsigned int riscv_rs1(unsigned int insn) +{ + return (insn >> 15) & RISCV_REG_MASK; +} + +/* Field modification */ +static inline unsigned int riscv_set_opcode_funct3(unsigned int insn, + unsigned int opcode, + unsigned int funct3) +{ + return (insn & ~RISCV_OPCODE_FUNCT3_MASK) | (funct3 << 12) | opcode; +} + +/* Common instruction synthesis */ +static inline unsigned int riscv_ld_insn(unsigned int rd, unsigned int rs1) +{ + return (rs1 << 15) | (rd << 7) | + (RISCV_FUNCT3_LD << 12) | RISCV_OPCODE_LOAD; +} + +static inline unsigned int riscv_auipc_insn(unsigned int rd) +{ + return (rd << 7) | RISCV_OPCODE_AUIPC; +} + +/* Instruction encoding */ +static inline unsigned int encode_riscv_jal(unsigned int rd, long offset) +{ + unsigned long imm; + + if ((offset & 1) || offset < -(1L << 20) || offset >= (1L << 20)) + ERROR("RISC-V JAL target out of range"); + + imm = (unsigned long)offset; + return (unsigned int)(((imm & 0x100000) << 11) | + ((imm & 0x7fe) << 20) | + ((imm & 0x800) << 9) | + (imm & 0xff000)) | + (rd << 7) | RISCV_OPCODE_JAL; +} + +static inline void encode_riscv_auipc_jalr(unsigned int rd, long offset, + unsigned int *auipc, + unsigned int *jalr) +{ + long hi20, lo12; + + if (offset < -(1L << 31) || offset >= (1L << 31)) + ERROR("RISC-V AUIPC/JALR target out of range"); + + hi20 = (offset + 0x800) & ~0xfffL; + lo12 = offset - hi20; + *auipc = ((unsigned int)hi20 & 0xfffff000u) | + (rd << 7) | RISCV_OPCODE_AUIPC; + *jalr = (((unsigned int)lo12 & 0xfffu) << 20) | + (rd << 15) | RISCV_OPCODE_JALR; +} + +#endif /* _KPATCH_RISCV_INSN_H_ */ diff --git a/kpatch/kpatch b/kpatch/kpatch index a90e29dad..81431baa2 100755 --- a/kpatch/kpatch +++ b/kpatch/kpatch @@ -149,7 +149,12 @@ else fi get_module_name () { - get_module_section_string "$1" .gnu.linkonce.this_module + local modname + modname="$(get_module_section_string "$1" .gnu.linkonce.this_module 2>/dev/null)" && [[ -n "$modname" ]] && { + echo "$modname" + return + } + modinfo -F name "$1" } init_sysfs_var() { diff --git a/test/difftree.sh b/test/difftree.sh index 9a0ebf446..3a98a6012 100755 --- a/test/difftree.sh +++ b/test/difftree.sh @@ -41,6 +41,11 @@ do arch/x86/boot/header.o|\ arch/x86/boot/compressed/efi_stub_64.o|\ arch/x86/boot/compressed/piggy.o|\ + arch/riscv/boot/version.o|\ + arch/riscv/boot/compressed/efi_stub_64.o|\ + arch/riscv/boot/compressed/piggy.o|\ + arch/riscv/boot/header.o|\ + arch/riscv/kernel/head.o|\ kernel/system_certificates.o|\ .*.o) continue diff --git a/test/integration/kpatch-test b/test/integration/kpatch-test index 33a8ba749..86d6d34d4 100755 --- a/test/integration/kpatch-test +++ b/test/integration/kpatch-test @@ -299,8 +299,22 @@ run_combined_test() { # save existing dmesg so we can detect new content save_dmesg() { - SAVED_DMESG="kpatch-test timestamp: $(date --rfc-3339=ns)" - echo "$SAVED_DMESG" > /dev/kmsg + local marker + + marker="kpatch-test timestamp: $(date --rfc-3339=ns)" + echo "$marker" > /dev/kmsg + SAVED_DMESG=$(dmesg --notime | awk -v marker="$marker" ' + length($0) >= length(marker) && + substr($0, length($0) - length(marker) + 1) == marker { + line = $0 + } + END {print line} + ') + + if [[ -z "$SAVED_DMESG" ]]; then + error "failed to save dmesg marker" + return 1 + fi } # new dmesg entries since our saved entry @@ -343,6 +357,10 @@ if ! support_klp_replace ; then echo "KLP replace is not supported on '${PRETTY_NAME}', disabling." fi +# Allow custom integration tests to invoke kpatch-build with the same binary +# and options as the main test run. +export KPATCHBUILD KPATCHBUILD_OPTS + for file in "${PATCH_LIST[@]}"; do if [[ $QUICK != 1 || "$file" =~ -FAIL ]]; then build_module "$file" @@ -353,7 +371,9 @@ build_combined_module unload_all -save_dmesg +if ! save_dmesg; then + exit 1 +fi if [ "${DYNDEBUG_ENABLED}" == "1" ]; then prev_dyndebug=$(sudo sh -c "grep klp_try_switch_task ${DYNDEBUG_CONTROL}" | awk '{print $3;}') diff --git a/test/integration/linux-6.12.95/data-new-LOADED.test b/test/integration/linux-6.12.95/data-new-LOADED.test new file mode 100644 index 000000000..6252e777b --- /dev/null +++ b/test/integration/linux-6.12.95/data-new-LOADED.test @@ -0,0 +1,4 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-or-later + +grep "kpatch: 5" /proc/meminfo diff --git a/test/integration/linux-6.12.95/data-new.patch b/test/integration/linux-6.12.95/data-new.patch new file mode 100644 index 000000000..1f269000e --- /dev/null +++ b/test/integration/linux-6.12.95/data-new.patch @@ -0,0 +1,20 @@ +diff -Nupr src.orig/fs/proc/meminfo.c src/fs/proc/meminfo.c +--- src.orig/fs/proc/meminfo.c ++++ src/fs/proc/meminfo.c +@@ -31,6 +31,8 @@ static void show_val_kb(struct seq_file + seq_write(m, " kB\n", 4); + } + ++static int foo = 5; ++ + static int meminfo_proc_show(struct seq_file *m, void *v) + { + struct sysinfo i; +@@ -155,6 +157,7 @@ static int meminfo_proc_show(struct seq_ + show_val_kb(m, "CmaFree: ", + global_zone_page_state(NR_FREE_CMA_PAGES)); + #endif ++ seq_printf(m, "kpatch: %d\n", foo); + + #ifdef CONFIG_UNACCEPTED_MEMORY + show_val_kb(m, "Unaccepted: ", diff --git a/test/integration/linux-6.12.95/gcc-static-local-var-6.patch b/test/integration/linux-6.12.95/gcc-static-local-var-6.patch new file mode 100644 index 000000000..7fb69da49 --- /dev/null +++ b/test/integration/linux-6.12.95/gcc-static-local-var-6.patch @@ -0,0 +1,22 @@ +diff -Nupr src.orig/net/ipv6/netfilter.c src/net/ipv6/netfilter.c +--- src.orig/net/ipv6/netfilter.c ++++ src/net/ipv6/netfilter.c +@@ -97,6 +97,8 @@ static int nf_ip6_reroute(struct sk_buff + return 0; + } + ++#include "kpatch-macros.h" ++ + int __nf_ip6_route(struct net *net, struct dst_entry **dst, + struct flowi *fl, bool strict) + { +@@ -110,6 +112,9 @@ int __nf_ip6_route(struct net *net, stru + struct dst_entry *result; + int err; + ++ if (!jiffies) ++ printk("kpatch nf_ip6_route foo\n"); ++ + result = ip6_route_output(net, sk, &fl->u.ip6); + err = result->error; + if (err) diff --git a/test/integration/linux-6.12.95/macro-callbacks.patch b/test/integration/linux-6.12.95/macro-callbacks.patch new file mode 100644 index 000000000..07722f9b6 --- /dev/null +++ b/test/integration/linux-6.12.95/macro-callbacks.patch @@ -0,0 +1,54 @@ +diff -Nupr src.orig/fs/aio.c src/fs/aio.c +--- src.orig/fs/aio.c ++++ src/fs/aio.c +@@ -50,6 +50,50 @@ + + #define KIOCB_KEY 0 + ++#include ++#include "kpatch-macros.h" ++ ++static const char *const module_state[] = { ++ [MODULE_STATE_LIVE] = "[MODULE_STATE_LIVE] Normal state", ++ [MODULE_STATE_COMING] = "[MODULE_STATE_COMING] Full formed, running module_init", ++ [MODULE_STATE_GOING] = "[MODULE_STATE_GOING] Going away", ++ [MODULE_STATE_UNFORMED] = "[MODULE_STATE_UNFORMED] Still setting it up", ++}; ++ ++static void callback_info(const char *callback, patch_object *obj) ++{ ++ if (obj->mod) ++ pr_info("%s: %s -> %s\n", callback, obj->mod->name, ++ module_state[obj->mod->state]); ++ else ++ pr_info("%s: vmlinux\n", callback); ++} ++ ++static int pre_patch_callback(patch_object *obj) ++{ ++ callback_info(__func__, obj); ++ return 0; ++} ++KPATCH_PRE_PATCH_CALLBACK(pre_patch_callback); ++ ++static void post_patch_callback(patch_object *obj) ++{ ++ callback_info(__func__, obj); ++} ++KPATCH_POST_PATCH_CALLBACK(post_patch_callback); ++ ++static void pre_unpatch_callback(patch_object *obj) ++{ ++ callback_info(__func__, obj); ++} ++KPATCH_PRE_UNPATCH_CALLBACK(pre_unpatch_callback); ++ ++static void post_unpatch_callback(patch_object *obj) ++{ ++ callback_info(__func__, obj); ++} ++KPATCH_POST_UNPATCH_CALLBACK(post_unpatch_callback); ++ + #define AIO_RING_MAGIC 0xa10a10a1 + #define AIO_RING_COMPAT_FEATURES 1 + #define AIO_RING_INCOMPAT_FEATURES 0 diff --git a/test/integration/linux-6.12.95/multiple.test b/test/integration/linux-6.12.95/multiple.test new file mode 100644 index 000000000..103776dfe --- /dev/null +++ b/test/integration/linux-6.12.95/multiple.test @@ -0,0 +1,8 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-or-later + +SCRIPTDIR="$(readlink -f $(dirname $(type -p $0)))" + +declare -a blacklist=() + +source ${SCRIPTDIR}/../common/multiple.template diff --git a/test/integration/linux-6.12.95/new-function.patch b/test/integration/linux-6.12.95/new-function.patch new file mode 100644 index 000000000..50bca4814 --- /dev/null +++ b/test/integration/linux-6.12.95/new-function.patch @@ -0,0 +1,25 @@ +diff -Nupr src.orig/drivers/tty/n_tty.c src/drivers/tty/n_tty.c +--- src.orig/drivers/tty/n_tty.c ++++ src/drivers/tty/n_tty.c +@@ -2343,7 +2343,7 @@ more_to_be_read: + * (note that the process_output*() functions take this lock themselves) + */ + +-static ssize_t n_tty_write(struct tty_struct *tty, struct file *file, ++static ssize_t noinline kpatch_n_tty_write(struct tty_struct *tty, struct file *file, + const u8 *buf, size_t nr) + { + const u8 *b = buf; +@@ -2428,6 +2428,12 @@ break_out: + return (b - buf) ? b - buf : retval; + } + ++static ssize_t __attribute__((optimize("-fno-optimize-sibling-calls"))) n_tty_write(struct tty_struct *tty, struct file *file, ++ const unsigned char *buf, size_t nr) ++{ ++ return kpatch_n_tty_write(tty, file, buf, nr); ++} ++ + /** + * n_tty_poll - poll method for N_TTY + * @tty: terminal device diff --git a/test/integration/linux-6.12.95/new-globals.patch b/test/integration/linux-6.12.95/new-globals.patch new file mode 100644 index 000000000..bed830f8c --- /dev/null +++ b/test/integration/linux-6.12.95/new-globals.patch @@ -0,0 +1,35 @@ +diff -Nupr src.orig/fs/proc/cmdline.c src/fs/proc/cmdline.c +--- src.orig/fs/proc/cmdline.c ++++ src/fs/proc/cmdline.c +@@ -22,3 +22,11 @@ static int __init proc_cmdline_init(void + return 0; + } + fs_initcall(proc_cmdline_init); ++ ++#include ++void kpatch_print_message(void); ++void kpatch_print_message(void) ++{ ++ if (!jiffies) ++ printk("hello there!\n"); ++} +diff -Nupr src.orig/fs/proc/meminfo.c src/fs/proc/meminfo.c +--- src.orig/fs/proc/meminfo.c ++++ src/fs/proc/meminfo.c +@@ -21,6 +21,8 @@ + #include + #include "internal.h" + ++void kpatch_print_message(void); ++ + void __attribute__((weak)) arch_report_meminfo(struct seq_file *m) + { + } +@@ -57,6 +59,7 @@ static int meminfo_proc_show(struct seq_ + sreclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B); + sunreclaim = global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B); + ++ kpatch_print_message(); + show_val_kb(m, "MemTotal: ", i.totalram); + show_val_kb(m, "MemFree: ", i.freeram); + show_val_kb(m, "MemAvailable: ", available); diff --git a/test/integration/linux-6.12.95/shadow-newpid-LOADED.test b/test/integration/linux-6.12.95/shadow-newpid-LOADED.test new file mode 100644 index 000000000..f3f123c04 --- /dev/null +++ b/test/integration/linux-6.12.95/shadow-newpid-LOADED.test @@ -0,0 +1,4 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-or-later + +grep -q newpid: /proc/$$/status diff --git a/test/integration/linux-6.12.95/shadow-newpid.patch b/test/integration/linux-6.12.95/shadow-newpid.patch new file mode 100644 index 000000000..00a04a877 --- /dev/null +++ b/test/integration/linux-6.12.95/shadow-newpid.patch @@ -0,0 +1,84 @@ +diff -Nupr src.orig/fs/proc/array.c src/fs/proc/array.c +--- src.orig/fs/proc/array.c ++++ src/fs/proc/array.c +@@ -145,7 +145,7 @@ static inline const char *get_task_state + return task_state_array[task_state_index(tsk)]; + } + +-static inline void task_state(struct seq_file *m, struct pid_namespace *ns, ++static __always_inline void task_state(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *p) + { + struct user_namespace *user_ns = seq_user_ns(m); +@@ -395,12 +395,19 @@ static inline void task_seccomp(struct s + seq_putc(m, '\n'); + } + ++#include + static inline void task_context_switch_counts(struct seq_file *m, + struct task_struct *p) + { ++ int *newpid; ++ + seq_put_decimal_ull(m, "voluntary_ctxt_switches:\t", p->nvcsw); + seq_put_decimal_ull(m, "\nnonvoluntary_ctxt_switches:\t", p->nivcsw); + seq_putc(m, '\n'); ++ ++ newpid = klp_shadow_get(p, 0); ++ if (newpid) ++ seq_printf(m, "newpid:\t%d\n", *newpid); + } + + static void task_cpus_allowed(struct seq_file *m, struct task_struct *task) +diff -Nupr src.orig/kernel/exit.c src/kernel/exit.c +--- src.orig/kernel/exit.c ++++ src/kernel/exit.c +@@ -889,6 +889,7 @@ static void synchronize_group_exit + coredump_task_exit(tsk, core_state); + } + ++#include + void __noreturn do_exit(long code) + { + struct task_struct *tsk = current; +@@ -964,6 +965,8 @@ void __noreturn do_exit(long code) + sched_autogroup_exit_task(tsk); + cgroup_exit(tsk); + ++ klp_shadow_free(tsk, 0, NULL); ++ + /* + * FIXME: do that only when needed, using sched_exit tracepoint + */ +diff -Nupr src.orig/kernel/fork.c src/kernel/fork.c +--- src.orig/kernel/fork.c ++++ src/kernel/fork.c +@@ -2561,6 +2561,7 @@ struct task_struct *create_io_thread(int + * + * args->exit_signal is expected to be checked for sanity by the caller. + */ ++#include + pid_t kernel_clone(struct kernel_clone_args *args) + { + u64 clone_flags = args->flags; +@@ -2569,6 +2570,8 @@ pid_t kernel_clone(struct kernel_clone_a + struct task_struct *p; + int trace = 0; + pid_t nr; ++ int *newpid; ++ static int ctr = 0; + + /* + * For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument +@@ -2608,6 +2611,11 @@ pid_t kernel_clone(struct kernel_clone_a + if (IS_ERR(p)) + return PTR_ERR(p); + ++ newpid = klp_shadow_get_or_alloc(p, 0, sizeof(*newpid), GFP_KERNEL, ++ NULL, NULL); ++ if (newpid) ++ *newpid = ctr++; ++ + /* + * Do this prior waking up the new thread - the thread pointer + * might get invalid after that point, if the thread exits quickly. diff --git a/test/integration/linux-6.12.95/skip-vmlinux-in-tree-module.patch.disabled b/test/integration/linux-6.12.95/skip-vmlinux-in-tree-module.patch.disabled new file mode 100644 index 000000000..c20899c82 --- /dev/null +++ b/test/integration/linux-6.12.95/skip-vmlinux-in-tree-module.patch.disabled @@ -0,0 +1,11 @@ +diff --git a/fs/xfs/xfs_stats.c b/fs/xfs/xfs_stats.c +--- a/fs/xfs/xfs_stats.c ++++ b/fs/xfs/xfs_stats.c +@@ -1,6 +1,6 @@ + // SPDX-License-Identifier: GPL-2.0 + /* +- * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. ++ * Copyright (c) 2000-2003, 2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ + #include "xfs.h" diff --git a/test/integration/linux-6.12.95/skip-vmlinux-in-tree-module.test b/test/integration/linux-6.12.95/skip-vmlinux-in-tree-module.test new file mode 100755 index 000000000..758c2230f --- /dev/null +++ b/test/integration/linux-6.12.95/skip-vmlinux-in-tree-module.test @@ -0,0 +1,90 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +arch=${KPATCH_TEST_ARCH:-$(uname -m)} +if [[ $arch != riscv64 ]]; then + echo "SKIP: --skip-vmlinux-final-link is RISC-V only" + exit 0 +fi + +if [[ " ${KPATCHBUILD_OPTS:-} " != *" --skip-vmlinux-final-link "* ]]; then + echo "SKIP: --skip-vmlinux-final-link is not enabled" + exit 0 +fi + +rootdir=$(readlink -f "$(dirname "$0")/../../..") +kpatch_build=${KPATCHBUILD:-$rootdir/kpatch-build/kpatch-build} +kpatch_build_opts=${KPATCHBUILD_OPTS:-} +patch=$(readlink -f "$(dirname "$0")/skip-vmlinux-in-tree-module.patch.disabled") +output=$(mktemp) +module_symvers_state=$(mktemp) + +read -r -a opts <<< "$kpatch_build_opts" +kernel_src= +for ((i = 0; i < ${#opts[@]}; i++)); do + case ${opts[i]} in + -s|--sourcedir) + ((i++)) + kernel_src=${opts[i]:-} + ;; + --sourcedir=*) + kernel_src=${opts[i]#*=} + ;; + esac +done + +if [[ -z "$kernel_src" ]]; then + echo "SKIP: integration run does not use an explicit source directory" + exit 0 +fi +kernel_src=$(readlink -f "$kernel_src") +module_symvers=$kernel_src/Module.symvers + +cleanup() +{ + rm -f "$output" "$module_symvers_state" +} +trap cleanup EXIT + +if [[ ! -f "$kernel_src/fs/xfs/xfs_stats.o" ]] || + ! grep -Eq '(^|[[:space:]])-DMODULE([[:space:]]|$)' "$kernel_src/fs/xfs/.xfs_stats.o.cmd"; then + echo "SKIP: CONFIG_XFS_FS is not built as an in-tree module" + exit 0 +fi + +if [[ -e "$module_symvers" ]]; then + sha256sum "$module_symvers" > "$module_symvers_state" +else + printf 'absent\n' > "$module_symvers_state" +fi + +# KPATCHBUILD_OPTS intentionally expands into multiple arguments. +# shellcheck disable=SC2086 +if "$kpatch_build" $kpatch_build_opts -n test-skip-vmlinux-module \ + "$patch" > "$output" 2>&1; then + echo "in-tree module patch unexpectedly succeeded" + exit 1 +fi + +if ! grep -q -- \ + '--skip-vmlinux-final-link supports vmlinux objects only; fs/xfs/xfs_stats.c is built as a module' \ + "$output"; then + cat "$output" + echo "missing vmlinux-only rejection" + exit 1 +fi + +if grep -q 'Building patched source' "$output"; then + cat "$output" + echo "in-tree module patch reached the patched build" + exit 1 +fi + +if grep -q '^absent$' "$module_symvers_state"; then + [[ ! -e "$module_symvers" ]] +else + sha256sum --check --status "$module_symvers_state" +fi diff --git a/test/integration/linux-6.12.95/special-static.patch b/test/integration/linux-6.12.95/special-static.patch new file mode 100644 index 000000000..a85838ae3 --- /dev/null +++ b/test/integration/linux-6.12.95/special-static.patch @@ -0,0 +1,22 @@ +diff -Nupr src.orig/kernel/fork.c src/kernel/fork.c +--- src.orig/kernel/fork.c ++++ src/kernel/fork.c +@@ -1645,10 +1645,18 @@ static void posix_cpu_timers_init_group( + posix_cputimers_group_init(pct, cpu_limit); + } + ++static void kpatch_foo(void) ++{ ++ if (!jiffies) ++ printk("kpatch copy signal\n"); ++} ++ + static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) + { + struct signal_struct *sig; + ++ kpatch_foo(); ++ + if (clone_flags & CLONE_THREAD) + return 0; + diff --git a/test/integration/linux-6.12.95/symvers-disagreement-FAIL.patch b/test/integration/linux-6.12.95/symvers-disagreement-FAIL.patch new file mode 100644 index 000000000..295bd4a8a --- /dev/null +++ b/test/integration/linux-6.12.95/symvers-disagreement-FAIL.patch @@ -0,0 +1,43 @@ +From 2d6b7bce089e52563bd9c67df62f48e90b48047d Mon Sep 17 00:00:00 2001 +From: Julien Thierry +Date: Wed, 6 May 2020 14:30:57 +0100 +Subject: [PATCH] Symbol version change + +This change causes: +1) Some exported symbols in drivers/base/core.c to see their CRCs + change. +2) Changes usb_get_dev() referencing a get_device() whose CRC has + changed, causing the symbol and the new CRC to be included in the + __version section of the final module. + +This makes the final module unloadable for the target kernel. + +See "Exported symbol versioning" of the patch author guide for more +detail. + +--- + +diff -Nupr src.orig/drivers/base/core.c src/drivers/base/core.c +--- src.orig/drivers/base/core.c ++++ src/drivers/base/core.c +@@ -37,6 +37,8 @@ + #include "physical_location.h" + #include "power/power.h" + ++#include ++ + /* Device links support. */ + static LIST_HEAD(deferred_sync); + static unsigned int defer_sync_state_count = 1; +diff -Nupr src.orig/drivers/usb/core/usb.c src/drivers/usb/core/usb.c +--- src.orig/drivers/usb/core/usb.c ++++ src/drivers/usb/core/usb.c +@@ -767,6 +767,8 @@ EXPORT_SYMBOL_GPL(usb_alloc_dev); + */ + struct usb_device *usb_get_dev(struct usb_device *dev) + { ++ barrier(); ++ + if (dev) + get_device(&dev->dev); + return dev; diff --git a/test/integration/linux-6.12.95/syscall-LOADED.test b/test/integration/linux-6.12.95/syscall-LOADED.test new file mode 100644 index 000000000..24d84d57b --- /dev/null +++ b/test/integration/linux-6.12.95/syscall-LOADED.test @@ -0,0 +1,4 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-or-later + +uname -s | grep -q kpatch diff --git a/test/integration/linux-6.12.95/syscall.patch b/test/integration/linux-6.12.95/syscall.patch new file mode 100644 index 000000000..221e72261 --- /dev/null +++ b/test/integration/linux-6.12.95/syscall.patch @@ -0,0 +1,20 @@ +diff -Nupr src.orig/kernel/sys.c src/kernel/sys.c +--- src.orig/kernel/sys.c ++++ src/kernel/sys.c +@@ -1348,13 +1348,15 @@ static int override_release(char __user + return ret; + } + +-SYSCALL_DEFINE1(newuname, struct new_utsname __user *, name) ++#include "kpatch-syscall.h" ++KPATCH_SYSCALL_DEFINE1(newuname, struct new_utsname __user *, name) + { + struct new_utsname tmp; + + down_read(&uts_sem); + memcpy(&tmp, utsname(), sizeof(tmp)); + up_read(&uts_sem); ++ strcat(tmp.sysname, ".kpatch"); + if (copy_to_user(name, &tmp, sizeof(tmp))) + return -EFAULT; + diff --git a/test/integration/linux-6.12.95/warn-detect-FAIL.patch b/test/integration/linux-6.12.95/warn-detect-FAIL.patch new file mode 100644 index 000000000..b442030ba --- /dev/null +++ b/test/integration/linux-6.12.95/warn-detect-FAIL.patch @@ -0,0 +1,9 @@ +diff -Nupr src.orig/arch/riscv/kernel/setup.c src/arch/riscv/kernel/setup.c +--- src.orig/arch/riscv/kernel/setup.c ++++ src/arch/riscv/kernel/setup.c +@@ -1,4 +1,5 @@ + // SPDX-License-Identifier: GPL-2.0-or-later ++ + /* + * Copyright (C) 2009 Sunplus Core Technology Co., Ltd. + * Chen Liqin diff --git a/test/test-functions.sh b/test/test-functions.sh index 96fa9a799..3e3f85538 100644 --- a/test/test-functions.sh +++ b/test/test-functions.sh @@ -1,7 +1,7 @@ FILE=$1 assert_num_funcs() { - local num_funcs=$(nm $FILE | grep -i " t " | wc -l) + local num_funcs=$(objdump -t "$FILE" | awk '$3 == "F" && $4 ~ /^\.text($|\.)/ { count++ } END { print count + 0 }') if [[ $num_funcs != $1 ]]; then echo "$FILE: assertion failed: file has $num_funcs funcs, expected $1" 1>&2 diff --git a/test/unit/Makefile b/test/unit/Makefile index 91efbde3c..d83b5df6f 100644 --- a/test/unit/Makefile +++ b/test/unit/Makefile @@ -1,4 +1,4 @@ -ARCHES ?= loongarch64 aarch64 ppc64le x86_64 +ARCHES ?= loongarch64 aarch64 ppc64le x86_64 riscv64 .PHONY: all clean submodule-check diff --git a/test/unit/objs b/test/unit/objs index d31eed75b..a482f758a 160000 --- a/test/unit/objs +++ b/test/unit/objs @@ -1 +1 @@ -Subproject commit d31eed75b76312986b65a65133b9a100efcc3903 +Subproject commit a482f758ab231426685ca8cbd1c0d1b557c61d08