diff options
Diffstat (limited to 'sys/arm64')
| -rw-r--r-- | sys/arm64/arm64/machdep.c | 2 | ||||
| -rw-r--r-- | sys/arm64/arm64/pmap.c | 96 | ||||
| -rw-r--r-- | sys/arm64/conf/GENERIC-MMCCAM-NODEBUG | 14 | ||||
| -rw-r--r-- | sys/arm64/conf/GENERIC-NODEBUG | 31 | ||||
| -rw-r--r-- | sys/arm64/conf/std.arm64 | 2 | ||||
| -rw-r--r-- | sys/arm64/include/cpu.h | 3 | ||||
| -rw-r--r-- | sys/arm64/include/pmap.h | 2 |
7 files changed, 95 insertions, 55 deletions
diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c index 627b02e82d34..47c701e8588c 100644 --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -858,7 +858,7 @@ initarm(struct arm64_bootparams *abp) cninit(); set_ttbr0(abp->kern_ttbr0); - cpu_tlb_flushID(); + pmap_s1_invalidate_all_kernel(); if (!valid) panic("Invalid bus configuration: %s", diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index 48b62442e68f..8a4395aa1c89 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -190,6 +190,8 @@ pt_entry_t __read_mostly pmap_gp_attr; #define PMAP_SAN_PTE_BITS (ATTR_AF | ATTR_S1_XN | pmap_sh_attr | \ ATTR_KERN_GP | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_AP(ATTR_S1_AP_RW)) +static bool __read_mostly pmap_multiple_tlbi = false; + struct pmap_large_md_page { struct rwlock pv_lock; struct md_page pv_page; @@ -1297,7 +1299,7 @@ pmap_bootstrap_dmap(vm_size_t kernlen) } } - cpu_tlb_flushID(); + pmap_s1_invalidate_all_kernel(); bs_state.dmap_valid = true; @@ -1399,7 +1401,7 @@ pmap_bootstrap(void) /* And the l3 tables for the early devmap */ pmap_bootstrap_l3(VM_MAX_KERNEL_ADDRESS - (PMAP_MAPDEV_EARLY_SIZE)); - cpu_tlb_flushID(); + pmap_s1_invalidate_all_kernel(); #define alloc_pages(var, np) \ (var) = bs_state.freemempos; \ @@ -1723,6 +1725,51 @@ CPU_FEAT(feat_hafdbs, "Hardware management of the Access flag and dirty state", pmap_dbm_check, pmap_dbm_has_errata, pmap_dbm_enable, CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU); +static cpu_feat_en +pmap_multiple_tlbi_check(const struct cpu_feat *feat __unused, u_int midr) +{ + /* + * Cortex-A55 erratum 2441007 (Cat B rare) + * Present in all revisions + */ + if (CPU_IMPL(midr) == CPU_IMPL_ARM && + CPU_PART(midr) == CPU_PART_CORTEX_A55) + return (FEAT_DEFAULT_DISABLE); + + /* + * Cortex-A76 erratum 1286807 (Cat B rare) + * Present in r0p0 - r3p0 + * Fixed in r3p1 + */ + if (midr_check_var_part_range(midr, CPU_IMPL_ARM, CPU_PART_CORTEX_A76, + 0, 0, 3, 0)) + return (FEAT_DEFAULT_DISABLE); + + /* + * Cortex-A510 erratum 2441009 (Cat B rare) + * Present in r0p0 - r1p1 + * Fixed in r1p2 + */ + if (midr_check_var_part_range(midr, CPU_IMPL_ARM, CPU_PART_CORTEX_A510, + 0, 0, 1, 1)) + return (FEAT_DEFAULT_DISABLE); + + return (FEAT_ALWAYS_DISABLE); +} + +static bool +pmap_multiple_tlbi_enable(const struct cpu_feat *feat __unused, + cpu_feat_errata errata_status, u_int *errata_list __unused, + u_int errata_count __unused) +{ + pmap_multiple_tlbi = true; + return (true); +} + +CPU_FEAT(errata_multi_tlbi, "Multiple TLBI errata", + pmap_multiple_tlbi_check, NULL, pmap_multiple_tlbi_enable, + CPU_FEAT_EARLY_BOOT | CPU_FEAT_PER_CPU); + /* * Initialize the pmap module. * @@ -1876,9 +1923,17 @@ pmap_s1_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only) r = TLBI_VA(va); if (pmap == kernel_pmap) { pmap_s1_invalidate_kernel(r, final_only); + if (pmap_multiple_tlbi) { + dsb(ish); + pmap_s1_invalidate_kernel(r, final_only); + } } else { r |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); pmap_s1_invalidate_user(r, final_only); + if (pmap_multiple_tlbi) { + dsb(ish); + pmap_s1_invalidate_user(r, final_only); + } } dsb(ish); isb(); @@ -1920,12 +1975,24 @@ pmap_s1_invalidate_strided(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, end = TLBI_VA(eva); for (r = start; r < end; r += TLBI_VA(stride)) pmap_s1_invalidate_kernel(r, final_only); + + if (pmap_multiple_tlbi) { + dsb(ish); + for (r = start; r < end; r += TLBI_VA(stride)) + pmap_s1_invalidate_kernel(r, final_only); + } } else { start = end = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); start |= TLBI_VA(sva); end |= TLBI_VA(eva); for (r = start; r < end; r += TLBI_VA(stride)) pmap_s1_invalidate_user(r, final_only); + + if (pmap_multiple_tlbi) { + dsb(ish); + for (r = start; r < end; r += TLBI_VA(stride)) + pmap_s1_invalidate_user(r, final_only); + } } dsb(ish); isb(); @@ -1961,6 +2028,19 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pmap_s2_invalidate_range(pmap, sva, eva, final_only); } +void +pmap_s1_invalidate_all_kernel(void) +{ + dsb(ishst); + __asm __volatile("tlbi vmalle1is"); + dsb(ish); + if (pmap_multiple_tlbi) { + __asm __volatile("tlbi vmalle1is"); + dsb(ish); + } + isb(); +} + /* * Invalidates all cached intermediate- and final-level TLB entries for the * given virtual address space. @@ -1975,9 +2055,17 @@ pmap_s1_invalidate_all(pmap_t pmap) dsb(ishst); if (pmap == kernel_pmap) { __asm __volatile("tlbi vmalle1is"); + if (pmap_multiple_tlbi) { + dsb(ish); + __asm __volatile("tlbi vmalle1is"); + } } else { r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); __asm __volatile("tlbi aside1is, %0" : : "r" (r)); + if (pmap_multiple_tlbi) { + dsb(ish); + __asm __volatile("tlbi aside1is, %0" : : "r" (r)); + } } dsb(ish); isb(); @@ -7965,7 +8053,7 @@ pmap_mapbios(vm_paddr_t pa, vm_size_t size) pa += L2_SIZE; } if ((old_l2e & ATTR_DESCR_VALID) != 0) - pmap_s1_invalidate_all(kernel_pmap); + pmap_s1_invalidate_all_kernel(); else { /* * Because the old entries were invalid and the new @@ -8056,7 +8144,7 @@ pmap_unmapbios(void *p, vm_size_t size) } } if (preinit_map) { - pmap_s1_invalidate_all(kernel_pmap); + pmap_s1_invalidate_all_kernel(); return; } diff --git a/sys/arm64/conf/GENERIC-MMCCAM-NODEBUG b/sys/arm64/conf/GENERIC-MMCCAM-NODEBUG deleted file mode 100644 index b2e865129012..000000000000 --- a/sys/arm64/conf/GENERIC-MMCCAM-NODEBUG +++ /dev/null @@ -1,14 +0,0 @@ -# -# GENERIC-MMCCAM-NODEBUG -# -# Custom kernel for arm64 plus MMCCAM as opposed to the prior MMC stack. It is -# present to keep it building in tree since it wouldn't work in LINT. This -# version without debugging features. -# - -#NO_UNIVERSE - -include GENERIC-MMCCAM -include "std.nodebug" - -ident GENERIC-MMCCAM-NODEBUG diff --git a/sys/arm64/conf/GENERIC-NODEBUG b/sys/arm64/conf/GENERIC-NODEBUG deleted file mode 100644 index 086942dfaab1..000000000000 --- a/sys/arm64/conf/GENERIC-NODEBUG +++ /dev/null @@ -1,31 +0,0 @@ -# -# GENERIC-NODEBUG -- WITNESS and INVARIANTS free kernel configuration file -# for FreeBSD/arm64 -# -# This configuration file removes several debugging options, including -# WITNESS and INVARIANTS checking, which are known to have significant -# performance impact on running systems. When benchmarking new features -# this kernel should be used instead of the standard GENERIC. -# This kernel configuration should never appear outside of the HEAD -# of the FreeBSD tree. -# -# For more information on this file, please read the config(5) manual page, -# and/or the handbook section on Kernel Configuration Files: -# -# https://docs.freebsd.org/en/books/handbook/kernelconfig/#kernelconfig-config -# -# The handbook is also available locally in /usr/share/doc/handbook -# if you've installed the doc distribution, otherwise always see the -# FreeBSD World Wide Web server (https://www.FreeBSD.org/) for the -# latest information. -# -# An exhaustive list of options and more detailed explanations of the -# device lines is also present in the ../../conf/NOTES and NOTES files. -# If you are in doubt as to the purpose or necessity of a line, check first -# in NOTES. -# - -include GENERIC -include "std.nodebug" - -ident GENERIC-NODEBUG diff --git a/sys/arm64/conf/std.arm64 b/sys/arm64/conf/std.arm64 index a0568466cfaf..58f3748e2700 100644 --- a/sys/arm64/conf/std.arm64 +++ b/sys/arm64/conf/std.arm64 @@ -74,8 +74,6 @@ options PERTHREAD_SSP # Per-thread SSP canary # Debugging support. Always need this: options KDB # Enable kernel debugger support. options KDB_TRACE # Print a stack trace for a panic. -# For full debugger support use (turn off in stable branch): -include "std.debug" # Kernel Sanitizers #options COVERAGE # Generic kernel coverage. Used by KCOV diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h index f07b67d18abf..07a783138f42 100644 --- a/sys/arm64/include/cpu.h +++ b/sys/arm64/include/cpu.h @@ -193,9 +193,6 @@ (((mask) & PCPU_GET(midr)) == \ ((mask) & CPU_ID_RAW((impl), (part), (var), (rev)))) -#define CPU_MATCH_RAW(mask, devid) \ - (((mask) & PCPU_GET(midr)) == ((mask) & (devid))) - #if !defined(__ASSEMBLER__) static inline bool midr_check_var_part_range(u_int midr, u_int impl, u_int part, u_int var_low, diff --git a/sys/arm64/include/pmap.h b/sys/arm64/include/pmap.h index 357c1a0d8232..406b6e2c5e0a 100644 --- a/sys/arm64/include/pmap.h +++ b/sys/arm64/include/pmap.h @@ -175,6 +175,8 @@ int pmap_fault(pmap_t, uint64_t, uint64_t); struct pcb *pmap_switch(struct thread *); +void pmap_s1_invalidate_all_kernel(void); + extern void (*pmap_clean_stage2_tlbi)(void); extern void (*pmap_stage2_invalidate_range)(uint64_t, vm_offset_t, vm_offset_t, bool); |