@@ -62,6 +62,13 @@ static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs,
6262 irqentry_exit_to_kernel_mode_after_preempt (regs , state );
6363}
6464
65+ static __always_inline void arm64_syscall_enter_from_user_mode (struct pt_regs * regs )
66+ {
67+ enter_from_user_mode (regs );
68+ mte_disable_tco_entry (current );
69+ sme_enter_from_user_mode ();
70+ }
71+
6572/*
6673 * Handle IRQ/context state management when entering from user mode.
6774 * Before this function is called it is not safe to call regular kernel code,
@@ -70,20 +77,30 @@ static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs,
7077static __always_inline void arm64_enter_from_user_mode (struct pt_regs * regs )
7178{
7279 enter_from_user_mode (regs );
80+ rseq_note_user_irq_entry ();
7381 mte_disable_tco_entry (current );
7482 sme_enter_from_user_mode ();
7583}
7684
85+ static __always_inline void arm64_syscall_exit_to_user_mode (struct pt_regs * regs )
86+ {
87+ local_irq_disable ();
88+ syscall_exit_to_user_mode_prepare (regs );
89+ local_daif_mask ();
90+ sme_exit_to_user_mode ();
91+ mte_check_tfsr_exit ();
92+ exit_to_user_mode ();
93+ }
94+
7795/*
7896 * Handle IRQ/context state management when exiting to user mode.
7997 * After this function returns it is not safe to call regular kernel code,
8098 * instrumentable code, or any code which may trigger an exception.
8199 */
82-
83100static __always_inline void arm64_exit_to_user_mode (struct pt_regs * regs )
84101{
85102 local_irq_disable ();
86- exit_to_user_mode_prepare_legacy (regs );
103+ irqentry_exit_to_user_mode_prepare (regs );
87104 local_daif_mask ();
88105 sme_exit_to_user_mode ();
89106 mte_check_tfsr_exit ();
@@ -92,7 +109,7 @@ static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs)
92109
93110asmlinkage void noinstr asm_exit_to_user_mode (struct pt_regs * regs )
94111{
95- arm64_exit_to_user_mode (regs );
112+ arm64_syscall_exit_to_user_mode (regs );
96113}
97114
98115/*
@@ -716,12 +733,12 @@ static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr)
716733
717734static void noinstr el0_svc (struct pt_regs * regs )
718735{
719- arm64_enter_from_user_mode (regs );
736+ arm64_syscall_enter_from_user_mode (regs );
720737 cortex_a76_erratum_1463225_svc_handler ();
721738 fpsimd_syscall_enter ();
722739 local_daif_restore (DAIF_PROCCTX );
723740 do_el0_svc (regs );
724- arm64_exit_to_user_mode (regs );
741+ arm64_syscall_exit_to_user_mode (regs );
725742 fpsimd_syscall_exit ();
726743}
727744
@@ -868,11 +885,11 @@ static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
868885
869886static void noinstr el0_svc_compat (struct pt_regs * regs )
870887{
871- arm64_enter_from_user_mode (regs );
888+ arm64_syscall_enter_from_user_mode (regs );
872889 cortex_a76_erratum_1463225_svc_handler ();
873890 local_daif_restore (DAIF_PROCCTX );
874891 do_el0_svc_compat (regs );
875- arm64_exit_to_user_mode (regs );
892+ arm64_syscall_exit_to_user_mode (regs );
876893}
877894
878895static void noinstr el0_bkpt32 (struct pt_regs * regs , unsigned long esr )
0 commit comments