risc-v: Accelerate schedule by percpu

Fetch TCB info in percpu scratch register for better performance.

Signed-off-by: Huang Qi <[email protected]>

arch/risc-v/include/arch.h

@@ -56,6 +56,21 @@
 #  define ARCH_SPGTS          (ARCH_PGT_MAX_LEVELS - 1)
 #endif
 
+/* Can be used by assembly code to access the structure, example:
+ *
+ * Get percpu structure:
+ * 1: csrr    a0, CSR_SCRATCH
+ *
+ * Get hartid:
+ * 2: REGLOAD a0, RISCV_PERCPU_HARTID(a0)
+ */
+
+#define RISCV_PERCPU_TCB        (0 * INT_REG_SIZE)
+#define RISCV_PERCPU_HARTID     (1 * INT_REG_SIZE)
+#define RISCV_PERCPU_IRQSTACK   (2 * INT_REG_SIZE)
+#define RISCV_PERCPU_USP        (3 * INT_REG_SIZE)
+#define RISCV_PERCPU_KSP        (4 * INT_REG_SIZE)
+
 /****************************************************************************
  * Public Types
  ****************************************************************************/
@@ -108,6 +123,32 @@ typedef struct arch_addrenv_s arch_addrenv_t;
 #endif /* __ASSEMBLY__ */
 #endif /* CONFIG_ARCH_ADDRENV */
 
+#ifndef __ASSEMBLY__
+
+/* Per CPU save area. Access to this structure can be gained via the scratch
+ * ([m/s]scratch) register. Prior to this, every CPU that
+ * wishes to access this information must call up_percpu_add_hart() which
+ * will set up [m/s]scratch to point to the CPUs own area
+ * Note: All theses functions are architecture specific for RISC-V.
+ */
+
+union arch_percpu_s
+{
+  union arch_percpu_s *next;      /* For sl list linkage */
+  struct
+  {
+    struct tcb_s       *tcb;       /* Current thread TCB */
+    uintreg_t           hartid;    /* Hart ID */
+    uintreg_t           irq_stack; /* Interrupt stack */
+    uintreg_t           usp;       /* Area to store user sp */
+    uintreg_t           ksp;       /* Area to load kernel sp */
+  };
+};
+
+typedef union arch_percpu_s arch_percpu_t;
+
+#endif /* __ASSEMBLY__ */
+
 /****************************************************************************
  * Public Function Prototypes
  ****************************************************************************/
@@ -120,6 +161,84 @@ extern "C"
 #define EXTERN extern
 #endif
 
+#ifndef __ASSEMBLY__
+
+/****************************************************************************
+ * Name: up_percpu_add_hart
+ *
+ * Description:
+ *   Get add a hart to the per CPU area
+ *
+ * Input Parameters:
+ *   hartid - Hart number
+ *
+ ****************************************************************************/
+
+void up_percpu_add_hart(uintptr_t hartid);
+
+/****************************************************************************
+ * Name: up_percpu_get_hartid
+ *
+ * Description:
+ *   Get the hart's own hartid by reading it from the per CPU area. This is
+ *   safe to use from lower privilege modes than M-mode.
+ *
+ * Returned Value:
+ *   Hart id
+ *
+ ****************************************************************************/
+
+uintptr_t up_percpu_get_hartid(void);
+
+/****************************************************************************
+ * Name: up_percpu_get_irqstack
+ *
+ * Description:
+ *   Get the hart's own IRQ stack by reading it from the per CPU area.
+ *
+ * Returned Value:
+ *   IRQ stack, or 0 if no IRQ stack is assigned
+ *
+ ****************************************************************************/
+
+uintptr_t up_percpu_get_irqstack(void);
+
+/****************************************************************************
+ * Name: up_percpu_set_kstack
+ *
+ * Description:
+ *   Set the current kernel stack, so it can be taken quickly into use when
+ *   a trap is taken.
+ *
+ * Input Parameters:
+ *   ksp - Pointer to the kernel stack
+ *
+ * Returned Value:
+ *   None
+ *
+ ****************************************************************************/
+
+void up_percpu_set_kstack(uintptr_t ksp);
+
+/****************************************************************************
+ * Name: up_percpu_set_thread
+ *
+ * Description:
+ *   Set the current thread (tcb), so it can be found quickly when a trap is
+ *   taken.
+ *
+ * Input Parameters:
+ *   tcb - Pointer to the current thread's tcb
+ *
+ * Returned Value:
+ *   None
+ *
+ ****************************************************************************/
+
+void up_percpu_set_thread(struct tcb_s *tcb);
+
+#endif
+
 #undef EXTERN
 #ifdef __cplusplus
 }

arch/risc-v/include/irq.h

@@ -37,6 +37,7 @@
 
 #include <sys/types.h>
 
+#include <arch/arch.h>
 #include <arch/csr.h>
 #include <arch/chip/irq.h>
 #include <arch/mode.h>
@@ -703,6 +704,15 @@ irqstate_t up_irq_enable(void);
 int up_cpu_index(void) noinstrument_function;
 #endif /* CONFIG_ARCH_HAVE_MULTICPU */
 
+/****************************************************************************
+ * Schedule acceleration macros
+ ****************************************************************************/
+
+#ifdef CONFIG_RISCV_PERCPU_SCRATCH
+#define up_this_task() (((arch_percpu_t *)READ_CSR(CSR_SCRATCH))->tcb)
+#define up_update_task(t) up_percpu_set_thread(t)
+#endif
+
 /****************************************************************************
  * Name: up_this_cpu
  *

arch/risc-v/src/bl808/bl808_start.c

@@ -275,7 +275,7 @@ void bl808_start(int mhartid)
 
       /* Initialize the per CPU areas */
 
-      riscv_percpu_add_hart(mhartid);
+      up_percpu_add_hart(mhartid);
     }
 
   /* Disable MMU */

arch/risc-v/src/bl808/chip.h

@@ -37,7 +37,6 @@
 #include "hardware/bl808_plic.h"
 
 #include "riscv_internal.h"
-#include "riscv_percpu.h"
 
 /****************************************************************************
  * Macro Definitions

arch/risc-v/src/common/riscv_cpustart.c

@@ -37,11 +37,12 @@
 #include <nuttx/spinlock.h>
 #include <nuttx/sched_note.h>
 
+#include <arch/irq.h>
+
 #include "sched/sched.h"
 #include "init/init.h"
 #include "riscv_internal.h"
 #include "riscv_ipi.h"
-#include "riscv_percpu.h"
 
 #ifdef CONFIG_BUILD_KERNEL
 #  include "riscv_mmu.h"
@@ -69,6 +70,8 @@
 
 void riscv_cpu_boot(int cpu)
 {
+  struct tcb_s *tcb;
+
   /* Clear IPI for CPU(cpu) */
 
   riscv_ipi_clear(cpu);
@@ -88,7 +91,7 @@ void riscv_cpu_boot(int cpu)
 #ifdef CONFIG_RISCV_PERCPU_SCRATCH
   /* Initialize the per CPU areas */
 
-  riscv_percpu_add_hart(riscv_cpuid_to_hartid(cpu));
+  up_percpu_add_hart(riscv_cpuid_to_hartid(cpu));
 #endif
 
 #ifdef CONFIG_BUILD_KERNEL
@@ -100,8 +103,9 @@ void riscv_cpu_boot(int cpu)
 
   _info("CPU%d Started\n", this_cpu());
 
+  tcb = current_task(this_cpu());
+
 #ifdef CONFIG_STACK_COLORATION
-  struct tcb_s *tcb = this_task();
 
   /* If stack debug is enabled, then fill the stack with a
    * recognizable value that we can use later to test for high
@@ -111,6 +115,8 @@ void riscv_cpu_boot(int cpu)
   riscv_stack_color(tcb->stack_alloc_ptr, 0);
 #endif
 
+  up_update_task(tcb);
+
   /* TODO: Setup FPU */
 
   /* Clear machine software interrupt for CPU(cpu) */

arch/risc-v/src/common/riscv_exception_common.S

@@ -38,8 +38,6 @@
 
 #include "chip.h"
 
-#include "riscv_percpu.h"
-
 #include "riscv_macros.S"
 
 /*

arch/risc-v/src/common/riscv_macros.S

@@ -35,7 +35,6 @@
 #include <sys/types.h>
 
 #include "riscv_internal.h"
-#include "riscv_percpu.h"
 
 /****************************************************************************
  * Pre-processor Definitions