OSDev.org

The Place to Start for Operating System Developers
It is currently Tue Jun 25, 2019 6:23 pm

All times are UTC - 6 hours




Post new topic Reply to topic  [ 2 posts ] 
Author Message
 Post subject: Starting additional processors (cores) at low level
PostPosted: Thu Dec 27, 2018 7:26 am 
Offline

Joined: Thu Dec 27, 2018 12:30 am
Posts: 2
I try to teach my self a little bit of low level programming. At the moment I try to understand how SMP and starting additional processors work.

I think I understand, that the primary processor sends some APIC interprocessor interrupts to start additional processors (or cores). The additional CPU starts in real mode and has to use some "trampoline" to switch to protected mode.

I think I also understand that GRUB2 runs only on the primary processor. So I decided (again for the sole purpose of teaching myself) to write a grub module that starts additional CPUs, executes some little code on it and stop them again. I have already written user GRUB modules (again, to teach myself), so I think I know how to write them.

So I looked how XEN's hvmloader starts additional (virtual) processors inside a VM, and tried to port this code to GRUB2. See the code below.

The problem is that I reveice a triple fault when trying to "jump to protected mode", using a selector in the GDT I created before (the statement "ljmpl $0x08,$1f" generates the triple fault). Practically the same code works in XEN's hvmloader.

I think, meanwhile I understand what every statement of the assembler code does, but as I still get the triple fault, I think I still miss something.

Can someone help me further?

Here is the code of my grub module:
Code:
#include <grub/dl.h>
#include <grub/mm.h>
#include <grub/err.h>
#include <grub/env.h>
#include <grub/misc.h>
#include <grub/file.h>
#include <grub/disk.h>
#include <grub/term.h>
#include <grub/loader.h>
#include <grub/command.h>
#include <grub/i18n.h>

GRUB_MOD_LICENSE ( "GPLv3+" ) ;

#define AP_BOOT_EIP 0x1000
#define wmb()     barrier()
#define barrier() asm volatile ( "" : : : "memory" )
#define LAPIC_BASE_ADDRESS  0xfee00000
#define APIC_ICR      0x300
#define     APIC_ICR_BUSY            0x01000
#define     SET_APIC_DEST_FIELD(x)   ((x)<<24)
#define LAPIC_ID(vcpu_id)   ((vcpu_id) * 2)
#define APIC_ICR2     0x310
#define     APIC_DM_INIT             0x00500
#define     APIC_DM_STARTUP          0x00600

extern char ap_boot_start[], ap_boot_end[];

static int ap_callin, ap_cpuid;

asm (
    "    .text                       \n"
    "    .code16                     \n"
    "ap_boot_start: .code16          \n"
    "    mov   %cs,%ax               \n"
    "    mov   %ax,%ds               \n"
    "    lgdt  gdt_desr-ap_boot_start\n"
    "    xor   %ax, %ax              \n"
    "    inc   %ax                   \n"
    "    lmsw  %ax                   \n"
    "    ljmpl $0x08,$1f             \n"
    "gdt_desr:                       \n"
    "    .word gdt_end - gdt - 1     \n"
    "    .long gdt                   \n"
    "ap_boot_end: .code32            \n"
    "1:  mov   $0x10,%eax            \n"
    "    mov   %eax,%ds              \n"
    "    mov   %eax,%es              \n"
    "    mov   %eax,%ss              \n"
    "    movl  $stack_top,%esp       \n"
    "    movl  %esp,%ebp             \n"
    "    call  ap_start              \n"
    "1:  hlt                         \n"
    "    jmp  1b                     \n"
    "                                \n"
    "    .align 8                    \n"
    "gdt:                            \n"
    "    .quad 0x0000000000000000    \n"
    "    .quad 0x00cf9a000000ffff    \n" /* 0x08: Flat code segment */
    "    .quad 0x00cf92000000ffff    \n" /* 0x10: Flat data segment */
    "gdt_end:                        \n"
    "                                \n"
    "    .bss                        \n"
    "    .align    8                 \n"
    "stack:                          \n"
    "    .skip    0x4000             \n"
    "stack_top:                      \n"
    "    .text                       \n"
    );

void ap_start(void); /* non-static avoids unused-function compiler warning */
/*static*/ void ap_start(void)
{
    grub_printf(" - CPU%d ... \n", ap_cpuid);
    /* cacheattr_init(); */
    grub_printf("done.\n");
    wmb();
    ap_callin = 1;
}

static inline void cpu_relax(void)
{
    asm volatile ( "rep ; nop" : : : "memory" );
}

grub_uint32_t lapic_read(grub_uint32_t reg)
{
    return *(volatile grub_uint32_t *)(LAPIC_BASE_ADDRESS + reg);
}

void lapic_write(grub_uint32_t reg, grub_uint32_t val)
{
    grub_printf ( "LAPIC write, reg = 0x%x, val = 0x%x\n" , reg , val ) ;
    *(volatile grub_uint32_t *)(LAPIC_BASE_ADDRESS + reg) = val;
}

static void lapic_wait_ready(void)
{
    while ( lapic_read(APIC_ICR) & APIC_ICR_BUSY )
        cpu_relax();
}

static void boot_cpu(unsigned int cpu)
{
    unsigned int icr2 = SET_APIC_DEST_FIELD(LAPIC_ID(cpu));

    /* Initialise shared variables. */
    ap_cpuid = cpu;
    ap_callin = 0;
    wmb();

    /* Wake up the secondary processor: INIT-SIPI-SIPI... */
    grub_printf ( "starting CPU %d\n" , cpu ) ;
    lapic_wait_ready();
    grub_printf ( "INIT\n" ) ;
    lapic_write(APIC_ICR2, icr2);
    lapic_write(APIC_ICR, APIC_DM_INIT);
    lapic_wait_ready();
    grub_printf ( "SIPI\n" ) ;
    lapic_write(APIC_ICR2, icr2);
    lapic_write(APIC_ICR, APIC_DM_STARTUP | (AP_BOOT_EIP >> 12));
    lapic_wait_ready();
    grub_printf ( "SIPI\n" ) ;
    lapic_write(APIC_ICR2, icr2);
    lapic_write(APIC_ICR, APIC_DM_STARTUP | (AP_BOOT_EIP >> 12));
    lapic_wait_ready();
    /*
     * Wait for the secondary processor to complete initialisation.
     * Do not touch shared resources meanwhile.
     */
    while ( !ap_callin )
        cpu_relax();

    /* Take the secondary processor offline. */
    lapic_write(APIC_ICR2, icr2);
    lapic_write(APIC_ICR, APIC_DM_INIT);
    lapic_wait_ready();   
}

void smp_initialise(unsigned int nr_cpus)
{
    unsigned int i;
    unsigned char *pointer ;

    grub_printf ( "startcode:\n" ) ;
    for ( pointer = ap_boot_start ; pointer < (unsigned char *) ap_boot_end ; pointer ++ )
    {
      grub_printf ( "%02x  " , *pointer ) ;
    }
    grub_printf ( "\n" ) ;

    grub_printf ( "copying %d bytes of startcode\n" , ap_boot_end - ap_boot_start ) ;
    grub_memcpy((void *)AP_BOOT_EIP, ap_boot_start, ap_boot_end - ap_boot_start);
    grub_printf ( "verifying that startcode has been written...\n" ) ;
    if ( grub_memcmp ( (void *) AP_BOOT_EIP , ap_boot_start , ap_boot_end - ap_boot_start ) != 0 )
    {
      grub_printf ( "FATAL ERROR: startcode was NOT properly written!\n" ) ;
      grub_exit () ;
    }

    grub_printf("Multiprocessor initialisation:\n");
    ap_start();
    for ( i = 1; i < nr_cpus; i++ )
        boot_cpu(i);
}

static grub_err_t
grub_smp ( struct grub_command *cmd __attribute__ ((unused)) ,
             int argc , char *argv[] )
{
  grub_printf ( "starting grub2 smp initialization module...\n" ) ;
  smp_initialise ( 4 ) ;
}

static grub_command_t cmd_smp ;

GRUB_MOD_INIT ( smp )
{
  cmd_smp = grub_register_command ( "smp" , grub_smp , 0 , N_ ( "Initialize smp" ) ) ;
}

GRUB_MOD_FINI ( smp )
{
  grub_unregister_command ( cmd_smp ) ;
}


Top
 Profile  
 
 Post subject: Re: Starting additional processors (cores) at low level
PostPosted: Thu Dec 27, 2018 9:19 am 
Offline

Joined: Thu Dec 27, 2018 12:30 am
Posts: 2
After reading a lot of other examples I found that in other code lgdtl is used rather than lgdt, so I replace my lgdt by ldgtl, and... in principle it works now. I get random crashes that I can not yet explain, but the secondary CPU start. So I can continue coding and debugging...


Top
 Profile  
 
Display posts from previous:  Sort by  
Post new topic Reply to topic  [ 2 posts ] 

All times are UTC - 6 hours


Who is online

Users browsing this forum: No registered users and 3 guests


You cannot post new topics in this forum
You cannot reply to topics in this forum
You cannot edit your posts in this forum
You cannot delete your posts in this forum
You cannot post attachments in this forum

Search for:
Jump to:  
Powered by phpBB © 2000, 2002, 2005, 2007 phpBB Group