Attachment #152311 for bug #196542

View | Details | Raw Unified | Return to bug 196542
Collapse All | Expand All




{
	int i;

	printf("Initialize the logical ID to APIC ID table.\n");
	for (i = 0; i < MAXCPU; i++) {
		cpu_apic_ids[i] = -1;
		cpu_ipi_pending[i] = 0;
	}

	printf("Install an inter-CPU IPI for TLB invalidation\n");
	if (pmap_pcid_enabled) {
		setidt(IPI_INVLTLB, IDTVEC(invltlb_pcid), SDT_SYSIGT,
		    SEL_KPL, 0);

	}
	setidt(IPI_INVLRNG, IDTVEC(invlrng), SDT_SYSIGT, SEL_KPL, 0);

	printf("Install an inter-CPU IPI for cache invalidation.\n");
	setidt(IPI_INVLCACHE, IDTVEC(invlcache), SDT_SYSIGT, SEL_KPL, 0);

	printf("Install an inter-CPU IPI for all-CPU rendezvous\n");
	setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), SDT_SYSIGT, SEL_KPL, 0);

	printf("Install generic inter-CPU IPI handler\n");
	setidt(IPI_BITMAP_VECTOR, IDTVEC(ipi_intr_bitmap_handler),
	       SDT_SYSIGT, SEL_KPL, 0);

	printf("Install an inter-CPU IPI for CPU stop/restart\n");
	setidt(IPI_STOP, IDTVEC(cpustop), SDT_SYSIGT, SEL_KPL, 0);

	printf("Install an inter-CPU IPI for CPU suspend/resume\n");
	setidt(IPI_SUSPEND, IDTVEC(cpususpend), SDT_SYSIGT, SEL_KPL, 0);

	printf("Set boot_cpu_id if needed.\n");
	if (boot_cpu_id == -1) {
		boot_cpu_id = PCPU_GET(apic_id);
		cpu_info[boot_cpu_id].cpu_bsp = 1;

		KASSERT(boot_cpu_id == PCPU_GET(apic_id),
		    ("BSP's APIC ID doesn't match boot_cpu_id"));

	printf("Probe logical/physical core configuration.\n");
	topo_probe();

	printf("assign_cpu_ids\n");
	assign_cpu_ids();

	printf("Start each Application Processor\n");
	start_all_aps();

	printf("set_interrupt_apic_ids\n");
	set_interrupt_apic_ids();
}



	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);

	printf("install the AP 1st level boot code\n");
	pmap_kenter(va, boot_address);
	pmap_invalidate_page(kernel_pmap, va);
	bcopy(mptramp_start, (void *)va, bootMP_size);

	printf("Locate the page tables, they'll be below the trampoline\n");
	pt4 = (u_int64_t *)(uintptr_t)(mptramp_pagetables + KERNBASE);
	pt3 = pt4 + (PAGE_SIZE) / sizeof(u_int64_t);
	pt2 = pt3 + (PAGE_SIZE) / sizeof(u_int64_t);

	printf("Create the initial 1GB replicated page tables\n");
	for (i = 0; i < 512; i++) {
		printf("Each slot of the level 4 pages points to the same level 3 page\n");
		pt4[i] = (u_int64_t)(uintptr_t)(mptramp_pagetables + PAGE_SIZE);
		pt4[i] |= PG_V | PG_RW | PG_U;

		printf("Each slot of the level 3 pages points to the same level 2 page\n");
		pt3[i] = (u_int64_t)(uintptr_t)(mptramp_pagetables + (2 * PAGE_SIZE));
		pt3[i] |= PG_V | PG_RW | PG_U;

		printf("The level 2 page slots are mapped with 2MB pages for 1GB.\n");
		pt2[i] = i * (2 * 1024 * 1024);
		pt2[i] |= PG_V | PG_RW | PG_PS | PG_U;
	}

	printf("save the current value of the warm-start vector\n");
	mpbioswarmvec = *((u_int32_t *) WARMBOOT_OFF);
	outb(CMOS_REG, BIOS_RESET);
	mpbiosreason = inb(CMOS_DATA);

	printf("setup a vector to our boot code\n");
	*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
	*((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4);
	outb(CMOS_REG, BIOS_RESET);
	outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */

	printf("start each AP\n");
	for (cpu = 1; cpu < mp_ncpus; cpu++) {
		apic_id = cpu_apic_ids[cpu];

		printf("allocate and set up an idle stack data page\n");
		bootstacks[cpu] = (void *)kmem_malloc(kernel_arena,
		    KSTACK_PAGES * PAGE_SIZE, M_WAITOK | M_ZERO);
		doublefault_stack = (char *)kmem_malloc(kernel_arena,

		bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8;
		bootAP = cpu;

		printf("attempt to start the Application Processor\n");
		if (!start_ap(apic_id)) {
			printf("restore the warmstart vector\n");
			*(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec;
			panic("AP #%d (PHY# %d) failed!", cpu, apic_id);
		}

		CPU_SET(cpu, &all_cpus);	/* record AP in CPU map */
	}

	printf("restore the warmstart vector\n");
	*(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec;

	outb(CMOS_REG, BIOS_RESET);
	outb(CMOS_DATA, mpbiosreason);

	printf("number of APs actually started: %d\n", mp_naps);
	return mp_naps;
}


	int vector, ms;
	int cpus;

	printf("calculate the vector\n");
	vector = (boot_address >> 12) & 0xff;

	printf("used as a watchpoint to signal AP startup\n");
	cpus = mp_naps;

	ipi_startup(apic_id, vector);

	printf("Wait up to 5 seconds for it to start.\n");
	for (ms = 0; ms < 5000; ms++) {
		printf("wait ms: %d\n", ms);
		if (mp_naps > cpus)
			return 1;	/* return SUCCESS */
		DELAY(1000);

	 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
	 * ignored.
	 */
	printf("ipi_startup: lapic_ipi_raw - init ipi\n");
	lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
	    APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id);
	printf("ipi_startup: lapic_ipi_wait\n");
	lapic_ipi_wait(10000);
	printf("ipi_startup: delay\n");
	DELAY(10000);		/* wait ~10mS */

	/*

	 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
	 * will run.
	 */
	printf("ipi_startup: lapic_ipi_raw - startup ipi\n");
	lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
	    APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
	    vector, apic_id);
	printf("ipi_startup: lapic_ipi_wait\n");
	lapic_ipi_wait(10000);
	printf("ipi_startup: delay\n");
	DELAY(200);		/* wait ~200uS */

	/*

	 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
	 * recognized after hardware RESET or INIT IPI.
	 */
	printf("ipi_startup: lapic_ipi_raw - startup ipi #2\n");
	lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
	    APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
	    vector, apic_id);
	printf("ipi_startup: lapic_ipi_wait\n");
	lapic_ipi_wait(10000);
	printf("ipi_startup: delay\n");
	DELAY(200);		/* wait ~200uS */
}


Return to bug 196542

Lines 522-534 Link Here

(-)mp_machdep.c.patched (-30 / +42 lines)
522	{	522	{
523	int i;	523	int i;
524		524
525	/* Initialize the logical ID to APIC ID table. */	525	printf("Initialize the logical ID to APIC ID table.\n");
526	for (i = 0; i < MAXCPU; i++) {	526	for (i = 0; i < MAXCPU; i++) {
527	cpu_apic_ids[i] = -1;	527	cpu_apic_ids[i] = -1;
528	cpu_ipi_pending[i] = 0;	528	cpu_ipi_pending[i] = 0;
529	}	529	}
530		530
531	/* Install an inter-CPU IPI for TLB invalidation */	531	printf("Install an inter-CPU IPI for TLB invalidation\n");
532	if (pmap_pcid_enabled) {	532	if (pmap_pcid_enabled) {
533	setidt(IPI_INVLTLB, IDTVEC(invltlb_pcid), SDT_SYSIGT,	533	setidt(IPI_INVLTLB, IDTVEC(invltlb_pcid), SDT_SYSIGT,
534	SEL_KPL, 0);	534	SEL_KPL, 0);
Lines 540-562 Link Here
540	}	540	}
541	setidt(IPI_INVLRNG, IDTVEC(invlrng), SDT_SYSIGT, SEL_KPL, 0);	541	setidt(IPI_INVLRNG, IDTVEC(invlrng), SDT_SYSIGT, SEL_KPL, 0);
542		542
543	/* Install an inter-CPU IPI for cache invalidation. */	543	printf("Install an inter-CPU IPI for cache invalidation.\n");
544	setidt(IPI_INVLCACHE, IDTVEC(invlcache), SDT_SYSIGT, SEL_KPL, 0);	544	setidt(IPI_INVLCACHE, IDTVEC(invlcache), SDT_SYSIGT, SEL_KPL, 0);
545		545
546	/* Install an inter-CPU IPI for all-CPU rendezvous */	546	printf("Install an inter-CPU IPI for all-CPU rendezvous\n");
547	setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), SDT_SYSIGT, SEL_KPL, 0);	547	setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), SDT_SYSIGT, SEL_KPL, 0);
548		548
549	/* Install generic inter-CPU IPI handler */	549	printf("Install generic inter-CPU IPI handler\n");
550	setidt(IPI_BITMAP_VECTOR, IDTVEC(ipi_intr_bitmap_handler),	550	setidt(IPI_BITMAP_VECTOR, IDTVEC(ipi_intr_bitmap_handler),
551	SDT_SYSIGT, SEL_KPL, 0);	551	SDT_SYSIGT, SEL_KPL, 0);
552		552
553	/* Install an inter-CPU IPI for CPU stop/restart */	553	printf("Install an inter-CPU IPI for CPU stop/restart\n");
554	setidt(IPI_STOP, IDTVEC(cpustop), SDT_SYSIGT, SEL_KPL, 0);	554	setidt(IPI_STOP, IDTVEC(cpustop), SDT_SYSIGT, SEL_KPL, 0);
555		555
556	/* Install an inter-CPU IPI for CPU suspend/resume */	556	printf("Install an inter-CPU IPI for CPU suspend/resume\n");
557	setidt(IPI_SUSPEND, IDTVEC(cpususpend), SDT_SYSIGT, SEL_KPL, 0);	557	setidt(IPI_SUSPEND, IDTVEC(cpususpend), SDT_SYSIGT, SEL_KPL, 0);
558		558
559	/* Set boot_cpu_id if needed. */	559	printf("Set boot_cpu_id if needed.\n");
560	if (boot_cpu_id == -1) {	560	if (boot_cpu_id == -1) {
561	boot_cpu_id = PCPU_GET(apic_id);	561	boot_cpu_id = PCPU_GET(apic_id);
562	cpu_info[boot_cpu_id].cpu_bsp = 1;	562	cpu_info[boot_cpu_id].cpu_bsp = 1;
Lines 564-577 Link Here
564	KASSERT(boot_cpu_id == PCPU_GET(apic_id),	564	KASSERT(boot_cpu_id == PCPU_GET(apic_id),
565	("BSP's APIC ID doesn't match boot_cpu_id"));	565	("BSP's APIC ID doesn't match boot_cpu_id"));
566		566
567	/* Probe logical/physical core configuration. */	567	printf("Probe logical/physical core configuration.\n");
568	topo_probe();	568	topo_probe();
569		569
		570	printf("assign_cpu_ids\n");
570	assign_cpu_ids();	571	assign_cpu_ids();
571		572
572	/* Start each Application Processor */	573	printf("Start each Application Processor\n");
573	start_all_aps();	574	start_all_aps();
574		575
		576	printf("set_interrupt_apic_ids\n");
575	set_interrupt_apic_ids();	577	set_interrupt_apic_ids();
576	}	578	}
577		579
Lines 919-965 Link Here
919		921
920	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);	922	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
921		923
922	/* install the AP 1st level boot code */	924	printf("install the AP 1st level boot code\n");
923	pmap_kenter(va, boot_address);	925	pmap_kenter(va, boot_address);
924	pmap_invalidate_page(kernel_pmap, va);	926	pmap_invalidate_page(kernel_pmap, va);
925	bcopy(mptramp_start, (void *)va, bootMP_size);	927	bcopy(mptramp_start, (void *)va, bootMP_size);
926		928
927	/* Locate the page tables, they'll be below the trampoline */	929	printf("Locate the page tables, they'll be below the trampoline\n");
928	pt4 = (u_int64_t *)(uintptr_t)(mptramp_pagetables + KERNBASE);	930	pt4 = (u_int64_t *)(uintptr_t)(mptramp_pagetables + KERNBASE);
929	pt3 = pt4 + (PAGE_SIZE) / sizeof(u_int64_t);	931	pt3 = pt4 + (PAGE_SIZE) / sizeof(u_int64_t);
930	pt2 = pt3 + (PAGE_SIZE) / sizeof(u_int64_t);	932	pt2 = pt3 + (PAGE_SIZE) / sizeof(u_int64_t);
931		933
932	/* Create the initial 1GB replicated page tables */	934	printf("Create the initial 1GB replicated page tables\n");
933	for (i = 0; i < 512; i++) {	935	for (i = 0; i < 512; i++) {
934	/* Each slot of the level 4 pages points to the same level 3 page */	936	printf("Each slot of the level 4 pages points to the same level 3 page\n");
935	pt4[i] = (u_int64_t)(uintptr_t)(mptramp_pagetables + PAGE_SIZE);	937	pt4[i] = (u_int64_t)(uintptr_t)(mptramp_pagetables + PAGE_SIZE);
936	pt4[i] \|= PG_V \| PG_RW \| PG_U;	938	pt4[i] \|= PG_V \| PG_RW \| PG_U;
937		939
938	/* Each slot of the level 3 pages points to the same level 2 page */	940	printf("Each slot of the level 3 pages points to the same level 2 page\n");
939	pt3[i] = (u_int64_t)(uintptr_t)(mptramp_pagetables + (2 * PAGE_SIZE));	941	pt3[i] = (u_int64_t)(uintptr_t)(mptramp_pagetables + (2 * PAGE_SIZE));
940	pt3[i] \|= PG_V \| PG_RW \| PG_U;	942	pt3[i] \|= PG_V \| PG_RW \| PG_U;
941		943
942	/* The level 2 page slots are mapped with 2MB pages for 1GB. */	944	printf("The level 2 page slots are mapped with 2MB pages for 1GB.\n");
943	pt2[i] = i * (2 * 1024 * 1024);	945	pt2[i] = i * (2 * 1024 * 1024);
944	pt2[i] \|= PG_V \| PG_RW \| PG_PS \| PG_U;	946	pt2[i] \|= PG_V \| PG_RW \| PG_PS \| PG_U;
945	}	947	}
946		948
947	/* save the current value of the warm-start vector */	949	printf("save the current value of the warm-start vector\n");
948	mpbioswarmvec = ((u_int32_t ) WARMBOOT_OFF);	950	mpbioswarmvec = ((u_int32_t ) WARMBOOT_OFF);
949	outb(CMOS_REG, BIOS_RESET);	951	outb(CMOS_REG, BIOS_RESET);
950	mpbiosreason = inb(CMOS_DATA);	952	mpbiosreason = inb(CMOS_DATA);
951		953
952	/* setup a vector to our boot code */	954	printf("setup a vector to our boot code\n");
953	((volatile u_short ) WARMBOOT_OFF) = WARMBOOT_TARGET;	955	((volatile u_short ) WARMBOOT_OFF) = WARMBOOT_TARGET;
954	((volatile u_short ) WARMBOOT_SEG) = (boot_address >> 4);	956	((volatile u_short ) WARMBOOT_SEG) = (boot_address >> 4);
955	outb(CMOS_REG, BIOS_RESET);	957	outb(CMOS_REG, BIOS_RESET);
956	outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */	958	outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */
957		959
958	/* start each AP */	960	printf("start each AP\n");
959	for (cpu = 1; cpu < mp_ncpus; cpu++) {	961	for (cpu = 1; cpu < mp_ncpus; cpu++) {
960	apic_id = cpu_apic_ids[cpu];	962	apic_id = cpu_apic_ids[cpu];
961		963
962	/* allocate and set up an idle stack data page */	964	printf("allocate and set up an idle stack data page\n");
963	bootstacks[cpu] = (void *)kmem_malloc(kernel_arena,	965	bootstacks[cpu] = (void *)kmem_malloc(kernel_arena,
964	KSTACK_PAGES * PAGE_SIZE, M_WAITOK \| M_ZERO);	966	KSTACK_PAGES * PAGE_SIZE, M_WAITOK \| M_ZERO);
965	doublefault_stack = (char *)kmem_malloc(kernel_arena,	967	doublefault_stack = (char *)kmem_malloc(kernel_arena,
Lines 972-980 Link Here
972	bootSTK = (char )bootstacks[cpu] + KSTACK_PAGES PAGE_SIZE - 8;	974	bootSTK = (char )bootstacks[cpu] + KSTACK_PAGES PAGE_SIZE - 8;
973	bootAP = cpu;	975	bootAP = cpu;
974		976
975	/* attempt to start the Application Processor */	977	printf("attempt to start the Application Processor\n");
976	if (!start_ap(apic_id)) {	978	if (!start_ap(apic_id)) {
977	/* restore the warmstart vector */	979	printf("restore the warmstart vector\n");
978	(u_int32_t ) WARMBOOT_OFF = mpbioswarmvec;	980	(u_int32_t ) WARMBOOT_OFF = mpbioswarmvec;
979	panic("AP #%d (PHY# %d) failed!", cpu, apic_id);	981	panic("AP #%d (PHY# %d) failed!", cpu, apic_id);
980	}	982	}
Lines 982-994 Link Here
982	CPU_SET(cpu, &all_cpus); /* record AP in CPU map */	984	CPU_SET(cpu, &all_cpus); /* record AP in CPU map */
983	}	985	}
984		986
985	/* restore the warmstart vector */	987	printf("restore the warmstart vector\n");
986	(u_int32_t ) WARMBOOT_OFF = mpbioswarmvec;	988	(u_int32_t ) WARMBOOT_OFF = mpbioswarmvec;
987		989
988	outb(CMOS_REG, BIOS_RESET);	990	outb(CMOS_REG, BIOS_RESET);
989	outb(CMOS_DATA, mpbiosreason);	991	outb(CMOS_DATA, mpbiosreason);
990		992
991	/* number of APs actually started */	993	printf("number of APs actually started: %d\n", mp_naps);
992	return mp_naps;	994	return mp_naps;
993	}	995	}
994		996
Lines 1006-1021 Link Here
1006	int vector, ms;	1008	int vector, ms;
1007	int cpus;	1009	int cpus;
1008		1010
1009	/* calculate the vector */	1011	printf("calculate the vector\n");
1010	vector = (boot_address >> 12) & 0xff;	1012	vector = (boot_address >> 12) & 0xff;
1011		1013
1012	/* used as a watchpoint to signal AP startup */	1014	printf("used as a watchpoint to signal AP startup\n");
1013	cpus = mp_naps;	1015	cpus = mp_naps;
1014		1016
1015	ipi_startup(apic_id, vector);	1017	ipi_startup(apic_id, vector);
1016		1018
1017	/* Wait up to 5 seconds for it to start. */	1019	printf("Wait up to 5 seconds for it to start.\n");
1018	for (ms = 0; ms < 5000; ms++) {	1020	for (ms = 0; ms < 5000; ms++) {
		1021	printf("wait ms: %d\n", ms);
1019	if (mp_naps > cpus)	1022	if (mp_naps > cpus)
1020	return 1; /* return SUCCESS */	1023	return 1; /* return SUCCESS */
1021	DELAY(1000);	1024	DELAY(1000);
Lines 1072-1080 Link Here
1072	* bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be	1075	* bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
1073	* ignored.	1076	* ignored.
1074	*/	1077	*/
		1078	printf("ipi_startup: lapic_ipi_raw - init ipi\n");
1075	lapic_ipi_raw(APIC_DEST_DESTFLD \| APIC_TRIGMOD_EDGE \|	1079	lapic_ipi_raw(APIC_DEST_DESTFLD \| APIC_TRIGMOD_EDGE \|
1076	APIC_LEVEL_ASSERT \| APIC_DESTMODE_PHY \| APIC_DELMODE_INIT, apic_id);	1080	APIC_LEVEL_ASSERT \| APIC_DESTMODE_PHY \| APIC_DELMODE_INIT, apic_id);
1077	lapic_ipi_wait(-1);	1081	printf("ipi_startup: lapic_ipi_wait\n");
		1082	lapic_ipi_wait(10000);
		1083	printf("ipi_startup: delay\n");
1078	DELAY(10000); /* wait ~10mS */	1084	DELAY(10000); /* wait ~10mS */
1079		1085
1080	/*	1086	/*
Lines 1085-1094 Link Here
1085	* run. OR the previous INIT IPI was ignored. and this STARTUP IPI	1091	* run. OR the previous INIT IPI was ignored. and this STARTUP IPI
1086	* will run.	1092	* will run.
1087	*/	1093	*/
		1094	printf("ipi_startup: lapic_ipi_raw - startup ipi\n");
1088	lapic_ipi_raw(APIC_DEST_DESTFLD \| APIC_TRIGMOD_EDGE \|	1095	lapic_ipi_raw(APIC_DEST_DESTFLD \| APIC_TRIGMOD_EDGE \|
1089	APIC_LEVEL_DEASSERT \| APIC_DESTMODE_PHY \| APIC_DELMODE_STARTUP \|	1096	APIC_LEVEL_DEASSERT \| APIC_DESTMODE_PHY \| APIC_DELMODE_STARTUP \|
1090	vector, apic_id);	1097	vector, apic_id);
1091	lapic_ipi_wait(-1);	1098	printf("ipi_startup: lapic_ipi_wait\n");
		1099	lapic_ipi_wait(10000);
		1100	printf("ipi_startup: delay\n");
1092	DELAY(200); /* wait ~200uS */	1101	DELAY(200); /* wait ~200uS */
1093		1102
1094	/*	1103	/*
Lines 1097-1106 Link Here
1097	* this STARTUP IPI will be ignored, as only ONE STARTUP IPI is	1106	* this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
1098	* recognized after hardware RESET or INIT IPI.	1107	* recognized after hardware RESET or INIT IPI.
1099	*/	1108	*/
		1109	printf("ipi_startup: lapic_ipi_raw - startup ipi #2\n");
1100	lapic_ipi_raw(APIC_DEST_DESTFLD \| APIC_TRIGMOD_EDGE \|	1110	lapic_ipi_raw(APIC_DEST_DESTFLD \| APIC_TRIGMOD_EDGE \|
1101	APIC_LEVEL_DEASSERT \| APIC_DESTMODE_PHY \| APIC_DELMODE_STARTUP \|	1111	APIC_LEVEL_DEASSERT \| APIC_DESTMODE_PHY \| APIC_DELMODE_STARTUP \|
1102	vector, apic_id);	1112	vector, apic_id);
1103	lapic_ipi_wait(-1);	1113	printf("ipi_startup: lapic_ipi_wait\n");
		1114	lapic_ipi_wait(10000);
		1115	printf("ipi_startup: delay\n");
1104	DELAY(200); /* wait ~200uS */	1116	DELAY(200); /* wait ~200uS */
1105	}	1117	}
1106		1118