/*	$NetBSD: locore.S,v 1.11 2019/03/19 19:15:57 maxv Exp $	*/

/*
 * Copyright (c) 1998, 2000, 2007, 2008, 2016, 2017 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Charles M. Hannum and by Maxime Villard.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#define _LOCORE

/* Override user-land alignment before including asm.h */
#define	ALIGN_DATA	.align	8
#define ALIGN_TEXT	.align 16,0x90
#define _ALIGN_TEXT	ALIGN_TEXT

#include <machine/asm.h>
#include <machine/param.h>
#include <machine/pte.h>
#include <machine/psl.h>
#include <machine/segments.h>
#include <machine/specialreg.h>
#include <machine/trap.h>

#define _KERNEL
#include <machine/bootinfo.h>
#undef _KERNEL

#include "pdir.h"
#include "redef.h"

/* 32bit version of PTE_NX */
#define PTE_NX32	0x80000000

#define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1)
#define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES

#define PROC0_PML4_OFF	0
#define PROC0_STK_OFF	(PROC0_PML4_OFF + 1 * PAGE_SIZE)
#define PROC0_PTP3_OFF	(PROC0_STK_OFF + UPAGES * PAGE_SIZE)
#define PROC0_PTP2_OFF	(PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE)
#define PROC0_PTP1_OFF	(PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE)
#define TABLESIZE \
  ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \
    * PAGE_SIZE)

/*
 * fillkpt - Fill in a kernel page table
 *	eax = pte (page frame | control | status)
 *	ebx = page table address
 *	ecx = number of pages to map
 *
 * Each entry is 8 (PDE_SIZE) bytes long: we must set the 4 upper bytes to 0.
 */
#define fillkpt	\
	cmpl	$0,%ecx			;	/* zero-sized? */	\
	je 	2f			; \
1:	movl	$0,(PDE_SIZE-4)(%ebx)	;	/* upper 32 bits: 0 */	\
	movl	%eax,(%ebx)		;	/* store phys addr */	\
	addl	$PDE_SIZE,%ebx		;	/* next PTE/PDE */	\
	addl	$PAGE_SIZE,%eax		;	/* next phys page */	\
	loop	1b			; \
2:					;

/*
 * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit.
 */
#define fillkpt_nox \
	cmpl	$0,%ecx			;	/* zero-sized? */	\
	je 	2f			; \
	pushl	%ebp			; \
	movl	_C_LABEL(nox_flag),%ebp	; \
1:	movl	%ebp,(PDE_SIZE-4)(%ebx)	;	/* upper 32 bits: NX */ \
	movl	%eax,(%ebx)		;	/* store phys addr */	\
	addl	$PDE_SIZE,%ebx		;	/* next PTE/PDE */	\
	addl	$PAGE_SIZE,%eax		;	/* next phys page */	\
	loop	1b			; \
	popl	%ebp			; \
2:					;

/*
 * fillkpt_blank - Fill in a kernel page table with blank entries
 *	ebx = page table address
 *	ecx = number of pages to map
 */
#define fillkpt_blank	\
	cmpl	$0,%ecx			;	/* zero-sized? */	\
	je 	2f			; \
1:	movl	$0,(PDE_SIZE-4)(%ebx)	;	/* upper 32 bits: 0 */	\
	movl	$0,(%ebx)		;	/* lower 32 bits: 0 */	\
	addl	$PDE_SIZE,%ebx		;	/* next PTE/PDE */	\
	loop	1b			; \
2:					;

/*
 * Initialization
 */
	.data

	.globl	_C_LABEL(tablesize)
	.globl	_C_LABEL(nox_flag)
	.globl	_C_LABEL(cpuid_level)
	.globl	_C_LABEL(esym)
	.globl	_C_LABEL(eblob)
	.globl	_C_LABEL(atdevbase)
	.globl	_C_LABEL(PDPpaddr)
	.globl	_C_LABEL(boothowto)
	.globl	_C_LABEL(bootinfo)
	.globl	_C_LABEL(biosbasemem)
	.globl	_C_LABEL(biosextmem)
	.globl	_C_LABEL(stkpa)
	.globl	_C_LABEL(stkva)
	.globl	_C_LABEL(kernpa_start)
	.globl	_C_LABEL(kernpa_end)

	.type	_C_LABEL(tablesize), @object
_C_LABEL(tablesize):	.long	TABLESIZE
END(tablesize)
	.type	_C_LABEL(nox_flag), @object
LABEL(nox_flag)		.long	0	/* 32bit NOX flag, set if supported */
END(nox_flag)
	.type	_C_LABEL(cpuid_level), @object
LABEL(cpuid_level)	.long	-1	/* max. level accepted by cpuid instr */
END(cpuid_level)
	.type	_C_LABEL(esym), @object
LABEL(esym)		.quad	0	/* ptr to end of syms */
END(esym)
	.type	_C_LABEL(eblob), @object
LABEL(eblob)		.quad	0	/* ptr to end of modules */
END(eblob)
	.type	_C_LABEL(atdevbase), @object
LABEL(atdevbase)	.quad	0	/* location of start of iomem in virt */
END(atdevbase)
	.type	_C_LABEL(PDPpaddr), @object
LABEL(PDPpaddr)		.quad	0	/* paddr of PTD, for libkvm */
END(PDPpaddr)
	.type	_C_LABEL(biosbasemem), @object
LABEL(biosbasemem)	.long	0	/* base memory reported by BIOS */
END(biosbasemem)
	.type	_C_LABEL(biosextmem), @object
LABEL(biosextmem)	.long	0	/* extended memory reported by BIOS */
END(biosextmem)
	.type	_C_LABEL(stkpa), @object
LABEL(stkpa)		.quad	0
END(stkpa)
	.type	_C_LABEL(stkva), @object
LABEL(stkva)		.quad	0
END(stkva)
	.type	_C_LABEL(kernpa_start), @object
LABEL(kernpa_start)	.quad	0
END(kernpa_start)
	.type	_C_LABEL(kernpa_end), @object
LABEL(kernpa_end)	.quad	0
END(kernpa_end)

	.globl	gdt64_lo
	.globl	gdt64_start

#define GDT64_LIMIT gdt64_end-gdt64_start-1
/* Temporary gdt64, with base address in low memory */
	.type	_C_LABEL(gdt64_lo), @object
LABEL(gdt64_lo)
	.word	GDT64_LIMIT
	.quad	gdt64_start
END(gdt64_lo)
.align 64
#undef GDT64_LIMIT

	.type	_C_LABEL(gdt64_start), @object
LABEL(gdt64_start)
	.quad 0x0000000000000000	/* always empty */
	.quad 0x00af9a000000ffff	/* kernel CS */
	.quad 0x00cf92000000ffff	/* kernel DS */
END(gdt64_start)
gdt64_end:

	.type	_C_LABEL(farjmp64), @object
_C_LABEL(farjmp64):
	.long	longmode
	.word	GSEL(GCODE_SEL, SEL_KPL)
END(farjmp64)

	/* Space for the temporary stack */
	.size	tmpstk, tmpstk - .
	.space	512
tmpstk:

	.text

ENTRY(start)
	.code32

	/* Warm boot */
	movw	$0x1234,0x472

	/*
	 * Load parameters from the stack (32 bits):
	 *     boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem
	 * We are not interested in 'bootdev'.
	 */

	/* Load 'boothowto' */
	movl	4(%esp),%eax
	movl	%eax,_C_LABEL(boothowto)

	/* Load 'bootinfo' */
	movl	12(%esp),%eax
	testl	%eax,%eax		/* bootinfo = NULL? */
	jz	.Lbootinfo_finished

	movl	(%eax),%ebx		/* number of entries */
	movl	$_C_LABEL(bootinfo),%ebp
	movl	%ebp,%edx
	addl	$BOOTINFO_MAXSIZE,%ebp
	movl	%ebx,(%edx)
	addl	$4,%edx

.Lbootinfo_entryloop:
	testl	%ebx,%ebx		/* no remaining entries? */
	jz	.Lbootinfo_finished

	addl	$4,%eax
	movl	(%eax),%ecx		/* address of entry */
	pushl	%edi
	pushl	%esi
	pushl	%eax

	movl	(%ecx),%eax		/* btinfo_common::len (size of entry) */
	movl	%edx,%edi
	addl	(%ecx),%edx		/* update dest pointer */
	cmpl	%ebp,%edx		/* beyond bootinfo+BOOTINFO_MAXSIZE? */
	jg	.Lbootinfo_overflow

	movl	%ecx,%esi
	movl	%eax,%ecx

	/* If any modules were loaded, record where they end. */
	cmpl	$BTINFO_MODULELIST,4(%esi) /* btinfo_common::type */
	jne	0f
	pushl	12(%esi)		/* btinfo_modulelist::endpa */
	popl	_C_LABEL(eblob)
0:

	/* Record the information about the kernel. */
	cmpl	$BTINFO_PREKERN,4(%esi) /* btinfo_common::type */
	jne	0f
	pushl	8(%esi)		/* btinfo_prekern::kernpa_start */
	popl	_C_LABEL(kernpa_start)
	pushl	12(%esi)	/* btinfo_prekern::kernpa_end */
	popl	_C_LABEL(kernpa_end)
0:

	rep
	movsb				/* copy esi -> edi */
	popl	%eax
	popl	%esi
	popl	%edi
	subl	$1,%ebx			/* decrement the # of entries */
	jmp	.Lbootinfo_entryloop

.Lbootinfo_overflow:
	/*
	 * Cleanup for overflow case. Pop the registers, and correct the number
	 * of entries.
	 */
	popl	%eax
	popl	%esi
	popl	%edi
	movl	$_C_LABEL(bootinfo),%ebp
	movl	%ebp,%edx
	subl	%ebx,(%edx)		/* correct the number of entries */

.Lbootinfo_finished:
	/* Load 'esym' */
	movl	16(%esp),%eax
	movl	$_C_LABEL(esym),%ebp
	movl	%eax,(%ebp)

	/* Load 'biosextmem' */
	movl	20(%esp),%eax
	movl	$_C_LABEL(biosextmem),%ebp
	movl	%eax,(%ebp)

	/* Load 'biosbasemem' */
	movl	24(%esp),%eax
	movl	$_C_LABEL(biosbasemem),%ebp
	movl	%eax,(%ebp)

	/*
	 * Done with the parameters!
	 */

	/* First, reset the PSL. */
	pushl	$PSL_MBO
	popfl

	/* Switch to new stack now. */
	movl	$_C_LABEL(tmpstk),%esp

	xorl	%eax,%eax
	cpuid
	movl	%eax,_C_LABEL(cpuid_level)

	/*
	 * Retrieve the NX/XD flag. We use the 32bit version of PTE_NX.
	 */
	movl	$0x80000001,%eax
	cpuid
	andl	$CPUID_NOX,%edx
	jz	.Lno_NOX
	movl	$PTE_NX32,_C_LABEL(nox_flag)
.Lno_NOX:

/*
 * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will
 * be referred to as: L4 -> L3 -> L2 -> L1.
 *
 * Physical address space:
 * +---------------+----------+--------------+--------+---------------------+-
 * | PREKERN IMAGE |**UNUSED**| KERNEL IMAGE | [SYMS] | [PRELOADED MODULES] |
 * +---------------+----------+--------------+--------+---------------------+-
 *                (1)                       (2)      (3)                   (4)
 * ------------------+
 *  BOOTSTRAP TABLES |
 * ------------------+
 *                  (5)
 *
 * The virtual address space is the same, since it is identity-mapped (va = pa).
 * However, the KERNEL IMAGE is mapped as read-only: the prekern reads it, but
 * won't write to it. (Needed when relocating the kernel.)
 *
 * PROC0 STK is obviously not linked as a page level. It just happens to be
 * caught between L4 and L3.
 *
 * (PROC0 STK + L4 + L3 + L2 + L1) is later referred to as BOOTSTRAP TABLES.
 *
 * Important note: the prekern segments are properly 4k-aligned
 * (see prekern.ldscript), so there's no need to enforce alignment.
 */

	/* Find end of the prekern image; brings us on (1). */
	movl	$_C_LABEL(__prekern_end),%edi

	/* Find end of the kernel image; brings us on (2). */
	movl	_C_LABEL(kernpa_end),%eax
	testl	%eax,%eax
	jz	1f
	movl	%eax,%edi
1:

	/* Find end of the kernel symbols; brings us on (3). */
	movl	_C_LABEL(esym),%eax
	testl	%eax,%eax
	jz	1f
	movl	%eax,%edi
1:

	/* Find end of the kernel preloaded modules; brings us on (4). */
	movl	_C_LABEL(eblob),%eax
	testl	%eax,%eax
	jz	1f
	movl	%eax,%edi
1:

	/* We are on (3). Align up for BOOTSTRAP TABLES. */
	movl	%edi,%esi
	addl	$PGOFSET,%esi
	andl	$~PGOFSET,%esi

	/* We are on the BOOTSTRAP TABLES. Save L4's physical address. */
	movl	$_C_LABEL(PDPpaddr),%ebp
	movl	%esi,(%ebp)
	movl	$0,4(%ebp)

	/* Now, zero out the BOOTSTRAP TABLES (before filling them in). */
	movl	%esi,%edi
	xorl	%eax,%eax
	cld
	movl	$TABLESIZE,%ecx
	shrl	$2,%ecx
	rep
	stosl				/* copy eax -> edi */

/*
 * Build the page tables and levels. We go from L1 to L4, and link the levels
 * together.
 */
	/*
	 * Build L1.
	 */
	leal	(PROC0_PTP1_OFF)(%esi),%ebx

	/* Skip the area below the prekern text. */
	movl	$(PREKERNTEXTOFF - PREKERNBASE),%ecx
	shrl	$PGSHIFT,%ecx
	fillkpt_blank

	/* Map the prekern text RX. */
	movl	$(PREKERNTEXTOFF - PREKERNBASE),%eax	/* start of TEXT */
	movl	$_C_LABEL(__rodata_start),%ecx
	subl	%eax,%ecx
	shrl	$PGSHIFT,%ecx
	orl	$(PTE_P),%eax
	fillkpt

	/* Map the prekern rodata R. */
	movl	$_C_LABEL(__rodata_start),%eax
	movl	$_C_LABEL(__data_start),%ecx
	subl	%eax,%ecx
	shrl	$PGSHIFT,%ecx
	orl	$(PTE_P),%eax
	fillkpt_nox

	/* Map the prekern data+bss RW. */
	movl	$_C_LABEL(__data_start),%eax
	movl	$_C_LABEL(__prekern_end),%ecx
	subl	%eax,%ecx
	shrl	$PGSHIFT,%ecx
	orl	$(PTE_P|PTE_W),%eax
	fillkpt_nox

	/* Map a RO view of the kernel. */
	movl	$_C_LABEL(__prekern_end),%eax
	movl	%esi,%ecx		/* start of BOOTSTRAP TABLES */
	subl	%eax,%ecx
	shrl	$PGSHIFT,%ecx
	orl	$(PTE_P),%eax
	fillkpt_nox

	/* Map the BOOTSTRAP TABLES RW. */
	movl	%esi,%eax		/* start of BOOTSTRAP TABLES */
	movl	$TABLESIZE,%ecx		/* length of BOOTSTRAP TABLES */
	shrl	$PGSHIFT,%ecx
	orl	$(PTE_P|PTE_W),%eax
	fillkpt_nox

	/* Map the ISA I/O MEM RW. */
	movl	$IOM_BEGIN,%eax
	movl	$IOM_SIZE,%ecx	/* size of ISA I/O MEM */
	shrl	$PGSHIFT,%ecx
	orl	$(PTE_P|PTE_W/*|PTE_PCD*/),%eax
	fillkpt_nox

	/*
	 * Build L2. Linked to L1.
	 */
	leal	(PROC0_PTP2_OFF)(%esi),%ebx
	leal	(PROC0_PTP1_OFF)(%esi),%eax
	orl	$(PTE_P|PTE_W),%eax
	movl	$(NKL2_KIMG_ENTRIES+1),%ecx
	fillkpt

	/*
	 * Build L3. Linked to L2.
	 */
	leal	(PROC0_PTP3_OFF)(%esi),%ebx
	leal	(PROC0_PTP2_OFF)(%esi),%eax
	orl	$(PTE_P|PTE_W),%eax
	movl	$NKL3_KIMG_ENTRIES,%ecx
	fillkpt

	/*
	 * Build L4. Linked to L3.
	 */
	leal	(PROC0_PML4_OFF)(%esi),%ebx
	leal	(PROC0_PTP3_OFF)(%esi),%eax
	orl	$(PTE_P|PTE_W),%eax
	movl	$NKL4_KIMG_ENTRIES,%ecx
	fillkpt

	/* Install recursive top level PDE (one entry) */
	leal	(PROC0_PML4_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx
	leal	(PROC0_PML4_OFF)(%esi),%eax
	orl	$(PTE_P|PTE_W),%eax
	movl	$1,%ecx
	fillkpt_nox

	/*
	 * Startup checklist:
	 * 1. Enable PAE (and SSE while here).
	 */
	movl	%cr4,%eax
	orl	$(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax
	movl	%eax,%cr4

	/*
	 * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions,
	 *    and NOX if available.
	 */
	movl	$MSR_EFER,%ecx
	rdmsr
	xorl	%eax,%eax
	orl	$(EFER_LME|EFER_SCE),%eax
	movl	_C_LABEL(nox_flag),%ebx
	cmpl	$0,%ebx
	je 	.Lskip_NOX
	orl	$(EFER_NXE),%eax
.Lskip_NOX:
	wrmsr

	/*
	 * 3. Load %cr3 with pointer to PML4.
	 */
	movl	%esi,%eax
	movl	%eax,%cr3

	/*
	 * 4. Enable paging and the rest of it.
	 */
	movl	%cr0,%eax
	orl	$(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax
	movl	%eax,%cr0
	jmp	compat
compat:

	/*
	 * 5. Not quite done yet, we're now in a compatibility segment, in
	 *    legacy mode. We must jump to a long mode segment. Need to set up
	 *    a GDT with a long mode segment in it to do that.
	 */
	movl	$_C_LABEL(gdt64_lo),%eax
	lgdt	(%eax)
	movl	$_C_LABEL(farjmp64),%eax
	ljmp	*(%eax)

	.code64
longmode:

	/*
	 * We have arrived. Everything is identity-mapped.
	 */

	/* Store atdevbase. */
	movq	$TABLESIZE,%rdx
	addq	%rsi,%rdx
	movq	%rdx,_C_LABEL(atdevbase)(%rip)

	/* Set up bootstrap stack. */
	leaq	(PROC0_STK_OFF)(%rsi),%rax
	movq	%rax,_C_LABEL(stkpa)(%rip)
	leaq	(USPACE-FRAMESIZE)(%rax),%rsp
	xorq	%rbp,%rbp			/* mark end of frames */

	xorw	%ax,%ax
	movw	%ax,%gs
	movw	%ax,%fs

	movw	$GSEL(GDATA_SEL, SEL_KPL),%ax
	movw	%ax,%ss

	/* The first physical page available. */
	leaq	(TABLESIZE)(%rsi),%rdi

	/*
	 * Continue execution in C.
	 */
	call	_C_LABEL(init_prekern)

	ret
END(start)

/* -------------------------------------------------------------------------- */

ENTRY(cpuid)
	movq	%rbx,%r8
	movq	%rdi,%rax
	movq	%rsi,%rcx
	movq	%rdx,%rsi
	cpuid
	movl	%eax,0(%rsi)
	movl	%ebx,4(%rsi)
	movl	%ecx,8(%rsi)
	movl	%edx,12(%rsi)
	movq	%r8,%rbx
	ret
END(cpuid)

ENTRY(lidt)
	lidt	(%rdi)
	ret
END(lidt)

ENTRY(rdtsc)
	xorq	%rax,%rax
	rdtsc
	shlq	$32,%rdx
	orq	%rdx,%rax
	ret
END(rdtsc)

ENTRY(rdseed)
	rdseed	%rax
	jc	.Lrdseed_success
	movq	$(-1),%rax
	ret
.Lrdseed_success:
	movq	%rax,(%rdi)
	xorq	%rax,%rax
	ret
END(rdseed)

ENTRY(rdrand)
	rdrand	%rax
	jc	.Lrdrand_success
	movq	$(-1),%rax
	ret
.Lrdrand_success:
	movq	%rax,(%rdi)
	xorq	%rax,%rax
	ret
END(rdrand)

ENTRY(jump_kernel)
	movq	_C_LABEL(stkva),%rsp
	xorq	%rbp,%rbp
	callq	exec_kernel
END(jump_kernel)