mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-23 01:10:04 +00:00
![Paul Mackerras](/assets/img/avatar_default.png)
With this, if a guest does an H_ENTER with a read/write HPTE on a page which is currently read-only, we make the actual HPTE inserted be a read-only version of the HPTE. We now intercept protection faults as well as HPTE not found faults, and for a protection fault we work out whether it should be reflected to the guest (e.g. because the guest HPTE didn't allow write access to usermode) or handled by switching to kernel context and calling kvmppc_book3s_hv_page_fault, which will then request write access to the page and update the actual HPTE. Signed-off-by: Paul Mackerras <paulus@samba.org> Signed-off-by: Alexander Graf <agraf@suse.de> Signed-off-by: Avi Kivity <avi@redhat.com>
1712 lines
39 KiB
ArmAsm
1712 lines
39 KiB
ArmAsm
/*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License, version 2, as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
|
|
*
|
|
* Derived from book3s_rmhandlers.S and other files, which are:
|
|
*
|
|
* Copyright SUSE Linux Products GmbH 2009
|
|
*
|
|
* Authors: Alexander Graf <agraf@suse.de>
|
|
*/
|
|
|
|
#include <asm/ppc_asm.h>
|
|
#include <asm/kvm_asm.h>
|
|
#include <asm/reg.h>
|
|
#include <asm/mmu.h>
|
|
#include <asm/page.h>
|
|
#include <asm/ptrace.h>
|
|
#include <asm/hvcall.h>
|
|
#include <asm/asm-offsets.h>
|
|
#include <asm/exception-64s.h>
|
|
|
|
/*****************************************************************************
|
|
* *
|
|
* Real Mode handlers that need to be in the linear mapping *
|
|
* *
|
|
****************************************************************************/
|
|
|
|
.globl kvmppc_skip_interrupt
|
|
kvmppc_skip_interrupt:
|
|
mfspr r13,SPRN_SRR0
|
|
addi r13,r13,4
|
|
mtspr SPRN_SRR0,r13
|
|
GET_SCRATCH0(r13)
|
|
rfid
|
|
b .
|
|
|
|
.globl kvmppc_skip_Hinterrupt
|
|
kvmppc_skip_Hinterrupt:
|
|
mfspr r13,SPRN_HSRR0
|
|
addi r13,r13,4
|
|
mtspr SPRN_HSRR0,r13
|
|
GET_SCRATCH0(r13)
|
|
hrfid
|
|
b .
|
|
|
|
/*
|
|
* Call kvmppc_hv_entry in real mode.
|
|
* Must be called with interrupts hard-disabled.
|
|
*
|
|
* Input Registers:
|
|
*
|
|
* LR = return address to continue at after eventually re-enabling MMU
|
|
*/
|
|
_GLOBAL(kvmppc_hv_entry_trampoline)
|
|
mfmsr r10
|
|
LOAD_REG_ADDR(r5, kvmppc_hv_entry)
|
|
li r0,MSR_RI
|
|
andc r0,r10,r0
|
|
li r6,MSR_IR | MSR_DR
|
|
andc r6,r10,r6
|
|
mtmsrd r0,1 /* clear RI in MSR */
|
|
mtsrr0 r5
|
|
mtsrr1 r6
|
|
RFI
|
|
|
|
#define ULONG_SIZE 8
|
|
#define VCPU_GPR(n) (VCPU_GPRS + (n * ULONG_SIZE))
|
|
|
|
/******************************************************************************
|
|
* *
|
|
* Entry code *
|
|
* *
|
|
*****************************************************************************/
|
|
|
|
#define XICS_XIRR 4
|
|
#define XICS_QIRR 0xc
|
|
|
|
/*
|
|
* We come in here when wakened from nap mode on a secondary hw thread.
|
|
* Relocation is off and most register values are lost.
|
|
* r13 points to the PACA.
|
|
*/
|
|
.globl kvm_start_guest
|
|
kvm_start_guest:
|
|
ld r1,PACAEMERGSP(r13)
|
|
subi r1,r1,STACK_FRAME_OVERHEAD
|
|
ld r2,PACATOC(r13)
|
|
|
|
/* were we napping due to cede? */
|
|
lbz r0,HSTATE_NAPPING(r13)
|
|
cmpwi r0,0
|
|
bne kvm_end_cede
|
|
|
|
/* get vcpu pointer */
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
|
|
/* We got here with an IPI; clear it */
|
|
ld r5, HSTATE_XICS_PHYS(r13)
|
|
li r0, 0xff
|
|
li r6, XICS_QIRR
|
|
li r7, XICS_XIRR
|
|
lwzcix r8, r5, r7 /* ack the interrupt */
|
|
sync
|
|
stbcix r0, r5, r6 /* clear it */
|
|
stwcix r8, r5, r7 /* EOI it */
|
|
|
|
/* NV GPR values from power7_idle() will no longer be valid */
|
|
stb r0, PACA_NAPSTATELOST(r13)
|
|
|
|
.global kvmppc_hv_entry
|
|
kvmppc_hv_entry:
|
|
|
|
/* Required state:
|
|
*
|
|
* R4 = vcpu pointer
|
|
* MSR = ~IR|DR
|
|
* R13 = PACA
|
|
* R1 = host R1
|
|
* all other volatile GPRS = free
|
|
*/
|
|
mflr r0
|
|
std r0, HSTATE_VMHANDLER(r13)
|
|
|
|
ld r14, VCPU_GPR(r14)(r4)
|
|
ld r15, VCPU_GPR(r15)(r4)
|
|
ld r16, VCPU_GPR(r16)(r4)
|
|
ld r17, VCPU_GPR(r17)(r4)
|
|
ld r18, VCPU_GPR(r18)(r4)
|
|
ld r19, VCPU_GPR(r19)(r4)
|
|
ld r20, VCPU_GPR(r20)(r4)
|
|
ld r21, VCPU_GPR(r21)(r4)
|
|
ld r22, VCPU_GPR(r22)(r4)
|
|
ld r23, VCPU_GPR(r23)(r4)
|
|
ld r24, VCPU_GPR(r24)(r4)
|
|
ld r25, VCPU_GPR(r25)(r4)
|
|
ld r26, VCPU_GPR(r26)(r4)
|
|
ld r27, VCPU_GPR(r27)(r4)
|
|
ld r28, VCPU_GPR(r28)(r4)
|
|
ld r29, VCPU_GPR(r29)(r4)
|
|
ld r30, VCPU_GPR(r30)(r4)
|
|
ld r31, VCPU_GPR(r31)(r4)
|
|
|
|
/* Load guest PMU registers */
|
|
/* R4 is live here (vcpu pointer) */
|
|
li r3, 1
|
|
sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
|
|
mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
|
|
isync
|
|
lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */
|
|
lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */
|
|
lwz r6, VCPU_PMC + 8(r4)
|
|
lwz r7, VCPU_PMC + 12(r4)
|
|
lwz r8, VCPU_PMC + 16(r4)
|
|
lwz r9, VCPU_PMC + 20(r4)
|
|
BEGIN_FTR_SECTION
|
|
lwz r10, VCPU_PMC + 24(r4)
|
|
lwz r11, VCPU_PMC + 28(r4)
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
|
|
mtspr SPRN_PMC1, r3
|
|
mtspr SPRN_PMC2, r5
|
|
mtspr SPRN_PMC3, r6
|
|
mtspr SPRN_PMC4, r7
|
|
mtspr SPRN_PMC5, r8
|
|
mtspr SPRN_PMC6, r9
|
|
BEGIN_FTR_SECTION
|
|
mtspr SPRN_PMC7, r10
|
|
mtspr SPRN_PMC8, r11
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
|
|
ld r3, VCPU_MMCR(r4)
|
|
ld r5, VCPU_MMCR + 8(r4)
|
|
ld r6, VCPU_MMCR + 16(r4)
|
|
mtspr SPRN_MMCR1, r5
|
|
mtspr SPRN_MMCRA, r6
|
|
mtspr SPRN_MMCR0, r3
|
|
isync
|
|
|
|
/* Load up FP, VMX and VSX registers */
|
|
bl kvmppc_load_fp
|
|
|
|
BEGIN_FTR_SECTION
|
|
/* Switch DSCR to guest value */
|
|
ld r5, VCPU_DSCR(r4)
|
|
mtspr SPRN_DSCR, r5
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
|
|
|
|
/*
|
|
* Set the decrementer to the guest decrementer.
|
|
*/
|
|
ld r8,VCPU_DEC_EXPIRES(r4)
|
|
mftb r7
|
|
subf r3,r7,r8
|
|
mtspr SPRN_DEC,r3
|
|
stw r3,VCPU_DEC(r4)
|
|
|
|
ld r5, VCPU_SPRG0(r4)
|
|
ld r6, VCPU_SPRG1(r4)
|
|
ld r7, VCPU_SPRG2(r4)
|
|
ld r8, VCPU_SPRG3(r4)
|
|
mtspr SPRN_SPRG0, r5
|
|
mtspr SPRN_SPRG1, r6
|
|
mtspr SPRN_SPRG2, r7
|
|
mtspr SPRN_SPRG3, r8
|
|
|
|
/* Save R1 in the PACA */
|
|
std r1, HSTATE_HOST_R1(r13)
|
|
|
|
/* Increment yield count if they have a VPA */
|
|
ld r3, VCPU_VPA(r4)
|
|
cmpdi r3, 0
|
|
beq 25f
|
|
lwz r5, LPPACA_YIELDCOUNT(r3)
|
|
addi r5, r5, 1
|
|
stw r5, LPPACA_YIELDCOUNT(r3)
|
|
25:
|
|
/* Load up DAR and DSISR */
|
|
ld r5, VCPU_DAR(r4)
|
|
lwz r6, VCPU_DSISR(r4)
|
|
mtspr SPRN_DAR, r5
|
|
mtspr SPRN_DSISR, r6
|
|
|
|
/* Set partition DABR */
|
|
li r5,3
|
|
ld r6,VCPU_DABR(r4)
|
|
mtspr SPRN_DABRX,r5
|
|
mtspr SPRN_DABR,r6
|
|
|
|
BEGIN_FTR_SECTION
|
|
/* Restore AMR and UAMOR, set AMOR to all 1s */
|
|
ld r5,VCPU_AMR(r4)
|
|
ld r6,VCPU_UAMOR(r4)
|
|
li r7,-1
|
|
mtspr SPRN_AMR,r5
|
|
mtspr SPRN_UAMOR,r6
|
|
mtspr SPRN_AMOR,r7
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
|
|
|
|
/* Clear out SLB */
|
|
li r6,0
|
|
slbmte r6,r6
|
|
slbia
|
|
ptesync
|
|
|
|
BEGIN_FTR_SECTION
|
|
b 30f
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
|
|
/*
|
|
* POWER7 host -> guest partition switch code.
|
|
* We don't have to lock against concurrent tlbies,
|
|
* but we do have to coordinate across hardware threads.
|
|
*/
|
|
/* Increment entry count iff exit count is zero. */
|
|
ld r5,HSTATE_KVM_VCORE(r13)
|
|
addi r9,r5,VCORE_ENTRY_EXIT
|
|
21: lwarx r3,0,r9
|
|
cmpwi r3,0x100 /* any threads starting to exit? */
|
|
bge secondary_too_late /* if so we're too late to the party */
|
|
addi r3,r3,1
|
|
stwcx. r3,0,r9
|
|
bne 21b
|
|
|
|
/* Primary thread switches to guest partition. */
|
|
ld r9,VCPU_KVM(r4) /* pointer to struct kvm */
|
|
lwz r6,VCPU_PTID(r4)
|
|
cmpwi r6,0
|
|
bne 20f
|
|
ld r6,KVM_SDR1(r9)
|
|
lwz r7,KVM_LPID(r9)
|
|
li r0,LPID_RSVD /* switch to reserved LPID */
|
|
mtspr SPRN_LPID,r0
|
|
ptesync
|
|
mtspr SPRN_SDR1,r6 /* switch to partition page table */
|
|
mtspr SPRN_LPID,r7
|
|
isync
|
|
li r0,1
|
|
stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
|
|
b 10f
|
|
|
|
/* Secondary threads wait for primary to have done partition switch */
|
|
20: lbz r0,VCORE_IN_GUEST(r5)
|
|
cmpwi r0,0
|
|
beq 20b
|
|
|
|
/* Set LPCR and RMOR. */
|
|
10: ld r8,KVM_LPCR(r9)
|
|
mtspr SPRN_LPCR,r8
|
|
ld r8,KVM_RMOR(r9)
|
|
mtspr SPRN_RMOR,r8
|
|
isync
|
|
|
|
/* Check if HDEC expires soon */
|
|
mfspr r3,SPRN_HDEC
|
|
cmpwi r3,10
|
|
li r12,BOOK3S_INTERRUPT_HV_DECREMENTER
|
|
mr r9,r4
|
|
blt hdec_soon
|
|
|
|
/*
|
|
* Invalidate the TLB if we could possibly have stale TLB
|
|
* entries for this partition on this core due to the use
|
|
* of tlbiel.
|
|
* XXX maybe only need this on primary thread?
|
|
*/
|
|
ld r9,VCPU_KVM(r4) /* pointer to struct kvm */
|
|
lwz r5,VCPU_VCPUID(r4)
|
|
lhz r6,PACAPACAINDEX(r13)
|
|
rldimi r6,r5,0,62 /* XXX map as if threads 1:1 p:v */
|
|
lhz r8,VCPU_LAST_CPU(r4)
|
|
sldi r7,r6,1 /* see if this is the same vcpu */
|
|
add r7,r7,r9 /* as last ran on this pcpu */
|
|
lhz r0,KVM_LAST_VCPU(r7)
|
|
cmpw r6,r8 /* on the same cpu core as last time? */
|
|
bne 3f
|
|
cmpw r0,r5 /* same vcpu as this core last ran? */
|
|
beq 1f
|
|
3: sth r6,VCPU_LAST_CPU(r4) /* if not, invalidate partition TLB */
|
|
sth r5,KVM_LAST_VCPU(r7)
|
|
li r6,128
|
|
mtctr r6
|
|
li r7,0x800 /* IS field = 0b10 */
|
|
ptesync
|
|
2: tlbiel r7
|
|
addi r7,r7,0x1000
|
|
bdnz 2b
|
|
ptesync
|
|
1:
|
|
|
|
/* Save purr/spurr */
|
|
mfspr r5,SPRN_PURR
|
|
mfspr r6,SPRN_SPURR
|
|
std r5,HSTATE_PURR(r13)
|
|
std r6,HSTATE_SPURR(r13)
|
|
ld r7,VCPU_PURR(r4)
|
|
ld r8,VCPU_SPURR(r4)
|
|
mtspr SPRN_PURR,r7
|
|
mtspr SPRN_SPURR,r8
|
|
b 31f
|
|
|
|
/*
|
|
* PPC970 host -> guest partition switch code.
|
|
* We have to lock against concurrent tlbies,
|
|
* using native_tlbie_lock to lock against host tlbies
|
|
* and kvm->arch.tlbie_lock to lock against guest tlbies.
|
|
* We also have to invalidate the TLB since its
|
|
* entries aren't tagged with the LPID.
|
|
*/
|
|
30: ld r9,VCPU_KVM(r4) /* pointer to struct kvm */
|
|
|
|
/* first take native_tlbie_lock */
|
|
.section ".toc","aw"
|
|
toc_tlbie_lock:
|
|
.tc native_tlbie_lock[TC],native_tlbie_lock
|
|
.previous
|
|
ld r3,toc_tlbie_lock@toc(2)
|
|
lwz r8,PACA_LOCK_TOKEN(r13)
|
|
24: lwarx r0,0,r3
|
|
cmpwi r0,0
|
|
bne 24b
|
|
stwcx. r8,0,r3
|
|
bne 24b
|
|
isync
|
|
|
|
ld r7,KVM_LPCR(r9) /* use kvm->arch.lpcr to store HID4 */
|
|
li r0,0x18f
|
|
rotldi r0,r0,HID4_LPID5_SH /* all lpid bits in HID4 = 1 */
|
|
or r0,r7,r0
|
|
ptesync
|
|
sync
|
|
mtspr SPRN_HID4,r0 /* switch to reserved LPID */
|
|
isync
|
|
li r0,0
|
|
stw r0,0(r3) /* drop native_tlbie_lock */
|
|
|
|
/* invalidate the whole TLB */
|
|
li r0,256
|
|
mtctr r0
|
|
li r6,0
|
|
25: tlbiel r6
|
|
addi r6,r6,0x1000
|
|
bdnz 25b
|
|
ptesync
|
|
|
|
/* Take the guest's tlbie_lock */
|
|
addi r3,r9,KVM_TLBIE_LOCK
|
|
24: lwarx r0,0,r3
|
|
cmpwi r0,0
|
|
bne 24b
|
|
stwcx. r8,0,r3
|
|
bne 24b
|
|
isync
|
|
ld r6,KVM_SDR1(r9)
|
|
mtspr SPRN_SDR1,r6 /* switch to partition page table */
|
|
|
|
/* Set up HID4 with the guest's LPID etc. */
|
|
sync
|
|
mtspr SPRN_HID4,r7
|
|
isync
|
|
|
|
/* drop the guest's tlbie_lock */
|
|
li r0,0
|
|
stw r0,0(r3)
|
|
|
|
/* Check if HDEC expires soon */
|
|
mfspr r3,SPRN_HDEC
|
|
cmpwi r3,10
|
|
li r12,BOOK3S_INTERRUPT_HV_DECREMENTER
|
|
mr r9,r4
|
|
blt hdec_soon
|
|
|
|
/* Enable HDEC interrupts */
|
|
mfspr r0,SPRN_HID0
|
|
li r3,1
|
|
rldimi r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
|
|
sync
|
|
mtspr SPRN_HID0,r0
|
|
mfspr r0,SPRN_HID0
|
|
mfspr r0,SPRN_HID0
|
|
mfspr r0,SPRN_HID0
|
|
mfspr r0,SPRN_HID0
|
|
mfspr r0,SPRN_HID0
|
|
mfspr r0,SPRN_HID0
|
|
|
|
/* Load up guest SLB entries */
|
|
31: lwz r5,VCPU_SLB_MAX(r4)
|
|
cmpwi r5,0
|
|
beq 9f
|
|
mtctr r5
|
|
addi r6,r4,VCPU_SLB
|
|
1: ld r8,VCPU_SLB_E(r6)
|
|
ld r9,VCPU_SLB_V(r6)
|
|
slbmte r9,r8
|
|
addi r6,r6,VCPU_SLB_SIZE
|
|
bdnz 1b
|
|
9:
|
|
|
|
/* Restore state of CTRL run bit; assume 1 on entry */
|
|
lwz r5,VCPU_CTRL(r4)
|
|
andi. r5,r5,1
|
|
bne 4f
|
|
mfspr r6,SPRN_CTRLF
|
|
clrrdi r6,r6,1
|
|
mtspr SPRN_CTRLT,r6
|
|
4:
|
|
ld r6, VCPU_CTR(r4)
|
|
lwz r7, VCPU_XER(r4)
|
|
|
|
mtctr r6
|
|
mtxer r7
|
|
|
|
kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */
|
|
ld r6, VCPU_SRR0(r4)
|
|
ld r7, VCPU_SRR1(r4)
|
|
ld r10, VCPU_PC(r4)
|
|
ld r11, VCPU_MSR(r4) /* r11 = vcpu->arch.msr & ~MSR_HV */
|
|
|
|
rldicl r11, r11, 63 - MSR_HV_LG, 1
|
|
rotldi r11, r11, 1 + MSR_HV_LG
|
|
ori r11, r11, MSR_ME
|
|
|
|
/* Check if we can deliver an external or decrementer interrupt now */
|
|
ld r0,VCPU_PENDING_EXC(r4)
|
|
li r8,(1 << BOOK3S_IRQPRIO_EXTERNAL)
|
|
oris r8,r8,(1 << BOOK3S_IRQPRIO_EXTERNAL_LEVEL)@h
|
|
and r0,r0,r8
|
|
cmpdi cr1,r0,0
|
|
andi. r0,r11,MSR_EE
|
|
beq cr1,11f
|
|
BEGIN_FTR_SECTION
|
|
mfspr r8,SPRN_LPCR
|
|
ori r8,r8,LPCR_MER
|
|
mtspr SPRN_LPCR,r8
|
|
isync
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
|
|
beq 5f
|
|
li r0,BOOK3S_INTERRUPT_EXTERNAL
|
|
12: mr r6,r10
|
|
mr r10,r0
|
|
mr r7,r11
|
|
li r11,(MSR_ME << 1) | 1 /* synthesize MSR_SF | MSR_ME */
|
|
rotldi r11,r11,63
|
|
b 5f
|
|
11: beq 5f
|
|
mfspr r0,SPRN_DEC
|
|
cmpwi r0,0
|
|
li r0,BOOK3S_INTERRUPT_DECREMENTER
|
|
blt 12b
|
|
|
|
/* Move SRR0 and SRR1 into the respective regs */
|
|
5: mtspr SPRN_SRR0, r6
|
|
mtspr SPRN_SRR1, r7
|
|
li r0,0
|
|
stb r0,VCPU_CEDED(r4) /* cancel cede */
|
|
|
|
fast_guest_return:
|
|
mtspr SPRN_HSRR0,r10
|
|
mtspr SPRN_HSRR1,r11
|
|
|
|
/* Activate guest mode, so faults get handled by KVM */
|
|
li r9, KVM_GUEST_MODE_GUEST
|
|
stb r9, HSTATE_IN_GUEST(r13)
|
|
|
|
/* Enter guest */
|
|
|
|
ld r5, VCPU_LR(r4)
|
|
lwz r6, VCPU_CR(r4)
|
|
mtlr r5
|
|
mtcr r6
|
|
|
|
ld r0, VCPU_GPR(r0)(r4)
|
|
ld r1, VCPU_GPR(r1)(r4)
|
|
ld r2, VCPU_GPR(r2)(r4)
|
|
ld r3, VCPU_GPR(r3)(r4)
|
|
ld r5, VCPU_GPR(r5)(r4)
|
|
ld r6, VCPU_GPR(r6)(r4)
|
|
ld r7, VCPU_GPR(r7)(r4)
|
|
ld r8, VCPU_GPR(r8)(r4)
|
|
ld r9, VCPU_GPR(r9)(r4)
|
|
ld r10, VCPU_GPR(r10)(r4)
|
|
ld r11, VCPU_GPR(r11)(r4)
|
|
ld r12, VCPU_GPR(r12)(r4)
|
|
ld r13, VCPU_GPR(r13)(r4)
|
|
|
|
ld r4, VCPU_GPR(r4)(r4)
|
|
|
|
hrfid
|
|
b .
|
|
|
|
/******************************************************************************
|
|
* *
|
|
* Exit code *
|
|
* *
|
|
*****************************************************************************/
|
|
|
|
/*
|
|
* We come here from the first-level interrupt handlers.
|
|
*/
|
|
.globl kvmppc_interrupt
|
|
kvmppc_interrupt:
|
|
/*
|
|
* Register contents:
|
|
* R12 = interrupt vector
|
|
* R13 = PACA
|
|
* guest CR, R12 saved in shadow VCPU SCRATCH1/0
|
|
* guest R13 saved in SPRN_SCRATCH0
|
|
*/
|
|
/* abuse host_r2 as third scratch area; we get r2 from PACATOC(r13) */
|
|
std r9, HSTATE_HOST_R2(r13)
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
|
|
/* Save registers */
|
|
|
|
std r0, VCPU_GPR(r0)(r9)
|
|
std r1, VCPU_GPR(r1)(r9)
|
|
std r2, VCPU_GPR(r2)(r9)
|
|
std r3, VCPU_GPR(r3)(r9)
|
|
std r4, VCPU_GPR(r4)(r9)
|
|
std r5, VCPU_GPR(r5)(r9)
|
|
std r6, VCPU_GPR(r6)(r9)
|
|
std r7, VCPU_GPR(r7)(r9)
|
|
std r8, VCPU_GPR(r8)(r9)
|
|
ld r0, HSTATE_HOST_R2(r13)
|
|
std r0, VCPU_GPR(r9)(r9)
|
|
std r10, VCPU_GPR(r10)(r9)
|
|
std r11, VCPU_GPR(r11)(r9)
|
|
ld r3, HSTATE_SCRATCH0(r13)
|
|
lwz r4, HSTATE_SCRATCH1(r13)
|
|
std r3, VCPU_GPR(r12)(r9)
|
|
stw r4, VCPU_CR(r9)
|
|
|
|
/* Restore R1/R2 so we can handle faults */
|
|
ld r1, HSTATE_HOST_R1(r13)
|
|
ld r2, PACATOC(r13)
|
|
|
|
mfspr r10, SPRN_SRR0
|
|
mfspr r11, SPRN_SRR1
|
|
std r10, VCPU_SRR0(r9)
|
|
std r11, VCPU_SRR1(r9)
|
|
andi. r0, r12, 2 /* need to read HSRR0/1? */
|
|
beq 1f
|
|
mfspr r10, SPRN_HSRR0
|
|
mfspr r11, SPRN_HSRR1
|
|
clrrdi r12, r12, 2
|
|
1: std r10, VCPU_PC(r9)
|
|
std r11, VCPU_MSR(r9)
|
|
|
|
GET_SCRATCH0(r3)
|
|
mflr r4
|
|
std r3, VCPU_GPR(r13)(r9)
|
|
std r4, VCPU_LR(r9)
|
|
|
|
/* Unset guest mode */
|
|
li r0, KVM_GUEST_MODE_NONE
|
|
stb r0, HSTATE_IN_GUEST(r13)
|
|
|
|
stw r12,VCPU_TRAP(r9)
|
|
|
|
/* Save HEIR (HV emulation assist reg) in last_inst
|
|
if this is an HEI (HV emulation interrupt, e40) */
|
|
li r3,KVM_INST_FETCH_FAILED
|
|
BEGIN_FTR_SECTION
|
|
cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
|
|
bne 11f
|
|
mfspr r3,SPRN_HEIR
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
|
|
11: stw r3,VCPU_LAST_INST(r9)
|
|
|
|
/* these are volatile across C function calls */
|
|
mfctr r3
|
|
mfxer r4
|
|
std r3, VCPU_CTR(r9)
|
|
stw r4, VCPU_XER(r9)
|
|
|
|
BEGIN_FTR_SECTION
|
|
/* If this is a page table miss then see if it's theirs or ours */
|
|
cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
|
|
beq kvmppc_hdsi
|
|
cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE
|
|
beq kvmppc_hisi
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
|
|
|
|
/* See if this is a leftover HDEC interrupt */
|
|
cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
|
|
bne 2f
|
|
mfspr r3,SPRN_HDEC
|
|
cmpwi r3,0
|
|
bge ignore_hdec
|
|
2:
|
|
/* See if this is an hcall we can handle in real mode */
|
|
cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
|
|
beq hcall_try_real_mode
|
|
|
|
/* Check for mediated interrupts (could be done earlier really ...) */
|
|
BEGIN_FTR_SECTION
|
|
cmpwi r12,BOOK3S_INTERRUPT_EXTERNAL
|
|
bne+ 1f
|
|
andi. r0,r11,MSR_EE
|
|
beq 1f
|
|
mfspr r5,SPRN_LPCR
|
|
andi. r0,r5,LPCR_MER
|
|
bne bounce_ext_interrupt
|
|
1:
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
|
|
|
|
nohpte_cont:
|
|
hcall_real_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
|
|
/* Save DEC */
|
|
mfspr r5,SPRN_DEC
|
|
mftb r6
|
|
extsw r5,r5
|
|
add r5,r5,r6
|
|
std r5,VCPU_DEC_EXPIRES(r9)
|
|
|
|
/* Save more register state */
|
|
mfdar r6
|
|
mfdsisr r7
|
|
std r6, VCPU_DAR(r9)
|
|
stw r7, VCPU_DSISR(r9)
|
|
BEGIN_FTR_SECTION
|
|
/* don't overwrite fault_dar/fault_dsisr if HDSI */
|
|
cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
|
|
beq 6f
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
|
|
std r6, VCPU_FAULT_DAR(r9)
|
|
stw r7, VCPU_FAULT_DSISR(r9)
|
|
|
|
/* Save guest CTRL register, set runlatch to 1 */
|
|
6: mfspr r6,SPRN_CTRLF
|
|
stw r6,VCPU_CTRL(r9)
|
|
andi. r0,r6,1
|
|
bne 4f
|
|
ori r6,r6,1
|
|
mtspr SPRN_CTRLT,r6
|
|
4:
|
|
/* Read the guest SLB and save it away */
|
|
lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
|
|
mtctr r0
|
|
li r6,0
|
|
addi r7,r9,VCPU_SLB
|
|
li r5,0
|
|
1: slbmfee r8,r6
|
|
andis. r0,r8,SLB_ESID_V@h
|
|
beq 2f
|
|
add r8,r8,r6 /* put index in */
|
|
slbmfev r3,r6
|
|
std r8,VCPU_SLB_E(r7)
|
|
std r3,VCPU_SLB_V(r7)
|
|
addi r7,r7,VCPU_SLB_SIZE
|
|
addi r5,r5,1
|
|
2: addi r6,r6,1
|
|
bdnz 1b
|
|
stw r5,VCPU_SLB_MAX(r9)
|
|
|
|
/*
|
|
* Save the guest PURR/SPURR
|
|
*/
|
|
BEGIN_FTR_SECTION
|
|
mfspr r5,SPRN_PURR
|
|
mfspr r6,SPRN_SPURR
|
|
ld r7,VCPU_PURR(r9)
|
|
ld r8,VCPU_SPURR(r9)
|
|
std r5,VCPU_PURR(r9)
|
|
std r6,VCPU_SPURR(r9)
|
|
subf r5,r7,r5
|
|
subf r6,r8,r6
|
|
|
|
/*
|
|
* Restore host PURR/SPURR and add guest times
|
|
* so that the time in the guest gets accounted.
|
|
*/
|
|
ld r3,HSTATE_PURR(r13)
|
|
ld r4,HSTATE_SPURR(r13)
|
|
add r3,r3,r5
|
|
add r4,r4,r6
|
|
mtspr SPRN_PURR,r3
|
|
mtspr SPRN_SPURR,r4
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_201)
|
|
|
|
/* Clear out SLB */
|
|
li r5,0
|
|
slbmte r5,r5
|
|
slbia
|
|
ptesync
|
|
|
|
hdec_soon: /* r9 = vcpu, r12 = trap, r13 = paca */
|
|
BEGIN_FTR_SECTION
|
|
b 32f
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
|
|
/*
|
|
* POWER7 guest -> host partition switch code.
|
|
* We don't have to lock against tlbies but we do
|
|
* have to coordinate the hardware threads.
|
|
*/
|
|
/* Increment the threads-exiting-guest count in the 0xff00
|
|
bits of vcore->entry_exit_count */
|
|
lwsync
|
|
ld r5,HSTATE_KVM_VCORE(r13)
|
|
addi r6,r5,VCORE_ENTRY_EXIT
|
|
41: lwarx r3,0,r6
|
|
addi r0,r3,0x100
|
|
stwcx. r0,0,r6
|
|
bne 41b
|
|
lwsync
|
|
|
|
/*
|
|
* At this point we have an interrupt that we have to pass
|
|
* up to the kernel or qemu; we can't handle it in real mode.
|
|
* Thus we have to do a partition switch, so we have to
|
|
* collect the other threads, if we are the first thread
|
|
* to take an interrupt. To do this, we set the HDEC to 0,
|
|
* which causes an HDEC interrupt in all threads within 2ns
|
|
* because the HDEC register is shared between all 4 threads.
|
|
* However, we don't need to bother if this is an HDEC
|
|
* interrupt, since the other threads will already be on their
|
|
* way here in that case.
|
|
*/
|
|
cmpwi r3,0x100 /* Are we the first here? */
|
|
bge 43f
|
|
cmpwi r3,1 /* Are any other threads in the guest? */
|
|
ble 43f
|
|
cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
|
|
beq 40f
|
|
li r0,0
|
|
mtspr SPRN_HDEC,r0
|
|
40:
|
|
/*
|
|
* Send an IPI to any napping threads, since an HDEC interrupt
|
|
* doesn't wake CPUs up from nap.
|
|
*/
|
|
lwz r3,VCORE_NAPPING_THREADS(r5)
|
|
lwz r4,VCPU_PTID(r9)
|
|
li r0,1
|
|
sldi r0,r0,r4
|
|
andc. r3,r3,r0 /* no sense IPI'ing ourselves */
|
|
beq 43f
|
|
mulli r4,r4,PACA_SIZE /* get paca for thread 0 */
|
|
subf r6,r4,r13
|
|
42: andi. r0,r3,1
|
|
beq 44f
|
|
ld r8,HSTATE_XICS_PHYS(r6) /* get thread's XICS reg addr */
|
|
li r0,IPI_PRIORITY
|
|
li r7,XICS_QIRR
|
|
stbcix r0,r7,r8 /* trigger the IPI */
|
|
44: srdi. r3,r3,1
|
|
addi r6,r6,PACA_SIZE
|
|
bne 42b
|
|
|
|
/* Secondary threads wait for primary to do partition switch */
|
|
43: ld r4,VCPU_KVM(r9) /* pointer to struct kvm */
|
|
ld r5,HSTATE_KVM_VCORE(r13)
|
|
lwz r3,VCPU_PTID(r9)
|
|
cmpwi r3,0
|
|
beq 15f
|
|
HMT_LOW
|
|
13: lbz r3,VCORE_IN_GUEST(r5)
|
|
cmpwi r3,0
|
|
bne 13b
|
|
HMT_MEDIUM
|
|
b 16f
|
|
|
|
/* Primary thread waits for all the secondaries to exit guest */
|
|
15: lwz r3,VCORE_ENTRY_EXIT(r5)
|
|
srwi r0,r3,8
|
|
clrldi r3,r3,56
|
|
cmpw r3,r0
|
|
bne 15b
|
|
isync
|
|
|
|
/* Primary thread switches back to host partition */
|
|
ld r6,KVM_HOST_SDR1(r4)
|
|
lwz r7,KVM_HOST_LPID(r4)
|
|
li r8,LPID_RSVD /* switch to reserved LPID */
|
|
mtspr SPRN_LPID,r8
|
|
ptesync
|
|
mtspr SPRN_SDR1,r6 /* switch to partition page table */
|
|
mtspr SPRN_LPID,r7
|
|
isync
|
|
li r0,0
|
|
stb r0,VCORE_IN_GUEST(r5)
|
|
lis r8,0x7fff /* MAX_INT@h */
|
|
mtspr SPRN_HDEC,r8
|
|
|
|
16: ld r8,KVM_HOST_LPCR(r4)
|
|
mtspr SPRN_LPCR,r8
|
|
isync
|
|
b 33f
|
|
|
|
/*
|
|
* PPC970 guest -> host partition switch code.
|
|
* We have to lock against concurrent tlbies, and
|
|
* we have to flush the whole TLB.
|
|
*/
|
|
32: ld r4,VCPU_KVM(r9) /* pointer to struct kvm */
|
|
|
|
/* Take the guest's tlbie_lock */
|
|
lwz r8,PACA_LOCK_TOKEN(r13)
|
|
addi r3,r4,KVM_TLBIE_LOCK
|
|
24: lwarx r0,0,r3
|
|
cmpwi r0,0
|
|
bne 24b
|
|
stwcx. r8,0,r3
|
|
bne 24b
|
|
isync
|
|
|
|
ld r7,KVM_HOST_LPCR(r4) /* use kvm->arch.host_lpcr for HID4 */
|
|
li r0,0x18f
|
|
rotldi r0,r0,HID4_LPID5_SH /* all lpid bits in HID4 = 1 */
|
|
or r0,r7,r0
|
|
ptesync
|
|
sync
|
|
mtspr SPRN_HID4,r0 /* switch to reserved LPID */
|
|
isync
|
|
li r0,0
|
|
stw r0,0(r3) /* drop guest tlbie_lock */
|
|
|
|
/* invalidate the whole TLB */
|
|
li r0,256
|
|
mtctr r0
|
|
li r6,0
|
|
25: tlbiel r6
|
|
addi r6,r6,0x1000
|
|
bdnz 25b
|
|
ptesync
|
|
|
|
/* take native_tlbie_lock */
|
|
ld r3,toc_tlbie_lock@toc(2)
|
|
24: lwarx r0,0,r3
|
|
cmpwi r0,0
|
|
bne 24b
|
|
stwcx. r8,0,r3
|
|
bne 24b
|
|
isync
|
|
|
|
ld r6,KVM_HOST_SDR1(r4)
|
|
mtspr SPRN_SDR1,r6 /* switch to host page table */
|
|
|
|
/* Set up host HID4 value */
|
|
sync
|
|
mtspr SPRN_HID4,r7
|
|
isync
|
|
li r0,0
|
|
stw r0,0(r3) /* drop native_tlbie_lock */
|
|
|
|
lis r8,0x7fff /* MAX_INT@h */
|
|
mtspr SPRN_HDEC,r8
|
|
|
|
/* Disable HDEC interrupts */
|
|
mfspr r0,SPRN_HID0
|
|
li r3,0
|
|
rldimi r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
|
|
sync
|
|
mtspr SPRN_HID0,r0
|
|
mfspr r0,SPRN_HID0
|
|
mfspr r0,SPRN_HID0
|
|
mfspr r0,SPRN_HID0
|
|
mfspr r0,SPRN_HID0
|
|
mfspr r0,SPRN_HID0
|
|
mfspr r0,SPRN_HID0
|
|
|
|
/* load host SLB entries */
|
|
33: ld r8,PACA_SLBSHADOWPTR(r13)
|
|
|
|
.rept SLB_NUM_BOLTED
|
|
ld r5,SLBSHADOW_SAVEAREA(r8)
|
|
ld r6,SLBSHADOW_SAVEAREA+8(r8)
|
|
andis. r7,r5,SLB_ESID_V@h
|
|
beq 1f
|
|
slbmte r6,r5
|
|
1: addi r8,r8,16
|
|
.endr
|
|
|
|
/* Save and reset AMR and UAMOR before turning on the MMU */
|
|
BEGIN_FTR_SECTION
|
|
mfspr r5,SPRN_AMR
|
|
mfspr r6,SPRN_UAMOR
|
|
std r5,VCPU_AMR(r9)
|
|
std r6,VCPU_UAMOR(r9)
|
|
li r6,0
|
|
mtspr SPRN_AMR,r6
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
|
|
|
|
/* Restore host DABR and DABRX */
|
|
ld r5,HSTATE_DABR(r13)
|
|
li r6,7
|
|
mtspr SPRN_DABR,r5
|
|
mtspr SPRN_DABRX,r6
|
|
|
|
/* Switch DSCR back to host value */
|
|
BEGIN_FTR_SECTION
|
|
mfspr r8, SPRN_DSCR
|
|
ld r7, HSTATE_DSCR(r13)
|
|
std r8, VCPU_DSCR(r7)
|
|
mtspr SPRN_DSCR, r7
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
|
|
|
|
/* Save non-volatile GPRs */
|
|
std r14, VCPU_GPR(r14)(r9)
|
|
std r15, VCPU_GPR(r15)(r9)
|
|
std r16, VCPU_GPR(r16)(r9)
|
|
std r17, VCPU_GPR(r17)(r9)
|
|
std r18, VCPU_GPR(r18)(r9)
|
|
std r19, VCPU_GPR(r19)(r9)
|
|
std r20, VCPU_GPR(r20)(r9)
|
|
std r21, VCPU_GPR(r21)(r9)
|
|
std r22, VCPU_GPR(r22)(r9)
|
|
std r23, VCPU_GPR(r23)(r9)
|
|
std r24, VCPU_GPR(r24)(r9)
|
|
std r25, VCPU_GPR(r25)(r9)
|
|
std r26, VCPU_GPR(r26)(r9)
|
|
std r27, VCPU_GPR(r27)(r9)
|
|
std r28, VCPU_GPR(r28)(r9)
|
|
std r29, VCPU_GPR(r29)(r9)
|
|
std r30, VCPU_GPR(r30)(r9)
|
|
std r31, VCPU_GPR(r31)(r9)
|
|
|
|
/* Save SPRGs */
|
|
mfspr r3, SPRN_SPRG0
|
|
mfspr r4, SPRN_SPRG1
|
|
mfspr r5, SPRN_SPRG2
|
|
mfspr r6, SPRN_SPRG3
|
|
std r3, VCPU_SPRG0(r9)
|
|
std r4, VCPU_SPRG1(r9)
|
|
std r5, VCPU_SPRG2(r9)
|
|
std r6, VCPU_SPRG3(r9)
|
|
|
|
/* Increment yield count if they have a VPA */
|
|
ld r8, VCPU_VPA(r9) /* do they have a VPA? */
|
|
cmpdi r8, 0
|
|
beq 25f
|
|
lwz r3, LPPACA_YIELDCOUNT(r8)
|
|
addi r3, r3, 1
|
|
stw r3, LPPACA_YIELDCOUNT(r8)
|
|
25:
|
|
/* Save PMU registers if requested */
|
|
/* r8 and cr0.eq are live here */
|
|
li r3, 1
|
|
sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
|
|
mfspr r4, SPRN_MMCR0 /* save MMCR0 */
|
|
mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
|
|
isync
|
|
beq 21f /* if no VPA, save PMU stuff anyway */
|
|
lbz r7, LPPACA_PMCINUSE(r8)
|
|
cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */
|
|
bne 21f
|
|
std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */
|
|
b 22f
|
|
21: mfspr r5, SPRN_MMCR1
|
|
mfspr r6, SPRN_MMCRA
|
|
std r4, VCPU_MMCR(r9)
|
|
std r5, VCPU_MMCR + 8(r9)
|
|
std r6, VCPU_MMCR + 16(r9)
|
|
mfspr r3, SPRN_PMC1
|
|
mfspr r4, SPRN_PMC2
|
|
mfspr r5, SPRN_PMC3
|
|
mfspr r6, SPRN_PMC4
|
|
mfspr r7, SPRN_PMC5
|
|
mfspr r8, SPRN_PMC6
|
|
BEGIN_FTR_SECTION
|
|
mfspr r10, SPRN_PMC7
|
|
mfspr r11, SPRN_PMC8
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
|
|
stw r3, VCPU_PMC(r9)
|
|
stw r4, VCPU_PMC + 4(r9)
|
|
stw r5, VCPU_PMC + 8(r9)
|
|
stw r6, VCPU_PMC + 12(r9)
|
|
stw r7, VCPU_PMC + 16(r9)
|
|
stw r8, VCPU_PMC + 20(r9)
|
|
BEGIN_FTR_SECTION
|
|
stw r10, VCPU_PMC + 24(r9)
|
|
stw r11, VCPU_PMC + 28(r9)
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
|
|
22:
|
|
/* save FP state */
|
|
mr r3, r9
|
|
bl .kvmppc_save_fp
|
|
|
|
/* Secondary threads go off to take a nap on POWER7 */
|
|
BEGIN_FTR_SECTION
|
|
lwz r0,VCPU_PTID(r3)
|
|
cmpwi r0,0
|
|
bne secondary_nap
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
|
|
|
|
/*
|
|
* Reload DEC. HDEC interrupts were disabled when
|
|
* we reloaded the host's LPCR value.
|
|
*/
|
|
ld r3, HSTATE_DECEXP(r13)
|
|
mftb r4
|
|
subf r4, r4, r3
|
|
mtspr SPRN_DEC, r4
|
|
|
|
/* Reload the host's PMU registers */
|
|
ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */
|
|
lbz r4, LPPACA_PMCINUSE(r3)
|
|
cmpwi r4, 0
|
|
beq 23f /* skip if not */
|
|
lwz r3, HSTATE_PMC(r13)
|
|
lwz r4, HSTATE_PMC + 4(r13)
|
|
lwz r5, HSTATE_PMC + 8(r13)
|
|
lwz r6, HSTATE_PMC + 12(r13)
|
|
lwz r8, HSTATE_PMC + 16(r13)
|
|
lwz r9, HSTATE_PMC + 20(r13)
|
|
BEGIN_FTR_SECTION
|
|
lwz r10, HSTATE_PMC + 24(r13)
|
|
lwz r11, HSTATE_PMC + 28(r13)
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
|
|
mtspr SPRN_PMC1, r3
|
|
mtspr SPRN_PMC2, r4
|
|
mtspr SPRN_PMC3, r5
|
|
mtspr SPRN_PMC4, r6
|
|
mtspr SPRN_PMC5, r8
|
|
mtspr SPRN_PMC6, r9
|
|
BEGIN_FTR_SECTION
|
|
mtspr SPRN_PMC7, r10
|
|
mtspr SPRN_PMC8, r11
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
|
|
ld r3, HSTATE_MMCR(r13)
|
|
ld r4, HSTATE_MMCR + 8(r13)
|
|
ld r5, HSTATE_MMCR + 16(r13)
|
|
mtspr SPRN_MMCR1, r4
|
|
mtspr SPRN_MMCRA, r5
|
|
mtspr SPRN_MMCR0, r3
|
|
isync
|
|
23:
|
|
/*
|
|
* For external and machine check interrupts, we need
|
|
* to call the Linux handler to process the interrupt.
|
|
* We do that by jumping to the interrupt vector address
|
|
* which we have in r12. The [h]rfid at the end of the
|
|
* handler will return to the book3s_hv_interrupts.S code.
|
|
* For other interrupts we do the rfid to get back
|
|
* to the book3s_interrupts.S code here.
|
|
*/
|
|
ld r8, HSTATE_VMHANDLER(r13)
|
|
ld r7, HSTATE_HOST_MSR(r13)
|
|
|
|
cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
|
|
beq 11f
|
|
cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
|
|
|
|
/* RFI into the highmem handler, or branch to interrupt handler */
|
|
12: mfmsr r6
|
|
mtctr r12
|
|
li r0, MSR_RI
|
|
andc r6, r6, r0
|
|
mtmsrd r6, 1 /* Clear RI in MSR */
|
|
mtsrr0 r8
|
|
mtsrr1 r7
|
|
beqctr
|
|
RFI
|
|
|
|
11:
|
|
BEGIN_FTR_SECTION
|
|
b 12b
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
|
|
mtspr SPRN_HSRR0, r8
|
|
mtspr SPRN_HSRR1, r7
|
|
ba 0x500
|
|
|
|
/*
|
|
* Check whether an HDSI is an HPTE not found fault or something else.
|
|
* If it is an HPTE not found fault that is due to the guest accessing
|
|
* a page that they have mapped but which we have paged out, then
|
|
* we continue on with the guest exit path. In all other cases,
|
|
* reflect the HDSI to the guest as a DSI.
|
|
*/
|
|
kvmppc_hdsi:
|
|
mfspr r4, SPRN_HDAR
|
|
mfspr r6, SPRN_HDSISR
|
|
/* HPTE not found fault or protection fault? */
|
|
andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
|
|
beq 1f /* if not, send it to the guest */
|
|
andi. r0, r11, MSR_DR /* data relocation enabled? */
|
|
beq 3f
|
|
clrrdi r0, r4, 28
|
|
PPC_SLBFEE_DOT(r5, r0) /* if so, look up SLB */
|
|
bne 1f /* if no SLB entry found */
|
|
4: std r4, VCPU_FAULT_DAR(r9)
|
|
stw r6, VCPU_FAULT_DSISR(r9)
|
|
|
|
/* Search the hash table. */
|
|
mr r3, r9 /* vcpu pointer */
|
|
li r7, 1 /* data fault */
|
|
bl .kvmppc_hpte_hv_fault
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
ld r10, VCPU_PC(r9)
|
|
ld r11, VCPU_MSR(r9)
|
|
li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
|
|
cmpdi r3, 0 /* retry the instruction */
|
|
beq 6f
|
|
cmpdi r3, -1 /* handle in kernel mode */
|
|
beq nohpte_cont
|
|
cmpdi r3, -2 /* MMIO emulation; need instr word */
|
|
beq 2f
|
|
|
|
/* Synthesize a DSI for the guest */
|
|
ld r4, VCPU_FAULT_DAR(r9)
|
|
mr r6, r3
|
|
1: mtspr SPRN_DAR, r4
|
|
mtspr SPRN_DSISR, r6
|
|
mtspr SPRN_SRR0, r10
|
|
mtspr SPRN_SRR1, r11
|
|
li r10, BOOK3S_INTERRUPT_DATA_STORAGE
|
|
li r11, (MSR_ME << 1) | 1 /* synthesize MSR_SF | MSR_ME */
|
|
rotldi r11, r11, 63
|
|
6: ld r7, VCPU_CTR(r9)
|
|
lwz r8, VCPU_XER(r9)
|
|
mtctr r7
|
|
mtxer r8
|
|
mr r4, r9
|
|
b fast_guest_return
|
|
|
|
3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */
|
|
ld r5, KVM_VRMA_SLB_V(r5)
|
|
b 4b
|
|
|
|
/* If this is for emulated MMIO, load the instruction word */
|
|
2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */
|
|
|
|
/* Set guest mode to 'jump over instruction' so if lwz faults
|
|
* we'll just continue at the next IP. */
|
|
li r0, KVM_GUEST_MODE_SKIP
|
|
stb r0, HSTATE_IN_GUEST(r13)
|
|
|
|
/* Do the access with MSR:DR enabled */
|
|
mfmsr r3
|
|
ori r4, r3, MSR_DR /* Enable paging for data */
|
|
mtmsrd r4
|
|
lwz r8, 0(r10)
|
|
mtmsrd r3
|
|
|
|
/* Store the result */
|
|
stw r8, VCPU_LAST_INST(r9)
|
|
|
|
/* Unset guest mode. */
|
|
li r0, KVM_GUEST_MODE_NONE
|
|
stb r0, HSTATE_IN_GUEST(r13)
|
|
b nohpte_cont
|
|
|
|
/*
|
|
* Similarly for an HISI, reflect it to the guest as an ISI unless
|
|
* it is an HPTE not found fault for a page that we have paged out.
|
|
*/
|
|
kvmppc_hisi:
|
|
andis. r0, r11, SRR1_ISI_NOPT@h
|
|
beq 1f
|
|
andi. r0, r11, MSR_IR /* instruction relocation enabled? */
|
|
beq 3f
|
|
clrrdi r0, r10, 28
|
|
PPC_SLBFEE_DOT(r5, r0) /* if so, look up SLB */
|
|
bne 1f /* if no SLB entry found */
|
|
4:
|
|
/* Search the hash table. */
|
|
mr r3, r9 /* vcpu pointer */
|
|
mr r4, r10
|
|
mr r6, r11
|
|
li r7, 0 /* instruction fault */
|
|
bl .kvmppc_hpte_hv_fault
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
ld r10, VCPU_PC(r9)
|
|
ld r11, VCPU_MSR(r9)
|
|
li r12, BOOK3S_INTERRUPT_H_INST_STORAGE
|
|
cmpdi r3, 0 /* retry the instruction */
|
|
beq 6f
|
|
cmpdi r3, -1 /* handle in kernel mode */
|
|
beq nohpte_cont
|
|
|
|
/* Synthesize an ISI for the guest */
|
|
mr r11, r3
|
|
1: mtspr SPRN_SRR0, r10
|
|
mtspr SPRN_SRR1, r11
|
|
li r10, BOOK3S_INTERRUPT_INST_STORAGE
|
|
li r11, (MSR_ME << 1) | 1 /* synthesize MSR_SF | MSR_ME */
|
|
rotldi r11, r11, 63
|
|
6: ld r7, VCPU_CTR(r9)
|
|
lwz r8, VCPU_XER(r9)
|
|
mtctr r7
|
|
mtxer r8
|
|
mr r4, r9
|
|
b fast_guest_return
|
|
|
|
3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */
|
|
ld r5, KVM_VRMA_SLB_V(r6)
|
|
b 4b
|
|
|
|
/*
|
|
* Try to handle an hcall in real mode.
|
|
* Returns to the guest if we handle it, or continues on up to
|
|
* the kernel if we can't (i.e. if we don't have a handler for
|
|
* it, or if the handler returns H_TOO_HARD).
|
|
*/
|
|
.globl hcall_try_real_mode
|
|
hcall_try_real_mode:
|
|
ld r3,VCPU_GPR(r3)(r9)
|
|
andi. r0,r11,MSR_PR
|
|
bne hcall_real_cont
|
|
clrrdi r3,r3,2
|
|
cmpldi r3,hcall_real_table_end - hcall_real_table
|
|
bge hcall_real_cont
|
|
LOAD_REG_ADDR(r4, hcall_real_table)
|
|
lwzx r3,r3,r4
|
|
cmpwi r3,0
|
|
beq hcall_real_cont
|
|
add r3,r3,r4
|
|
mtctr r3
|
|
mr r3,r9 /* get vcpu pointer */
|
|
ld r4,VCPU_GPR(r4)(r9)
|
|
bctrl
|
|
cmpdi r3,H_TOO_HARD
|
|
beq hcall_real_fallback
|
|
ld r4,HSTATE_KVM_VCPU(r13)
|
|
std r3,VCPU_GPR(r3)(r4)
|
|
ld r10,VCPU_PC(r4)
|
|
ld r11,VCPU_MSR(r4)
|
|
b fast_guest_return
|
|
|
|
/* We've attempted a real mode hcall, but it's punted it back
|
|
* to userspace. We need to restore some clobbered volatiles
|
|
* before resuming the pass-it-to-qemu path */
|
|
hcall_real_fallback:
|
|
li r12,BOOK3S_INTERRUPT_SYSCALL
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
|
|
b hcall_real_cont
|
|
|
|
.globl hcall_real_table
|
|
hcall_real_table:
|
|
.long 0 /* 0 - unused */
|
|
.long .kvmppc_h_remove - hcall_real_table
|
|
.long .kvmppc_h_enter - hcall_real_table
|
|
.long .kvmppc_h_read - hcall_real_table
|
|
.long 0 /* 0x10 - H_CLEAR_MOD */
|
|
.long 0 /* 0x14 - H_CLEAR_REF */
|
|
.long .kvmppc_h_protect - hcall_real_table
|
|
.long 0 /* 0x1c - H_GET_TCE */
|
|
.long .kvmppc_h_put_tce - hcall_real_table
|
|
.long 0 /* 0x24 - H_SET_SPRG0 */
|
|
.long .kvmppc_h_set_dabr - hcall_real_table
|
|
.long 0 /* 0x2c */
|
|
.long 0 /* 0x30 */
|
|
.long 0 /* 0x34 */
|
|
.long 0 /* 0x38 */
|
|
.long 0 /* 0x3c */
|
|
.long 0 /* 0x40 */
|
|
.long 0 /* 0x44 */
|
|
.long 0 /* 0x48 */
|
|
.long 0 /* 0x4c */
|
|
.long 0 /* 0x50 */
|
|
.long 0 /* 0x54 */
|
|
.long 0 /* 0x58 */
|
|
.long 0 /* 0x5c */
|
|
.long 0 /* 0x60 */
|
|
.long 0 /* 0x64 */
|
|
.long 0 /* 0x68 */
|
|
.long 0 /* 0x6c */
|
|
.long 0 /* 0x70 */
|
|
.long 0 /* 0x74 */
|
|
.long 0 /* 0x78 */
|
|
.long 0 /* 0x7c */
|
|
.long 0 /* 0x80 */
|
|
.long 0 /* 0x84 */
|
|
.long 0 /* 0x88 */
|
|
.long 0 /* 0x8c */
|
|
.long 0 /* 0x90 */
|
|
.long 0 /* 0x94 */
|
|
.long 0 /* 0x98 */
|
|
.long 0 /* 0x9c */
|
|
.long 0 /* 0xa0 */
|
|
.long 0 /* 0xa4 */
|
|
.long 0 /* 0xa8 */
|
|
.long 0 /* 0xac */
|
|
.long 0 /* 0xb0 */
|
|
.long 0 /* 0xb4 */
|
|
.long 0 /* 0xb8 */
|
|
.long 0 /* 0xbc */
|
|
.long 0 /* 0xc0 */
|
|
.long 0 /* 0xc4 */
|
|
.long 0 /* 0xc8 */
|
|
.long 0 /* 0xcc */
|
|
.long 0 /* 0xd0 */
|
|
.long 0 /* 0xd4 */
|
|
.long 0 /* 0xd8 */
|
|
.long 0 /* 0xdc */
|
|
.long .kvmppc_h_cede - hcall_real_table
|
|
.long 0 /* 0xe4 */
|
|
.long 0 /* 0xe8 */
|
|
.long 0 /* 0xec */
|
|
.long 0 /* 0xf0 */
|
|
.long 0 /* 0xf4 */
|
|
.long 0 /* 0xf8 */
|
|
.long 0 /* 0xfc */
|
|
.long 0 /* 0x100 */
|
|
.long 0 /* 0x104 */
|
|
.long 0 /* 0x108 */
|
|
.long 0 /* 0x10c */
|
|
.long 0 /* 0x110 */
|
|
.long 0 /* 0x114 */
|
|
.long 0 /* 0x118 */
|
|
.long 0 /* 0x11c */
|
|
.long 0 /* 0x120 */
|
|
.long .kvmppc_h_bulk_remove - hcall_real_table
|
|
hcall_real_table_end:
|
|
|
|
ignore_hdec:
|
|
mr r4,r9
|
|
b fast_guest_return
|
|
|
|
bounce_ext_interrupt:
|
|
mr r4,r9
|
|
mtspr SPRN_SRR0,r10
|
|
mtspr SPRN_SRR1,r11
|
|
li r10,BOOK3S_INTERRUPT_EXTERNAL
|
|
li r11,(MSR_ME << 1) | 1 /* synthesize MSR_SF | MSR_ME */
|
|
rotldi r11,r11,63
|
|
b fast_guest_return
|
|
|
|
_GLOBAL(kvmppc_h_set_dabr)
|
|
std r4,VCPU_DABR(r3)
|
|
mtspr SPRN_DABR,r4
|
|
li r3,0
|
|
blr
|
|
|
|
_GLOBAL(kvmppc_h_cede)
|
|
ori r11,r11,MSR_EE
|
|
std r11,VCPU_MSR(r3)
|
|
li r0,1
|
|
stb r0,VCPU_CEDED(r3)
|
|
sync /* order setting ceded vs. testing prodded */
|
|
lbz r5,VCPU_PRODDED(r3)
|
|
cmpwi r5,0
|
|
bne 1f
|
|
li r0,0 /* set trap to 0 to say hcall is handled */
|
|
stw r0,VCPU_TRAP(r3)
|
|
li r0,H_SUCCESS
|
|
std r0,VCPU_GPR(r3)(r3)
|
|
BEGIN_FTR_SECTION
|
|
b 2f /* just send it up to host on 970 */
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)
|
|
|
|
/*
|
|
* Set our bit in the bitmask of napping threads unless all the
|
|
* other threads are already napping, in which case we send this
|
|
* up to the host.
|
|
*/
|
|
ld r5,HSTATE_KVM_VCORE(r13)
|
|
lwz r6,VCPU_PTID(r3)
|
|
lwz r8,VCORE_ENTRY_EXIT(r5)
|
|
clrldi r8,r8,56
|
|
li r0,1
|
|
sld r0,r0,r6
|
|
addi r6,r5,VCORE_NAPPING_THREADS
|
|
31: lwarx r4,0,r6
|
|
or r4,r4,r0
|
|
PPC_POPCNTW(r7,r4)
|
|
cmpw r7,r8
|
|
bge 2f
|
|
stwcx. r4,0,r6
|
|
bne 31b
|
|
li r0,1
|
|
stb r0,HSTATE_NAPPING(r13)
|
|
/* order napping_threads update vs testing entry_exit_count */
|
|
lwsync
|
|
mr r4,r3
|
|
lwz r7,VCORE_ENTRY_EXIT(r5)
|
|
cmpwi r7,0x100
|
|
bge 33f /* another thread already exiting */
|
|
|
|
/*
|
|
* Although not specifically required by the architecture, POWER7
|
|
* preserves the following registers in nap mode, even if an SMT mode
|
|
* switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
|
|
* DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
|
|
*/
|
|
/* Save non-volatile GPRs */
|
|
std r14, VCPU_GPR(r14)(r3)
|
|
std r15, VCPU_GPR(r15)(r3)
|
|
std r16, VCPU_GPR(r16)(r3)
|
|
std r17, VCPU_GPR(r17)(r3)
|
|
std r18, VCPU_GPR(r18)(r3)
|
|
std r19, VCPU_GPR(r19)(r3)
|
|
std r20, VCPU_GPR(r20)(r3)
|
|
std r21, VCPU_GPR(r21)(r3)
|
|
std r22, VCPU_GPR(r22)(r3)
|
|
std r23, VCPU_GPR(r23)(r3)
|
|
std r24, VCPU_GPR(r24)(r3)
|
|
std r25, VCPU_GPR(r25)(r3)
|
|
std r26, VCPU_GPR(r26)(r3)
|
|
std r27, VCPU_GPR(r27)(r3)
|
|
std r28, VCPU_GPR(r28)(r3)
|
|
std r29, VCPU_GPR(r29)(r3)
|
|
std r30, VCPU_GPR(r30)(r3)
|
|
std r31, VCPU_GPR(r31)(r3)
|
|
|
|
/* save FP state */
|
|
bl .kvmppc_save_fp
|
|
|
|
/*
|
|
* Take a nap until a decrementer or external interrupt occurs,
|
|
* with PECE1 (wake on decr) and PECE0 (wake on external) set in LPCR
|
|
*/
|
|
li r0,0x80
|
|
stb r0,PACAPROCSTART(r13)
|
|
mfspr r5,SPRN_LPCR
|
|
ori r5,r5,LPCR_PECE0 | LPCR_PECE1
|
|
mtspr SPRN_LPCR,r5
|
|
isync
|
|
li r0, 0
|
|
std r0, HSTATE_SCRATCH0(r13)
|
|
ptesync
|
|
ld r0, HSTATE_SCRATCH0(r13)
|
|
1: cmpd r0, r0
|
|
bne 1b
|
|
nap
|
|
b .
|
|
|
|
kvm_end_cede:
|
|
/* Woken by external or decrementer interrupt */
|
|
ld r1, HSTATE_HOST_R1(r13)
|
|
ld r2, PACATOC(r13)
|
|
|
|
/* If we're a secondary thread and we got here by an IPI, ack it */
|
|
ld r4,HSTATE_KVM_VCPU(r13)
|
|
lwz r3,VCPU_PTID(r4)
|
|
cmpwi r3,0
|
|
beq 27f
|
|
mfspr r3,SPRN_SRR1
|
|
rlwinm r3,r3,44-31,0x7 /* extract wake reason field */
|
|
cmpwi r3,4 /* was it an external interrupt? */
|
|
bne 27f
|
|
ld r5, HSTATE_XICS_PHYS(r13)
|
|
li r0,0xff
|
|
li r6,XICS_QIRR
|
|
li r7,XICS_XIRR
|
|
lwzcix r8,r5,r7 /* ack the interrupt */
|
|
sync
|
|
stbcix r0,r5,r6 /* clear it */
|
|
stwcix r8,r5,r7 /* EOI it */
|
|
27:
|
|
/* load up FP state */
|
|
bl kvmppc_load_fp
|
|
|
|
/* Load NV GPRS */
|
|
ld r14, VCPU_GPR(r14)(r4)
|
|
ld r15, VCPU_GPR(r15)(r4)
|
|
ld r16, VCPU_GPR(r16)(r4)
|
|
ld r17, VCPU_GPR(r17)(r4)
|
|
ld r18, VCPU_GPR(r18)(r4)
|
|
ld r19, VCPU_GPR(r19)(r4)
|
|
ld r20, VCPU_GPR(r20)(r4)
|
|
ld r21, VCPU_GPR(r21)(r4)
|
|
ld r22, VCPU_GPR(r22)(r4)
|
|
ld r23, VCPU_GPR(r23)(r4)
|
|
ld r24, VCPU_GPR(r24)(r4)
|
|
ld r25, VCPU_GPR(r25)(r4)
|
|
ld r26, VCPU_GPR(r26)(r4)
|
|
ld r27, VCPU_GPR(r27)(r4)
|
|
ld r28, VCPU_GPR(r28)(r4)
|
|
ld r29, VCPU_GPR(r29)(r4)
|
|
ld r30, VCPU_GPR(r30)(r4)
|
|
ld r31, VCPU_GPR(r31)(r4)
|
|
|
|
/* clear our bit in vcore->napping_threads */
|
|
33: ld r5,HSTATE_KVM_VCORE(r13)
|
|
lwz r3,VCPU_PTID(r4)
|
|
li r0,1
|
|
sld r0,r0,r3
|
|
addi r6,r5,VCORE_NAPPING_THREADS
|
|
32: lwarx r7,0,r6
|
|
andc r7,r7,r0
|
|
stwcx. r7,0,r6
|
|
bne 32b
|
|
li r0,0
|
|
stb r0,HSTATE_NAPPING(r13)
|
|
|
|
/* see if any other thread is already exiting */
|
|
lwz r0,VCORE_ENTRY_EXIT(r5)
|
|
cmpwi r0,0x100
|
|
blt kvmppc_cede_reentry /* if not go back to guest */
|
|
|
|
/* some threads are exiting, so go to the guest exit path */
|
|
b hcall_real_fallback
|
|
|
|
/* cede when already previously prodded case */
|
|
1: li r0,0
|
|
stb r0,VCPU_PRODDED(r3)
|
|
sync /* order testing prodded vs. clearing ceded */
|
|
stb r0,VCPU_CEDED(r3)
|
|
li r3,H_SUCCESS
|
|
blr
|
|
|
|
/* we've ceded but we want to give control to the host */
|
|
2: li r3,H_TOO_HARD
|
|
blr
|
|
|
|
secondary_too_late:
|
|
ld r5,HSTATE_KVM_VCORE(r13)
|
|
HMT_LOW
|
|
13: lbz r3,VCORE_IN_GUEST(r5)
|
|
cmpwi r3,0
|
|
bne 13b
|
|
HMT_MEDIUM
|
|
ld r11,PACA_SLBSHADOWPTR(r13)
|
|
|
|
.rept SLB_NUM_BOLTED
|
|
ld r5,SLBSHADOW_SAVEAREA(r11)
|
|
ld r6,SLBSHADOW_SAVEAREA+8(r11)
|
|
andis. r7,r5,SLB_ESID_V@h
|
|
beq 1f
|
|
slbmte r6,r5
|
|
1: addi r11,r11,16
|
|
.endr
|
|
|
|
secondary_nap:
|
|
/* Clear any pending IPI - assume we're a secondary thread */
|
|
ld r5, HSTATE_XICS_PHYS(r13)
|
|
li r7, XICS_XIRR
|
|
lwzcix r3, r5, r7 /* ack any pending interrupt */
|
|
rlwinm. r0, r3, 0, 0xffffff /* any pending? */
|
|
beq 37f
|
|
sync
|
|
li r0, 0xff
|
|
li r6, XICS_QIRR
|
|
stbcix r0, r5, r6 /* clear the IPI */
|
|
stwcix r3, r5, r7 /* EOI it */
|
|
37: sync
|
|
|
|
/* increment the nap count and then go to nap mode */
|
|
ld r4, HSTATE_KVM_VCORE(r13)
|
|
addi r4, r4, VCORE_NAP_COUNT
|
|
lwsync /* make previous updates visible */
|
|
51: lwarx r3, 0, r4
|
|
addi r3, r3, 1
|
|
stwcx. r3, 0, r4
|
|
bne 51b
|
|
|
|
li r3, LPCR_PECE0
|
|
mfspr r4, SPRN_LPCR
|
|
rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
|
|
mtspr SPRN_LPCR, r4
|
|
isync
|
|
li r0, 0
|
|
std r0, HSTATE_SCRATCH0(r13)
|
|
ptesync
|
|
ld r0, HSTATE_SCRATCH0(r13)
|
|
1: cmpd r0, r0
|
|
bne 1b
|
|
nap
|
|
b .
|
|
|
|
/*
|
|
* Save away FP, VMX and VSX registers.
|
|
* r3 = vcpu pointer
|
|
*/
|
|
_GLOBAL(kvmppc_save_fp)
|
|
mfmsr r9
|
|
ori r8,r9,MSR_FP
|
|
#ifdef CONFIG_ALTIVEC
|
|
BEGIN_FTR_SECTION
|
|
oris r8,r8,MSR_VEC@h
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
|
|
#endif
|
|
#ifdef CONFIG_VSX
|
|
BEGIN_FTR_SECTION
|
|
oris r8,r8,MSR_VSX@h
|
|
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
|
|
#endif
|
|
mtmsrd r8
|
|
isync
|
|
#ifdef CONFIG_VSX
|
|
BEGIN_FTR_SECTION
|
|
reg = 0
|
|
.rept 32
|
|
li r6,reg*16+VCPU_VSRS
|
|
STXVD2X(reg,r6,r3)
|
|
reg = reg + 1
|
|
.endr
|
|
FTR_SECTION_ELSE
|
|
#endif
|
|
reg = 0
|
|
.rept 32
|
|
stfd reg,reg*8+VCPU_FPRS(r3)
|
|
reg = reg + 1
|
|
.endr
|
|
#ifdef CONFIG_VSX
|
|
ALT_FTR_SECTION_END_IFSET(CPU_FTR_VSX)
|
|
#endif
|
|
mffs fr0
|
|
stfd fr0,VCPU_FPSCR(r3)
|
|
|
|
#ifdef CONFIG_ALTIVEC
|
|
BEGIN_FTR_SECTION
|
|
reg = 0
|
|
.rept 32
|
|
li r6,reg*16+VCPU_VRS
|
|
stvx reg,r6,r3
|
|
reg = reg + 1
|
|
.endr
|
|
mfvscr vr0
|
|
li r6,VCPU_VSCR
|
|
stvx vr0,r6,r3
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
|
|
#endif
|
|
mfspr r6,SPRN_VRSAVE
|
|
stw r6,VCPU_VRSAVE(r3)
|
|
mtmsrd r9
|
|
isync
|
|
blr
|
|
|
|
/*
|
|
* Load up FP, VMX and VSX registers
|
|
* r4 = vcpu pointer
|
|
*/
|
|
.globl kvmppc_load_fp
|
|
kvmppc_load_fp:
|
|
mfmsr r9
|
|
ori r8,r9,MSR_FP
|
|
#ifdef CONFIG_ALTIVEC
|
|
BEGIN_FTR_SECTION
|
|
oris r8,r8,MSR_VEC@h
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
|
|
#endif
|
|
#ifdef CONFIG_VSX
|
|
BEGIN_FTR_SECTION
|
|
oris r8,r8,MSR_VSX@h
|
|
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
|
|
#endif
|
|
mtmsrd r8
|
|
isync
|
|
lfd fr0,VCPU_FPSCR(r4)
|
|
MTFSF_L(fr0)
|
|
#ifdef CONFIG_VSX
|
|
BEGIN_FTR_SECTION
|
|
reg = 0
|
|
.rept 32
|
|
li r7,reg*16+VCPU_VSRS
|
|
LXVD2X(reg,r7,r4)
|
|
reg = reg + 1
|
|
.endr
|
|
FTR_SECTION_ELSE
|
|
#endif
|
|
reg = 0
|
|
.rept 32
|
|
lfd reg,reg*8+VCPU_FPRS(r4)
|
|
reg = reg + 1
|
|
.endr
|
|
#ifdef CONFIG_VSX
|
|
ALT_FTR_SECTION_END_IFSET(CPU_FTR_VSX)
|
|
#endif
|
|
|
|
#ifdef CONFIG_ALTIVEC
|
|
BEGIN_FTR_SECTION
|
|
li r7,VCPU_VSCR
|
|
lvx vr0,r7,r4
|
|
mtvscr vr0
|
|
reg = 0
|
|
.rept 32
|
|
li r7,reg*16+VCPU_VRS
|
|
lvx reg,r7,r4
|
|
reg = reg + 1
|
|
.endr
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
|
|
#endif
|
|
lwz r7,VCPU_VRSAVE(r4)
|
|
mtspr SPRN_VRSAVE,r7
|
|
blr
|