649 lines
18 KiB
C
649 lines
18 KiB
C
|
/* SPDX-License-Identifier: GPL-2.0 */
|
||
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
||
|
|
|
||
|
|
#include <linux/kvm_host.h>
|
||
|
|
#include "x86.h"
|
||
|
|
#include "kvm_cache_regs.h"
|
||
|
|
#include "kvm_emulate.h"
|
||
|
|
#include "smm.h"
|
||
|
|
#include "cpuid.h"
|
||
|
|
#include "trace.h"
|
||
|
|
|
||
|
|
#define CHECK_SMRAM32_OFFSET(field, offset) \
|
||
|
|
ASSERT_STRUCT_OFFSET(struct kvm_smram_state_32, field, offset - 0xFE00)
|
||
|
|
|
||
|
|
#define CHECK_SMRAM64_OFFSET(field, offset) \
|
||
|
|
ASSERT_STRUCT_OFFSET(struct kvm_smram_state_64, field, offset - 0xFE00)
|
||
|
|
|
||
|
|
static void check_smram_offsets(void)
|
||
|
|
{
|
||
|
|
/* 32 bit SMRAM image */
|
||
|
|
CHECK_SMRAM32_OFFSET(reserved1, 0xFE00);
|
||
|
|
CHECK_SMRAM32_OFFSET(smbase, 0xFEF8);
|
||
|
|
CHECK_SMRAM32_OFFSET(smm_revision, 0xFEFC);
|
||
|
|
CHECK_SMRAM32_OFFSET(io_inst_restart, 0xFF00);
|
||
|
|
CHECK_SMRAM32_OFFSET(auto_hlt_restart, 0xFF02);
|
||
|
|
CHECK_SMRAM32_OFFSET(io_restart_rdi, 0xFF04);
|
||
|
|
CHECK_SMRAM32_OFFSET(io_restart_rcx, 0xFF08);
|
||
|
|
CHECK_SMRAM32_OFFSET(io_restart_rsi, 0xFF0C);
|
||
|
|
CHECK_SMRAM32_OFFSET(io_restart_rip, 0xFF10);
|
||
|
|
CHECK_SMRAM32_OFFSET(cr4, 0xFF14);
|
||
|
|
CHECK_SMRAM32_OFFSET(reserved2, 0xFF18);
|
||
|
|
CHECK_SMRAM32_OFFSET(int_shadow, 0xFF1A);
|
||
|
|
CHECK_SMRAM32_OFFSET(reserved3, 0xFF1B);
|
||
|
|
CHECK_SMRAM32_OFFSET(ds, 0xFF2C);
|
||
|
|
CHECK_SMRAM32_OFFSET(fs, 0xFF38);
|
||
|
|
CHECK_SMRAM32_OFFSET(gs, 0xFF44);
|
||
|
|
CHECK_SMRAM32_OFFSET(idtr, 0xFF50);
|
||
|
|
CHECK_SMRAM32_OFFSET(tr, 0xFF5C);
|
||
|
|
CHECK_SMRAM32_OFFSET(gdtr, 0xFF6C);
|
||
|
|
CHECK_SMRAM32_OFFSET(ldtr, 0xFF78);
|
||
|
|
CHECK_SMRAM32_OFFSET(es, 0xFF84);
|
||
|
|
CHECK_SMRAM32_OFFSET(cs, 0xFF90);
|
||
|
|
CHECK_SMRAM32_OFFSET(ss, 0xFF9C);
|
||
|
|
CHECK_SMRAM32_OFFSET(es_sel, 0xFFA8);
|
||
|
|
CHECK_SMRAM32_OFFSET(cs_sel, 0xFFAC);
|
||
|
|
CHECK_SMRAM32_OFFSET(ss_sel, 0xFFB0);
|
||
|
|
CHECK_SMRAM32_OFFSET(ds_sel, 0xFFB4);
|
||
|
|
CHECK_SMRAM32_OFFSET(fs_sel, 0xFFB8);
|
||
|
|
CHECK_SMRAM32_OFFSET(gs_sel, 0xFFBC);
|
||
|
|
CHECK_SMRAM32_OFFSET(ldtr_sel, 0xFFC0);
|
||
|
|
CHECK_SMRAM32_OFFSET(tr_sel, 0xFFC4);
|
||
|
|
CHECK_SMRAM32_OFFSET(dr7, 0xFFC8);
|
||
|
|
CHECK_SMRAM32_OFFSET(dr6, 0xFFCC);
|
||
|
|
CHECK_SMRAM32_OFFSET(gprs, 0xFFD0);
|
||
|
|
CHECK_SMRAM32_OFFSET(eip, 0xFFF0);
|
||
|
|
CHECK_SMRAM32_OFFSET(eflags, 0xFFF4);
|
||
|
|
CHECK_SMRAM32_OFFSET(cr3, 0xFFF8);
|
||
|
|
CHECK_SMRAM32_OFFSET(cr0, 0xFFFC);
|
||
|
|
|
||
|
|
/* 64 bit SMRAM image */
|
||
|
|
CHECK_SMRAM64_OFFSET(es, 0xFE00);
|
||
|
|
CHECK_SMRAM64_OFFSET(cs, 0xFE10);
|
||
|
|
CHECK_SMRAM64_OFFSET(ss, 0xFE20);
|
||
|
|
CHECK_SMRAM64_OFFSET(ds, 0xFE30);
|
||
|
|
CHECK_SMRAM64_OFFSET(fs, 0xFE40);
|
||
|
|
CHECK_SMRAM64_OFFSET(gs, 0xFE50);
|
||
|
|
CHECK_SMRAM64_OFFSET(gdtr, 0xFE60);
|
||
|
|
CHECK_SMRAM64_OFFSET(ldtr, 0xFE70);
|
||
|
|
CHECK_SMRAM64_OFFSET(idtr, 0xFE80);
|
||
|
|
CHECK_SMRAM64_OFFSET(tr, 0xFE90);
|
||
|
|
CHECK_SMRAM64_OFFSET(io_restart_rip, 0xFEA0);
|
||
|
|
CHECK_SMRAM64_OFFSET(io_restart_rcx, 0xFEA8);
|
||
|
|
CHECK_SMRAM64_OFFSET(io_restart_rsi, 0xFEB0);
|
||
|
|
CHECK_SMRAM64_OFFSET(io_restart_rdi, 0xFEB8);
|
||
|
|
CHECK_SMRAM64_OFFSET(io_restart_dword, 0xFEC0);
|
||
|
|
CHECK_SMRAM64_OFFSET(reserved1, 0xFEC4);
|
||
|
|
CHECK_SMRAM64_OFFSET(io_inst_restart, 0xFEC8);
|
||
|
|
CHECK_SMRAM64_OFFSET(auto_hlt_restart, 0xFEC9);
|
||
|
|
CHECK_SMRAM64_OFFSET(amd_nmi_mask, 0xFECA);
|
||
|
|
CHECK_SMRAM64_OFFSET(int_shadow, 0xFECB);
|
||
|
|
CHECK_SMRAM64_OFFSET(reserved2, 0xFECC);
|
||
|
|
CHECK_SMRAM64_OFFSET(efer, 0xFED0);
|
||
|
|
CHECK_SMRAM64_OFFSET(svm_guest_flag, 0xFED8);
|
||
|
|
CHECK_SMRAM64_OFFSET(svm_guest_vmcb_gpa, 0xFEE0);
|
||
|
|
CHECK_SMRAM64_OFFSET(svm_guest_virtual_int, 0xFEE8);
|
||
|
|
CHECK_SMRAM64_OFFSET(reserved3, 0xFEF0);
|
||
|
|
CHECK_SMRAM64_OFFSET(smm_revison, 0xFEFC);
|
||
|
|
CHECK_SMRAM64_OFFSET(smbase, 0xFF00);
|
||
|
|
CHECK_SMRAM64_OFFSET(reserved4, 0xFF04);
|
||
|
|
CHECK_SMRAM64_OFFSET(ssp, 0xFF18);
|
||
|
|
CHECK_SMRAM64_OFFSET(svm_guest_pat, 0xFF20);
|
||
|
|
CHECK_SMRAM64_OFFSET(svm_host_efer, 0xFF28);
|
||
|
|
CHECK_SMRAM64_OFFSET(svm_host_cr4, 0xFF30);
|
||
|
|
CHECK_SMRAM64_OFFSET(svm_host_cr3, 0xFF38);
|
||
|
|
CHECK_SMRAM64_OFFSET(svm_host_cr0, 0xFF40);
|
||
|
|
CHECK_SMRAM64_OFFSET(cr4, 0xFF48);
|
||
|
|
CHECK_SMRAM64_OFFSET(cr3, 0xFF50);
|
||
|
|
CHECK_SMRAM64_OFFSET(cr0, 0xFF58);
|
||
|
|
CHECK_SMRAM64_OFFSET(dr7, 0xFF60);
|
||
|
|
CHECK_SMRAM64_OFFSET(dr6, 0xFF68);
|
||
|
|
CHECK_SMRAM64_OFFSET(rflags, 0xFF70);
|
||
|
|
CHECK_SMRAM64_OFFSET(rip, 0xFF78);
|
||
|
|
CHECK_SMRAM64_OFFSET(gprs, 0xFF80);
|
||
|
|
|
||
|
|
BUILD_BUG_ON(sizeof(union kvm_smram) != 512);
|
||
|
|
}
|
||
|
|
|
||
|
|
#undef CHECK_SMRAM64_OFFSET
|
||
|
|
#undef CHECK_SMRAM32_OFFSET
|
||
|
|
|
||
|
|
|
||
|
|
void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm)
|
||
|
|
{
|
||
|
|
trace_kvm_smm_transition(vcpu->vcpu_id, vcpu->arch.smbase, entering_smm);
|
||
|
|
|
||
|
|
if (entering_smm) {
|
||
|
|
vcpu->arch.hflags |= HF_SMM_MASK;
|
||
|
|
} else {
|
||
|
|
vcpu->arch.hflags &= ~(HF_SMM_MASK | HF_SMM_INSIDE_NMI_MASK);
|
||
|
|
|
||
|
|
/* Process a latched INIT or SMI, if any. */
|
||
|
|
kvm_make_request(KVM_REQ_EVENT, vcpu);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Even if KVM_SET_SREGS2 loaded PDPTRs out of band,
|
||
|
|
* on SMM exit we still need to reload them from
|
||
|
|
* guest memory
|
||
|
|
*/
|
||
|
|
vcpu->arch.pdptrs_from_userspace = false;
|
||
|
|
}
|
||
|
|
|
||
|
|
kvm_mmu_reset_context(vcpu);
|
||
|
|
}
|
||
|
|
|
||
|
|
void process_smi(struct kvm_vcpu *vcpu)
|
||
|
|
{
|
||
|
|
vcpu->arch.smi_pending = true;
|
||
|
|
kvm_make_request(KVM_REQ_EVENT, vcpu);
|
||
|
|
}
|
||
|
|
|
||
|
|
static u32 enter_smm_get_segment_flags(struct kvm_segment *seg)
|
||
|
|
{
|
||
|
|
u32 flags = 0;
|
||
|
|
flags |= seg->g << 23;
|
||
|
|
flags |= seg->db << 22;
|
||
|
|
flags |= seg->l << 21;
|
||
|
|
flags |= seg->avl << 20;
|
||
|
|
flags |= seg->present << 15;
|
||
|
|
flags |= seg->dpl << 13;
|
||
|
|
flags |= seg->s << 12;
|
||
|
|
flags |= seg->type << 8;
|
||
|
|
return flags;
|
||
|
|
}
|
||
|
|
|
||
|
|
static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu,
|
||
|
|
struct kvm_smm_seg_state_32 *state,
|
||
|
|
u32 *selector, int n)
|
||
|
|
{
|
||
|
|
struct kvm_segment seg;
|
||
|
|
|
||
|
|
kvm_get_segment(vcpu, &seg, n);
|
||
|
|
*selector = seg.selector;
|
||
|
|
state->base = seg.base;
|
||
|
|
state->limit = seg.limit;
|
||
|
|
state->flags = enter_smm_get_segment_flags(&seg);
|
||
|
|
}
|
||
|
|
|
||
|
|
#ifdef CONFIG_X86_64
|
||
|
|
static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu,
|
||
|
|
struct kvm_smm_seg_state_64 *state,
|
||
|
|
int n)
|
||
|
|
{
|
||
|
|
struct kvm_segment seg;
|
||
|
|
|
||
|
|
kvm_get_segment(vcpu, &seg, n);
|
||
|
|
state->selector = seg.selector;
|
||
|
|
state->attributes = enter_smm_get_segment_flags(&seg) >> 8;
|
||
|
|
state->limit = seg.limit;
|
||
|
|
state->base = seg.base;
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
static void enter_smm_save_state_32(struct kvm_vcpu *vcpu,
|
||
|
|
struct kvm_smram_state_32 *smram)
|
||
|
|
{
|
||
|
|
struct desc_ptr dt;
|
||
|
|
unsigned long val;
|
||
|
|
int i;
|
||
|
|
|
||
|
|
smram->cr0 = kvm_read_cr0(vcpu);
|
||
|
|
smram->cr3 = kvm_read_cr3(vcpu);
|
||
|
|
smram->eflags = kvm_get_rflags(vcpu);
|
||
|
|
smram->eip = kvm_rip_read(vcpu);
|
||
|
|
|
||
|
|
for (i = 0; i < 8; i++)
|
||
|
|
smram->gprs[i] = kvm_register_read_raw(vcpu, i);
|
||
|
|
|
||
|
|
kvm_get_dr(vcpu, 6, &val);
|
||
|
|
smram->dr6 = (u32)val;
|
||
|
|
kvm_get_dr(vcpu, 7, &val);
|
||
|
|
smram->dr7 = (u32)val;
|
||
|
|
|
||
|
|
enter_smm_save_seg_32(vcpu, &smram->tr, &smram->tr_sel, VCPU_SREG_TR);
|
||
|
|
enter_smm_save_seg_32(vcpu, &smram->ldtr, &smram->ldtr_sel, VCPU_SREG_LDTR);
|
||
|
|
|
||
|
|
static_call(kvm_x86_get_gdt)(vcpu, &dt);
|
||
|
|
smram->gdtr.base = dt.address;
|
||
|
|
smram->gdtr.limit = dt.size;
|
||
|
|
|
||
|
|
static_call(kvm_x86_get_idt)(vcpu, &dt);
|
||
|
|
smram->idtr.base = dt.address;
|
||
|
|
smram->idtr.limit = dt.size;
|
||
|
|
|
||
|
|
enter_smm_save_seg_32(vcpu, &smram->es, &smram->es_sel, VCPU_SREG_ES);
|
||
|
|
enter_smm_save_seg_32(vcpu, &smram->cs, &smram->cs_sel, VCPU_SREG_CS);
|
||
|
|
enter_smm_save_seg_32(vcpu, &smram->ss, &smram->ss_sel, VCPU_SREG_SS);
|
||
|
|
|
||
|
|
enter_smm_save_seg_32(vcpu, &smram->ds, &smram->ds_sel, VCPU_SREG_DS);
|
||
|
|
enter_smm_save_seg_32(vcpu, &smram->fs, &smram->fs_sel, VCPU_SREG_FS);
|
||
|
|
enter_smm_save_seg_32(vcpu, &smram->gs, &smram->gs_sel, VCPU_SREG_GS);
|
||
|
|
|
||
|
|
smram->cr4 = kvm_read_cr4(vcpu);
|
||
|
|
smram->smm_revision = 0x00020000;
|
||
|
|
smram->smbase = vcpu->arch.smbase;
|
||
|
|
|
||
|
|
smram->int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
|
||
|
|
}
|
||
|
|
|
||
|
|
#ifdef CONFIG_X86_64
|
||
|
|
static void enter_smm_save_state_64(struct kvm_vcpu *vcpu,
|
||
|
|
struct kvm_smram_state_64 *smram)
|
||
|
|
{
|
||
|
|
struct desc_ptr dt;
|
||
|
|
unsigned long val;
|
||
|
|
int i;
|
||
|
|
|
||
|
|
for (i = 0; i < 16; i++)
|
||
|
|
smram->gprs[15 - i] = kvm_register_read_raw(vcpu, i);
|
||
|
|
|
||
|
|
smram->rip = kvm_rip_read(vcpu);
|
||
|
|
smram->rflags = kvm_get_rflags(vcpu);
|
||
|
|
|
||
|
|
|
||
|
|
kvm_get_dr(vcpu, 6, &val);
|
||
|
|
smram->dr6 = val;
|
||
|
|
kvm_get_dr(vcpu, 7, &val);
|
||
|
|
smram->dr7 = val;
|
||
|
|
|
||
|
|
smram->cr0 = kvm_read_cr0(vcpu);
|
||
|
|
smram->cr3 = kvm_read_cr3(vcpu);
|
||
|
|
smram->cr4 = kvm_read_cr4(vcpu);
|
||
|
|
|
||
|
|
smram->smbase = vcpu->arch.smbase;
|
||
|
|
smram->smm_revison = 0x00020064;
|
||
|
|
|
||
|
|
smram->efer = vcpu->arch.efer;
|
||
|
|
|
||
|
|
enter_smm_save_seg_64(vcpu, &smram->tr, VCPU_SREG_TR);
|
||
|
|
|
||
|
|
static_call(kvm_x86_get_idt)(vcpu, &dt);
|
||
|
|
smram->idtr.limit = dt.size;
|
||
|
|
smram->idtr.base = dt.address;
|
||
|
|
|
||
|
|
enter_smm_save_seg_64(vcpu, &smram->ldtr, VCPU_SREG_LDTR);
|
||
|
|
|
||
|
|
static_call(kvm_x86_get_gdt)(vcpu, &dt);
|
||
|
|
smram->gdtr.limit = dt.size;
|
||
|
|
smram->gdtr.base = dt.address;
|
||
|
|
|
||
|
|
enter_smm_save_seg_64(vcpu, &smram->es, VCPU_SREG_ES);
|
||
|
|
enter_smm_save_seg_64(vcpu, &smram->cs, VCPU_SREG_CS);
|
||
|
|
enter_smm_save_seg_64(vcpu, &smram->ss, VCPU_SREG_SS);
|
||
|
|
enter_smm_save_seg_64(vcpu, &smram->ds, VCPU_SREG_DS);
|
||
|
|
enter_smm_save_seg_64(vcpu, &smram->fs, VCPU_SREG_FS);
|
||
|
|
enter_smm_save_seg_64(vcpu, &smram->gs, VCPU_SREG_GS);
|
||
|
|
|
||
|
|
smram->int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
void enter_smm(struct kvm_vcpu *vcpu)
|
||
|
|
{
|
||
|
|
struct kvm_segment cs, ds;
|
||
|
|
struct desc_ptr dt;
|
||
|
|
unsigned long cr0;
|
||
|
|
union kvm_smram smram;
|
||
|
|
|
||
|
|
check_smram_offsets();
|
||
|
|
|
||
|
|
memset(smram.bytes, 0, sizeof(smram.bytes));
|
||
|
|
|
||
|
|
#ifdef CONFIG_X86_64
|
||
|
|
if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
|
||
|
|
enter_smm_save_state_64(vcpu, &smram.smram64);
|
||
|
|
else
|
||
|
|
#endif
|
||
|
|
enter_smm_save_state_32(vcpu, &smram.smram32);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Give enter_smm() a chance to make ISA-specific changes to the vCPU
|
||
|
|
* state (e.g. leave guest mode) after we've saved the state into the
|
||
|
|
* SMM state-save area.
|
||
|
|
*
|
||
|
|
* Kill the VM in the unlikely case of failure, because the VM
|
||
|
|
* can be in undefined state in this case.
|
||
|
|
*/
|
||
|
|
if (static_call(kvm_x86_enter_smm)(vcpu, &smram))
|
||
|
|
goto error;
|
||
|
|
|
||
|
|
kvm_smm_changed(vcpu, true);
|
||
|
|
|
||
|
|
if (kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, &smram, sizeof(smram)))
|
||
|
|
goto error;
|
||
|
|
|
||
|
|
if (static_call(kvm_x86_get_nmi_mask)(vcpu))
|
||
|
|
vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
|
||
|
|
else
|
||
|
|
static_call(kvm_x86_set_nmi_mask)(vcpu, true);
|
||
|
|
|
||
|
|
kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
|
||
|
|
kvm_rip_write(vcpu, 0x8000);
|
||
|
|
|
||
|
|
static_call(kvm_x86_set_interrupt_shadow)(vcpu, 0);
|
||
|
|
|
||
|
|
cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
|
||
|
|
static_call(kvm_x86_set_cr0)(vcpu, cr0);
|
||
|
|
vcpu->arch.cr0 = cr0;
|
||
|
|
|
||
|
|
static_call(kvm_x86_set_cr4)(vcpu, 0);
|
||
|
|
|
||
|
|
/* Undocumented: IDT limit is set to zero on entry to SMM. */
|
||
|
|
dt.address = dt.size = 0;
|
||
|
|
static_call(kvm_x86_set_idt)(vcpu, &dt);
|
||
|
|
|
||
|
|
if (WARN_ON_ONCE(kvm_set_dr(vcpu, 7, DR7_FIXED_1)))
|
||
|
|
goto error;
|
||
|
|
|
||
|
|
cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
|
||
|
|
cs.base = vcpu->arch.smbase;
|
||
|
|
|
||
|
|
ds.selector = 0;
|
||
|
|
ds.base = 0;
|
||
|
|
|
||
|
|
cs.limit = ds.limit = 0xffffffff;
|
||
|
|
cs.type = ds.type = 0x3;
|
||
|
|
cs.dpl = ds.dpl = 0;
|
||
|
|
cs.db = ds.db = 0;
|
||
|
|
cs.s = ds.s = 1;
|
||
|
|
cs.l = ds.l = 0;
|
||
|
|
cs.g = ds.g = 1;
|
||
|
|
cs.avl = ds.avl = 0;
|
||
|
|
cs.present = ds.present = 1;
|
||
|
|
cs.unusable = ds.unusable = 0;
|
||
|
|
cs.padding = ds.padding = 0;
|
||
|
|
|
||
|
|
kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
|
||
|
|
kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);
|
||
|
|
kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);
|
||
|
|
kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);
|
||
|
|
kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);
|
||
|
|
kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);
|
||
|
|
|
||
|
|
#ifdef CONFIG_X86_64
|
||
|
|
if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
|
||
|
|
if (static_call(kvm_x86_set_efer)(vcpu, 0))
|
||
|
|
goto error;
|
||
|
|
#endif
|
||
|
|
|
||
|
|
kvm_update_cpuid_runtime(vcpu);
|
||
|
|
kvm_mmu_reset_context(vcpu);
|
||
|
|
return;
|
||
|
|
error:
|
||
|
|
kvm_vm_dead(vcpu->kvm);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void rsm_set_desc_flags(struct kvm_segment *desc, u32 flags)
|
||
|
|
{
|
||
|
|
desc->g = (flags >> 23) & 1;
|
||
|
|
desc->db = (flags >> 22) & 1;
|
||
|
|
desc->l = (flags >> 21) & 1;
|
||
|
|
desc->avl = (flags >> 20) & 1;
|
||
|
|
desc->present = (flags >> 15) & 1;
|
||
|
|
desc->dpl = (flags >> 13) & 3;
|
||
|
|
desc->s = (flags >> 12) & 1;
|
||
|
|
desc->type = (flags >> 8) & 15;
|
||
|
|
|
||
|
|
desc->unusable = !desc->present;
|
||
|
|
desc->padding = 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
static int rsm_load_seg_32(struct kvm_vcpu *vcpu,
|
||
|
|
const struct kvm_smm_seg_state_32 *state,
|
||
|
|
u16 selector, int n)
|
||
|
|
{
|
||
|
|
struct kvm_segment desc;
|
||
|
|
|
||
|
|
desc.selector = selector;
|
||
|
|
desc.base = state->base;
|
||
|
|
desc.limit = state->limit;
|
||
|
|
rsm_set_desc_flags(&desc, state->flags);
|
||
|
|
kvm_set_segment(vcpu, &desc, n);
|
||
|
|
return X86EMUL_CONTINUE;
|
||
|
|
}
|
||
|
|
|
||
|
|
#ifdef CONFIG_X86_64
|
||
|
|
|
||
|
|
static int rsm_load_seg_64(struct kvm_vcpu *vcpu,
|
||
|
|
const struct kvm_smm_seg_state_64 *state,
|
||
|
|
int n)
|
||
|
|
{
|
||
|
|
struct kvm_segment desc;
|
||
|
|
|
||
|
|
desc.selector = state->selector;
|
||
|
|
rsm_set_desc_flags(&desc, state->attributes << 8);
|
||
|
|
desc.limit = state->limit;
|
||
|
|
desc.base = state->base;
|
||
|
|
kvm_set_segment(vcpu, &desc, n);
|
||
|
|
return X86EMUL_CONTINUE;
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
static int rsm_enter_protected_mode(struct kvm_vcpu *vcpu,
|
||
|
|
u64 cr0, u64 cr3, u64 cr4)
|
||
|
|
{
|
||
|
|
int bad;
|
||
|
|
u64 pcid;
|
||
|
|
|
||
|
|
/* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
|
||
|
|
pcid = 0;
|
||
|
|
if (cr4 & X86_CR4_PCIDE) {
|
||
|
|
pcid = cr3 & 0xfff;
|
||
|
|
cr3 &= ~0xfff;
|
||
|
|
}
|
||
|
|
|
||
|
|
bad = kvm_set_cr3(vcpu, cr3);
|
||
|
|
if (bad)
|
||
|
|
return X86EMUL_UNHANDLEABLE;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* First enable PAE, long mode needs it before CR0.PG = 1 is set.
|
||
|
|
* Then enable protected mode. However, PCID cannot be enabled
|
||
|
|
* if EFER.LMA=0, so set it separately.
|
||
|
|
*/
|
||
|
|
bad = kvm_set_cr4(vcpu, cr4 & ~X86_CR4_PCIDE);
|
||
|
|
if (bad)
|
||
|
|
return X86EMUL_UNHANDLEABLE;
|
||
|
|
|
||
|
|
bad = kvm_set_cr0(vcpu, cr0);
|
||
|
|
if (bad)
|
||
|
|
return X86EMUL_UNHANDLEABLE;
|
||
|
|
|
||
|
|
if (cr4 & X86_CR4_PCIDE) {
|
||
|
|
bad = kvm_set_cr4(vcpu, cr4);
|
||
|
|
if (bad)
|
||
|
|
return X86EMUL_UNHANDLEABLE;
|
||
|
|
if (pcid) {
|
||
|
|
bad = kvm_set_cr3(vcpu, cr3 | pcid);
|
||
|
|
if (bad)
|
||
|
|
return X86EMUL_UNHANDLEABLE;
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
return X86EMUL_CONTINUE;
|
||
|
|
}
|
||
|
|
|
||
|
|
static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
|
||
|
|
const struct kvm_smram_state_32 *smstate)
|
||
|
|
{
|
||
|
|
struct kvm_vcpu *vcpu = ctxt->vcpu;
|
||
|
|
struct desc_ptr dt;
|
||
|
|
int i, r;
|
||
|
|
|
||
|
|
ctxt->eflags = smstate->eflags | X86_EFLAGS_FIXED;
|
||
|
|
ctxt->_eip = smstate->eip;
|
||
|
|
|
||
|
|
for (i = 0; i < 8; i++)
|
||
|
|
*reg_write(ctxt, i) = smstate->gprs[i];
|
||
|
|
|
||
|
|
if (kvm_set_dr(vcpu, 6, smstate->dr6))
|
||
|
|
return X86EMUL_UNHANDLEABLE;
|
||
|
|
if (kvm_set_dr(vcpu, 7, smstate->dr7))
|
||
|
|
return X86EMUL_UNHANDLEABLE;
|
||
|
|
|
||
|
|
rsm_load_seg_32(vcpu, &smstate->tr, smstate->tr_sel, VCPU_SREG_TR);
|
||
|
|
rsm_load_seg_32(vcpu, &smstate->ldtr, smstate->ldtr_sel, VCPU_SREG_LDTR);
|
||
|
|
|
||
|
|
dt.address = smstate->gdtr.base;
|
||
|
|
dt.size = smstate->gdtr.limit;
|
||
|
|
static_call(kvm_x86_set_gdt)(vcpu, &dt);
|
||
|
|
|
||
|
|
dt.address = smstate->idtr.base;
|
||
|
|
dt.size = smstate->idtr.limit;
|
||
|
|
static_call(kvm_x86_set_idt)(vcpu, &dt);
|
||
|
|
|
||
|
|
rsm_load_seg_32(vcpu, &smstate->es, smstate->es_sel, VCPU_SREG_ES);
|
||
|
|
rsm_load_seg_32(vcpu, &smstate->cs, smstate->cs_sel, VCPU_SREG_CS);
|
||
|
|
rsm_load_seg_32(vcpu, &smstate->ss, smstate->ss_sel, VCPU_SREG_SS);
|
||
|
|
|
||
|
|
rsm_load_seg_32(vcpu, &smstate->ds, smstate->ds_sel, VCPU_SREG_DS);
|
||
|
|
rsm_load_seg_32(vcpu, &smstate->fs, smstate->fs_sel, VCPU_SREG_FS);
|
||
|
|
rsm_load_seg_32(vcpu, &smstate->gs, smstate->gs_sel, VCPU_SREG_GS);
|
||
|
|
|
||
|
|
vcpu->arch.smbase = smstate->smbase;
|
||
|
|
|
||
|
|
r = rsm_enter_protected_mode(vcpu, smstate->cr0,
|
||
|
|
smstate->cr3, smstate->cr4);
|
||
|
|
|
||
|
|
if (r != X86EMUL_CONTINUE)
|
||
|
|
return r;
|
||
|
|
|
||
|
|
static_call(kvm_x86_set_interrupt_shadow)(vcpu, 0);
|
||
|
|
ctxt->interruptibility = (u8)smstate->int_shadow;
|
||
|
|
|
||
|
|
return r;
|
||
|
|
}
|
||
|
|
|
||
|
|
#ifdef CONFIG_X86_64
|
||
|
|
static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
|
||
|
|
const struct kvm_smram_state_64 *smstate)
|
||
|
|
{
|
||
|
|
struct kvm_vcpu *vcpu = ctxt->vcpu;
|
||
|
|
struct desc_ptr dt;
|
||
|
|
int i, r;
|
||
|
|
|
||
|
|
for (i = 0; i < 16; i++)
|
||
|
|
*reg_write(ctxt, i) = smstate->gprs[15 - i];
|
||
|
|
|
||
|
|
ctxt->_eip = smstate->rip;
|
||
|
|
ctxt->eflags = smstate->rflags | X86_EFLAGS_FIXED;
|
||
|
|
|
||
|
|
if (kvm_set_dr(vcpu, 6, smstate->dr6))
|
||
|
|
return X86EMUL_UNHANDLEABLE;
|
||
|
|
if (kvm_set_dr(vcpu, 7, smstate->dr7))
|
||
|
|
return X86EMUL_UNHANDLEABLE;
|
||
|
|
|
||
|
|
vcpu->arch.smbase = smstate->smbase;
|
||
|
|
|
||
|
|
if (kvm_set_msr(vcpu, MSR_EFER, smstate->efer & ~EFER_LMA))
|
||
|
|
return X86EMUL_UNHANDLEABLE;
|
||
|
|
|
||
|
|
rsm_load_seg_64(vcpu, &smstate->tr, VCPU_SREG_TR);
|
||
|
|
|
||
|
|
dt.size = smstate->idtr.limit;
|
||
|
|
dt.address = smstate->idtr.base;
|
||
|
|
static_call(kvm_x86_set_idt)(vcpu, &dt);
|
||
|
|
|
||
|
|
rsm_load_seg_64(vcpu, &smstate->ldtr, VCPU_SREG_LDTR);
|
||
|
|
|
||
|
|
dt.size = smstate->gdtr.limit;
|
||
|
|
dt.address = smstate->gdtr.base;
|
||
|
|
static_call(kvm_x86_set_gdt)(vcpu, &dt);
|
||
|
|
|
||
|
|
r = rsm_enter_protected_mode(vcpu, smstate->cr0, smstate->cr3, smstate->cr4);
|
||
|
|
if (r != X86EMUL_CONTINUE)
|
||
|
|
return r;
|
||
|
|
|
||
|
|
rsm_load_seg_64(vcpu, &smstate->es, VCPU_SREG_ES);
|
||
|
|
rsm_load_seg_64(vcpu, &smstate->cs, VCPU_SREG_CS);
|
||
|
|
rsm_load_seg_64(vcpu, &smstate->ss, VCPU_SREG_SS);
|
||
|
|
rsm_load_seg_64(vcpu, &smstate->ds, VCPU_SREG_DS);
|
||
|
|
rsm_load_seg_64(vcpu, &smstate->fs, VCPU_SREG_FS);
|
||
|
|
rsm_load_seg_64(vcpu, &smstate->gs, VCPU_SREG_GS);
|
||
|
|
|
||
|
|
static_call(kvm_x86_set_interrupt_shadow)(vcpu, 0);
|
||
|
|
ctxt->interruptibility = (u8)smstate->int_shadow;
|
||
|
|
|
||
|
|
return X86EMUL_CONTINUE;
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
int emulator_leave_smm(struct x86_emulate_ctxt *ctxt)
|
||
|
|
{
|
||
|
|
struct kvm_vcpu *vcpu = ctxt->vcpu;
|
||
|
|
unsigned long cr0;
|
||
|
|
union kvm_smram smram;
|
||
|
|
u64 smbase;
|
||
|
|
int ret;
|
||
|
|
|
||
|
|
smbase = vcpu->arch.smbase;
|
||
|
|
|
||
|
|
ret = kvm_vcpu_read_guest(vcpu, smbase + 0xfe00, smram.bytes, sizeof(smram));
|
||
|
|
if (ret < 0)
|
||
|
|
return X86EMUL_UNHANDLEABLE;
|
||
|
|
|
||
|
|
if ((vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK) == 0)
|
||
|
|
static_call(kvm_x86_set_nmi_mask)(vcpu, false);
|
||
|
|
|
||
|
|
kvm_smm_changed(vcpu, false);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Get back to real mode, to prepare a safe state in which to load
|
||
|
|
* CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
|
||
|
|
* supports long mode.
|
||
|
|
*/
|
||
|
|
#ifdef CONFIG_X86_64
|
||
|
|
if (guest_cpuid_has(vcpu, X86_FEATURE_LM)) {
|
||
|
|
struct kvm_segment cs_desc;
|
||
|
|
unsigned long cr4;
|
||
|
|
|
||
|
|
/* Zero CR4.PCIDE before CR0.PG. */
|
||
|
|
cr4 = kvm_read_cr4(vcpu);
|
||
|
|
if (cr4 & X86_CR4_PCIDE)
|
||
|
|
kvm_set_cr4(vcpu, cr4 & ~X86_CR4_PCIDE);
|
||
|
|
|
||
|
|
/* A 32-bit code segment is required to clear EFER.LMA. */
|
||
|
|
memset(&cs_desc, 0, sizeof(cs_desc));
|
||
|
|
cs_desc.type = 0xb;
|
||
|
|
cs_desc.s = cs_desc.g = cs_desc.present = 1;
|
||
|
|
kvm_set_segment(vcpu, &cs_desc, VCPU_SREG_CS);
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
/* For the 64-bit case, this will clear EFER.LMA. */
|
||
|
|
cr0 = kvm_read_cr0(vcpu);
|
||
|
|
if (cr0 & X86_CR0_PE)
|
||
|
|
kvm_set_cr0(vcpu, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
|
||
|
|
|
||
|
|
#ifdef CONFIG_X86_64
|
||
|
|
if (guest_cpuid_has(vcpu, X86_FEATURE_LM)) {
|
||
|
|
unsigned long cr4, efer;
|
||
|
|
|
||
|
|
/* Clear CR4.PAE before clearing EFER.LME. */
|
||
|
|
cr4 = kvm_read_cr4(vcpu);
|
||
|
|
if (cr4 & X86_CR4_PAE)
|
||
|
|
kvm_set_cr4(vcpu, cr4 & ~X86_CR4_PAE);
|
||
|
|
|
||
|
|
/* And finally go back to 32-bit mode. */
|
||
|
|
efer = 0;
|
||
|
|
kvm_set_msr(vcpu, MSR_EFER, efer);
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Give leave_smm() a chance to make ISA-specific changes to the vCPU
|
||
|
|
* state (e.g. enter guest mode) before loading state from the SMM
|
||
|
|
* state-save area.
|
||
|
|
*/
|
||
|
|
if (static_call(kvm_x86_leave_smm)(vcpu, &smram))
|
||
|
|
return X86EMUL_UNHANDLEABLE;
|
||
|
|
|
||
|
|
#ifdef CONFIG_X86_64
|
||
|
|
if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
|
||
|
|
return rsm_load_state_64(ctxt, &smram.smram64);
|
||
|
|
else
|
||
|
|
#endif
|
||
|
|
return rsm_load_state_32(ctxt, &smram.smram32);
|
||
|
|
}
|