introcore.c

Go to the documentation of this file.

/*
 * Copyright (c) 2020 Bitdefender
 * SPDX-License-Identifier: Apache-2.0
 */
#include "introcore.h"
#include "debugger.h"
#include "guests.h"
#include "introapi.h"
#include "introcpu.h"
#include "gpacache.h"
#include "winprocess.h"
#include "lixprocess.h"

#ifdef INT_COMPILER_MSVC
#    include "../../autogen/ver.h"
#endif // INT_COMPILER_MSVC

void *gLock = NULL;

const INT_VERSION_INFO IntHviVersion =
{
    .VersionInfo =
    {
        .Build = INTRO_VERSION_BUILDNUMBER & 0xFFFF,
        .Revision = INTRO_VERSION_REVISION,
        .Minor = INTRO_VERSION_MINOR,
        .Major = INTRO_VERSION_MAJOR
    }
};

INTRO_ERROR_CONTEXT gErrorContext = { 0 };

typedef struct _MULTI_PAGE_MAP
{
    LIST_ENTRY  Link;       
    void        *HostPtr;   
    void        *OrigAlloc; 
    QWORD       Gva;        
    DWORD       Length;     
} MULTI_PAGE_MAP, *PMULTI_PAGE_MAP;

LIST_HEAD gMultiPageMaps = LIST_HEAD_INIT(gMultiPageMaps);

BOOLEAN gAbortLoad = FALSE;

extern BOOLEAN gSse42Supported;

extern BOOLEAN gInsideDebugger;

const QWORD gByteMaskToBitMask[256] =
{
    0x0000000000000000, 0x00000000000000ff, 0x000000000000ff00, 0x000000000000ffff,
    0x0000000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0x0000000000ffffff,
    0x00000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0x00000000ff00ffff,
    0x00000000ffff0000, 0x00000000ffff00ff, 0x00000000ffffff00, 0x00000000ffffffff,
    0x000000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0x000000ff0000ffff,
    0x000000ff00ff0000, 0x000000ff00ff00ff, 0x000000ff00ffff00, 0x000000ff00ffffff,
    0x000000ffff000000, 0x000000ffff0000ff, 0x000000ffff00ff00, 0x000000ffff00ffff,
    0x000000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0x000000ffffffffff,
    0x0000ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0x0000ff000000ffff,
    0x0000ff0000ff0000, 0x0000ff0000ff00ff, 0x0000ff0000ffff00, 0x0000ff0000ffffff,
    0x0000ff00ff000000, 0x0000ff00ff0000ff, 0x0000ff00ff00ff00, 0x0000ff00ff00ffff,
    0x0000ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0x0000ff00ffffffff,
    0x0000ffff00000000, 0x0000ffff000000ff, 0x0000ffff0000ff00, 0x0000ffff0000ffff,
    0x0000ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0x0000ffff00ffffff,
    0x0000ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0x0000ffffff00ffff,
    0x0000ffffffff0000, 0x0000ffffffff00ff, 0x0000ffffffffff00, 0x0000ffffffffffff,
    0x00ff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0x00ff00000000ffff,
    0x00ff000000ff0000, 0x00ff000000ff00ff, 0x00ff000000ffff00, 0x00ff000000ffffff,
    0x00ff0000ff000000, 0x00ff0000ff0000ff, 0x00ff0000ff00ff00, 0x00ff0000ff00ffff,
    0x00ff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0x00ff0000ffffffff,
    0x00ff00ff00000000, 0x00ff00ff000000ff, 0x00ff00ff0000ff00, 0x00ff00ff0000ffff,
    0x00ff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0x00ff00ff00ffffff,
    0x00ff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0x00ff00ffff00ffff,
    0x00ff00ffffff0000, 0x00ff00ffffff00ff, 0x00ff00ffffffff00, 0x00ff00ffffffffff,
    0x00ffff0000000000, 0x00ffff00000000ff, 0x00ffff000000ff00, 0x00ffff000000ffff,
    0x00ffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0x00ffff0000ffffff,
    0x00ffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0x00ffff00ff00ffff,
    0x00ffff00ffff0000, 0x00ffff00ffff00ff, 0x00ffff00ffffff00, 0x00ffff00ffffffff,
    0x00ffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0x00ffffff0000ffff,
    0x00ffffff00ff0000, 0x00ffffff00ff00ff, 0x00ffffff00ffff00, 0x00ffffff00ffffff,
    0x00ffffffff000000, 0x00ffffffff0000ff, 0x00ffffffff00ff00, 0x00ffffffff00ffff,
    0x00ffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0x00ffffffffffffff,
    0xff00000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0xff0000000000ffff,
    0xff00000000ff0000, 0xff00000000ff00ff, 0xff00000000ffff00, 0xff00000000ffffff,
    0xff000000ff000000, 0xff000000ff0000ff, 0xff000000ff00ff00, 0xff000000ff00ffff,
    0xff000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0xff000000ffffffff,
    0xff0000ff00000000, 0xff0000ff000000ff, 0xff0000ff0000ff00, 0xff0000ff0000ffff,
    0xff0000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0xff0000ff00ffffff,
    0xff0000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0xff0000ffff00ffff,
    0xff0000ffffff0000, 0xff0000ffffff00ff, 0xff0000ffffffff00, 0xff0000ffffffffff,
    0xff00ff0000000000, 0xff00ff00000000ff, 0xff00ff000000ff00, 0xff00ff000000ffff,
    0xff00ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0xff00ff0000ffffff,
    0xff00ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0xff00ff00ff00ffff,
    0xff00ff00ffff0000, 0xff00ff00ffff00ff, 0xff00ff00ffffff00, 0xff00ff00ffffffff,
    0xff00ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0xff00ffff0000ffff,
    0xff00ffff00ff0000, 0xff00ffff00ff00ff, 0xff00ffff00ffff00, 0xff00ffff00ffffff,
    0xff00ffffff000000, 0xff00ffffff0000ff, 0xff00ffffff00ff00, 0xff00ffffff00ffff,
    0xff00ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0xff00ffffffffffff,
    0xffff000000000000, 0xffff0000000000ff, 0xffff00000000ff00, 0xffff00000000ffff,
    0xffff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0xffff000000ffffff,
    0xffff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0xffff0000ff00ffff,
    0xffff0000ffff0000, 0xffff0000ffff00ff, 0xffff0000ffffff00, 0xffff0000ffffffff,
    0xffff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0xffff00ff0000ffff,
    0xffff00ff00ff0000, 0xffff00ff00ff00ff, 0xffff00ff00ffff00, 0xffff00ff00ffffff,
    0xffff00ffff000000, 0xffff00ffff0000ff, 0xffff00ffff00ff00, 0xffff00ffff00ffff,
    0xffff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0xffff00ffffffffff,
    0xffffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0xffffff000000ffff,
    0xffffff0000ff0000, 0xffffff0000ff00ff, 0xffffff0000ffff00, 0xffffff0000ffffff,
    0xffffff00ff000000, 0xffffff00ff0000ff, 0xffffff00ff00ff00, 0xffffff00ff00ffff,
    0xffffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0xffffff00ffffffff,
    0xffffffff00000000, 0xffffffff000000ff, 0xffffffff0000ff00, 0xffffffff0000ffff,
    0xffffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0xffffffff00ffffff,
    0xffffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0xffffffffff00ffff,
    0xffffffffffff0000, 0xffffffffffff00ff, 0xffffffffffffff00, 0xffffffffffffffff,
};


static __forceinline BOOLEAN
IsSse42Supported(
    void
    )
{
    int regs[4] = {0};

    __cpuid(regs, 1);

    if (regs[2] & (1 << 20))
    {
        return TRUE;
    }

    return FALSE;
}


void
IntPreinit(
    void
    )
{
    IntGlueReset();
}


//
// IntInit
//
INTSTATUS
IntInit(
    _Inout_ GLUE_IFACE *GlueInterface,
    _In_ UPPER_IFACE const *UpperInterface
    )
{
    INTSTATUS status;

    status = IntGlueInit(GlueInterface, UpperInterface);
    if (!INT_SUCCESS(status))
    {
        return status;
    }

    //
    // Populate our part of the interface
    //
    GlueInterface->NewGuestNotification             = IntNewGuestNotification;
    GlueInterface->DisableIntro                     = IntDisableIntro;
    GlueInterface->NotifyGuestPowerStateChange      = IntNotifyGuestPowerStateChange;
    GlueInterface->DebugProcessCommand              = IntProcessDebugCommand;
    GlueInterface->UpdateExceptions                 = IntUpdateExceptions;
    GlueInterface->GetExceptionsVersion             = IntGetExceptionsVersion;
    GlueInterface->GetGuestInfo                     = IntGetGuestInfo;
    GlueInterface->InjectProcessAgent               = IntInjectProcessAgentInGuest;
    GlueInterface->InjectFileAgent                  = IntInjectFileAgentInGuest;
    GlueInterface->SetIntroAbortStatus              = IntAbortEnableIntro;
    GlueInterface->AddExceptionFromAlert            = IntAddExceptionFromAlert;
    GlueInterface->RemoveException                  = IntRemoveException;
    GlueInterface->FlushAlertExceptions             = IntFlushAlertExceptions;
    GlueInterface->AddRemoveProtectedProcessUtf16   = IntAddRemoveProtectedProcessUtf16;
    GlueInterface->AddRemoveProtectedProcessUtf8    = IntAddRemoveProtectedProcessUtf8;
    GlueInterface->RemoveAllProtectedProcesses      = IntRemoveAllProtectedProcesses;
    GlueInterface->GetCurrentInstructionLength      = IntGetCurrentInstructionLength;
    GlueInterface->GetCurrentInstructionMnemonic    = IntGetCurrentInstructionMnemonic;
    GlueInterface->IterateVirtualAddressSpace       = IntIterateVaSpace;
    GlueInterface->ModifyDynamicOptions             = IntModifyDynamicOptions;
    GlueInterface->FlushGpaCache                    = IntFlushGpaCache;
    GlueInterface->GetCurrentIntroOptions           = IntGetCurrentIntroOptions;
    GlueInterface->UpdateSupport                    = IntUpdateSupport;
    GlueInterface->GetSupportVersion                = IntGetSupportVersion;
    GlueInterface->SetLogLevel                      = IntSetLogLevel;
    GlueInterface->GetVersionString                 = IntGetVersionString;

    gSse42Supported = IsSse42Supported();

    LOG("IntroCore initialised: version %d.%d.%d, build %05d, changeset %s, built on %s %s from branch %s\n",
        INTRO_VERSION_MAJOR, INTRO_VERSION_MINOR, INTRO_VERSION_REVISION, INTRO_VERSION_BUILDNUMBER,
        INTRO_VERSION_CHANGESET, __DATE__, __TIME__, INTRO_VERSION_BRANCH);

    status = IntSpinLockInit(&gLock, "INTRO GLOCK");
    if (!INT_SUCCESS(status))
    {
        ERROR("[ERROR] IntSpinLockInit failed: %08x\n", status);
        return status;
    }

    return INT_STATUS_SUCCESS;
}


//
// IntUninit
//
INTSTATUS
IntUninit(
    void
    )
{
    LIST_ENTRY *list;

    TRACE("[INFO] Unloading introspection library...\n");

    if (gGuest.Initialized)
    {
        INTSTATUS status = IntDisableIntro(gIntHandle, IG_DISABLE_IGNORE_SAFENESS);
        if (!INT_SUCCESS(status))
        {
            ERROR("[ERROR] IntDisableIntro failed: 0x%08x\n", status);
        }
    }

    if (gLock)
    {
        IntSpinLockUnInit(&gLock);
    }

    list = gMultiPageMaps.Flink;
    while (list != &gMultiPageMaps)
    {
        MULTI_PAGE_MAP *pPage = CONTAINING_RECORD(list, MULTI_PAGE_MAP, Link);
        list = list->Flink;

        HpFreeAndNullWithTag(&pPage->OrigAlloc, IC_TAG_MLMP);
        RemoveEntryList(&pPage->Link);
        HpFreeAndNullWithTag(&pPage, IC_TAG_MLMP);
    }

    IntGlueReset();

    return INT_STATUS_SUCCESS;
}


static INTSTATUS
IntVirtMemReadWrite(
    _In_ QWORD VirtualAddress,
    _In_ DWORD Length,
    _In_opt_ QWORD Cr3,
    _When_(Write == TRUE, _In_reads_bytes_(Length))
    _When_(Write == FALSE, _Out_writes_bytes_(Length))
    void *Buffer,
    _Out_opt_ DWORD *RetLength,
    _In_ BOOLEAN Write
    )
{
    INTSTATUS status;
    DWORD left = Length;
    QWORD gva = VirtualAddress;
    BYTE *buffer = Buffer;

    if (0 == Length)
    {
        return INT_STATUS_INVALID_PARAMETER_2;
    }

    if (0 == Cr3)
    {
        status = IntCr3Read(IG_CURRENT_VCPU, &Cr3);
        if (!INT_SUCCESS(status))
        {
            return status;
        }
    }

    do
    {
        DWORD size = MIN(left, PAGE_REMAINING(gva));
        void *p;

        status = IntVirtMemMap(gva, size, Cr3, 0, &p);
        if (!INT_SUCCESS(status))
        {
            goto cleanup_and_exit;
        }

        if (__unlikely(Write))
        {
            memcpy(p, buffer, size);
        }
        else
        {
            memcpy(buffer, p, size);
        }

        IntVirtMemUnmap(&p);

        gva += size;
        left -= size;
        buffer += size;
    } while (gva < VirtualAddress + Length);

    status = INT_STATUS_SUCCESS;

cleanup_and_exit:
    if (RetLength != NULL)
    {
        *RetLength = Length - left;
    }

    return status;
}


INTSTATUS
IntVirtMemSet(
    _In_ QWORD VirtualAddress,
    _In_ DWORD Length,
    _In_opt_ QWORD Cr3,
    _In_ BYTE Value
    )
{
    INTSTATUS status;
    DWORD left = Length;
    QWORD gva = VirtualAddress;

    if (0 == Length)
    {
        return INT_STATUS_INVALID_PARAMETER_2;
    }

    if (0 == Cr3)
    {
        status = IntCr3Read(IG_CURRENT_VCPU, &Cr3);
        if (!INT_SUCCESS(status))
        {
            return status;
        }
    }

    do
    {
        DWORD size = MIN(left, PAGE_REMAINING(gva));
        void *p;

        status = IntVirtMemMap(gva, size, Cr3, 0, &p);
        if (!INT_SUCCESS(status))
        {
            return status;
        }

        memset(p, Value, size);
        IntVirtMemUnmap(&p);

        gva += size;
        left -= size;
    } while (gva < VirtualAddress + Length);

    return INT_STATUS_SUCCESS;
}


static INTSTATUS
IntPhysMemReadWriteAnySize(
    _In_ QWORD PhysicalAddress,
    _In_ DWORD Length,
    _When_(Write == TRUE, _In_reads_bytes_(Length))
    _When_(Write == FALSE, _Out_writes_bytes_(Length))
    void *Buffer,
    _Out_opt_ DWORD *RetLength,
    _In_ BOOLEAN Write
    )
{
    INTSTATUS status;
    DWORD left = Length;
    QWORD gpa = PhysicalAddress;
    BYTE *buffer = Buffer;

    if (0 == Length)
    {
        return INT_STATUS_INVALID_PARAMETER_2;
    }

    while (gpa < PhysicalAddress + Length)
    {
        DWORD size = MIN(left, PAGE_REMAINING(gpa));
        void *p;

        status = IntPhysMemMap(gpa, size, 0, &p);
        if (!INT_SUCCESS(status))
        {
            goto cleanup_and_exit;
        }

        if (__unlikely(Write))
        {
            memcpy(p, buffer, size);
        }
        else
        {
            memcpy(buffer, p, size);
        }

        IntVirtMemUnmap(&p);

        gpa += size;
        left -= size;
        buffer += size;
    }

    status = INT_STATUS_SUCCESS;

cleanup_and_exit:
    if (RetLength != NULL)
    {
        *RetLength = Length - left;
    }

    return status;
}


static INTSTATUS
IntPhysMemReadWrite(
    _In_ QWORD PhysicalAddress,
    _In_ DWORD Length,
    _Inout_updates_bytes_(Length) void *Buffer,
    _Out_opt_ DWORD *RetLength,
    _In_ BOOLEAN Write
    )
{
    INTSTATUS status;
    void *p;
    DWORD copiedSize = 0;

    if (PAGE_COUNT(PhysicalAddress, Length) > 1)
    {
        return INT_STATUS_INVALID_PARAMETER_1;
    }

    if (0 == Length)
    {
        return INT_STATUS_INVALID_PARAMETER_2;
    }

    status = IntPhysMemMap(PhysicalAddress, PAGE_REMAINING(PhysicalAddress), 0, &p);
    if (!INT_SUCCESS(status))
    {
        TRACE("[ERROR] IntPhysMemMap failed: 0x%08x\n", status);
        goto cleanup_and_exit;
    }

    if (Write)
    {
        memcpy(p, Buffer, Length);
    }
    else
    {
        memcpy(Buffer, p, Length);
    }

    copiedSize = Length;

    IntPhysMemUnmap(&p);

    status = INT_STATUS_SUCCESS;

cleanup_and_exit:
    if (RetLength != NULL)
    {
        *RetLength = copiedSize;
    }

    return status;
}


INTSTATUS
IntVirtMemRead(
    _In_ QWORD Gva,
    _In_ DWORD Length,
    _In_opt_ QWORD Cr3,
    _Out_writes_bytes_(Length) void *Buffer,
    _Out_opt_ DWORD *RetLength
    )
{
    return IntVirtMemReadWrite(Gva, Length, Cr3, Buffer, RetLength, FALSE);
}


INTSTATUS
IntVirtMemWrite(
    _In_ QWORD Gva,
    _In_ DWORD Length,
    _In_opt_ QWORD Cr3,
    _In_reads_bytes_(Length) void *Buffer
    )
{
    return IntVirtMemReadWrite(Gva, Length, Cr3, Buffer, NULL, TRUE);
}


INTSTATUS
IntKernVirtMemRead(
    _In_ QWORD KernelGva,
    _In_ DWORD Length,
    _Out_writes_bytes_(Length) void *Buffer,
    _Out_opt_ DWORD *RetLength
    )
{
    return IntVirtMemReadWrite(KernelGva, Length, gGuest.Mm.SystemCr3, Buffer, RetLength, FALSE);
}


INTSTATUS
IntKernVirtMemWrite(
    _In_ QWORD KernelGva,
    _In_ DWORD Length,
    _In_reads_bytes_(Length) void *Buffer
    )
{
    return IntVirtMemReadWrite(KernelGva, Length, gGuest.Mm.SystemCr3, Buffer, NULL, TRUE);
}


INTSTATUS
IntPhysicalMemRead(
    _In_ QWORD PhysicalAddress,
    _In_ DWORD Length,
    _Out_writes_bytes_(Length) void *Buffer,
    _Out_opt_ DWORD *RetLength
    )
{
    return IntPhysMemReadWrite(PhysicalAddress, Length, Buffer, RetLength, FALSE);
}


INTSTATUS
IntPhysicalMemWrite(
    _In_ QWORD PhysicalAddress,
    _In_ DWORD Length,
    _In_reads_bytes_(Length) void *Buffer
    )
{
    return IntPhysMemReadWrite(PhysicalAddress, Length, Buffer, NULL, TRUE);
}


INTSTATUS
IntPhysicalMemReadAnySize(
    _In_ QWORD PhysicalAddress,
    _In_ DWORD Length,
    _Out_writes_bytes_(Length) void *Buffer,
    _Out_opt_ DWORD *RetLength
    )
{
    return IntPhysMemReadWriteAnySize(PhysicalAddress, Length, Buffer, RetLength, FALSE);
}


INTSTATUS
IntPhysicalMemWriteAnySize(
    _In_ QWORD PhysicalAddress,
    _In_ DWORD Length,
    _In_reads_bytes_(Length) void *Buffer
    )
{
    return IntPhysMemReadWriteAnySize(PhysicalAddress, Length, Buffer, NULL, TRUE);
}


INTSTATUS
IntKernVirtMemFetchQword(
    _In_ QWORD GuestVirtualAddress,
    _Out_ QWORD *Data
    )
{
    return IntKernVirtMemRead(GuestVirtualAddress, 8,  Data, NULL);
}


INTSTATUS
IntKernVirtMemFetchDword(
    _In_ QWORD GuestVirtualAddress,
    _Out_ DWORD *Data
    )
{
    return IntKernVirtMemRead(GuestVirtualAddress, 4, Data, NULL);
}


INTSTATUS
IntKernVirtMemFetchWordSize(
    _In_ QWORD GuestVirtualAddress,
    _Out_ void *Data
    )
{
    return IntKernVirtMemRead(GuestVirtualAddress, gGuest.WordSize, Data, NULL);
}


INTSTATUS
IntVirtMemFetchQword(
    _In_ QWORD GuestVirtualAddress,
    _In_opt_ QWORD Cr3,
    _Out_ QWORD *Data
    )
{
    return IntVirtMemRead(GuestVirtualAddress, 8, Cr3, Data, NULL);
}


INTSTATUS
IntVirtMemFetchDword(
    _In_ QWORD GuestVirtualAddress,
    _In_opt_ QWORD Cr3,
    _Out_ DWORD *Data
    )
{
    return IntVirtMemRead(GuestVirtualAddress, 4, Cr3, Data, NULL);
}


INTSTATUS
IntVirtMemFetchWordSize(
    _In_ QWORD GuestVirtualAddress,
    _In_opt_ QWORD Cr3,
    _Out_ void *Data
    )
{
    return IntVirtMemRead(GuestVirtualAddress, gGuest.WordSize, Cr3, Data, NULL);
}


INTSTATUS
IntKernVirtMemPatchQword(
    _In_ QWORD GuestVirtualAddress,
    _In_ QWORD Data
    )
{
    return IntKernVirtMemWrite(GuestVirtualAddress, 8, &Data);
}


INTSTATUS
IntKernVirtMemPatchDword(
    _In_ QWORD GuestVirtualAddress,
    _In_ DWORD Data
    )
{
    return IntKernVirtMemWrite(GuestVirtualAddress, 4, &Data);
}


INTSTATUS
IntKernVirtMemPatchWordSize(
    _In_ QWORD GuestVirtualAddress,
    _In_ QWORD Data
    )
{
    return IntKernVirtMemWrite(GuestVirtualAddress, gGuest.WordSize, &Data);
}


INTSTATUS
IntVirtMemPatchQword(
    _In_ QWORD GuestVirtualAddress,
    _In_opt_ QWORD Cr3,
    _In_ QWORD Data
    )
{
    return IntVirtMemWrite(GuestVirtualAddress, 8, Cr3, &Data);
}


INTSTATUS
IntVirtMemPatchDword(
    _In_ QWORD GuestVirtualAddress,
    _In_opt_ QWORD Cr3,
    _In_ DWORD Data
    )
{
    return IntVirtMemWrite(GuestVirtualAddress, 4, Cr3, &Data);
}


INTSTATUS
IntVirtMemPatchWordSize(
    _In_ QWORD GuestVirtualAddress,
    _In_opt_ QWORD Cr3,
    _In_ QWORD Data
    )
{
    return IntVirtMemWrite(GuestVirtualAddress, gGuest.WordSize, Cr3, &Data);
}


INTSTATUS
IntVirtMemFetchString(
    _In_ QWORD Gva,
    _In_ DWORD MaxLength,
    _In_opt_ QWORD Cr3,
    _Out_writes_z_(MaxLength) void *Buffer
    )
{
    INTSTATUS status;
    PCHAR pBuf;
    QWORD chunk;
    DWORD i, j;

    if (NULL == Buffer)
    {
        return INT_STATUS_INVALID_PARAMETER_4;
    }

    i = 0;
    chunk = 0;

    pBuf = (PCHAR)Buffer;

    // We will basically read 8 bytes at a time
    while (i < MaxLength)
    {
        // Read current chunk
        status = IntVirtMemRead(Gva + i, 8, Cr3, &chunk, NULL);
        if (!INT_SUCCESS(status))
        {
            return status;
        }

        // Check if we exceed maximum length, and if we have a NULL terminator
        for (j = 0; j < MIN(8u, MaxLength - i); j++)
        {
            pBuf[i++] = (CHAR)chunk;
            chunk >>= 8;

            if (pBuf[i - 1] == 0)
            {
                // Done! We found the NULL terminator
                return INT_STATUS_SUCCESS;
            }
        }
    }

    return INT_STATUS_NOT_FOUND;
}


static INTSTATUS
IntMapGpaForTranslation(
    _In_ QWORD Gpa,
    _Outptr_ void **HostPtr
    )
{
    //
    // The reason for the #ifdef is that on Napoca, the fastmap is more than enough.
    // The speed-up of using the cache is less than 5%, so don't bother.
    //
#ifdef USER_MODE
    if (__likely(gGuest.GpaCache))
    {
        return IntGpaCacheFindAndAdd(gGuest.GpaCache, Gpa, HostPtr);
    }
#endif // USER_MODE

    return IntPhysMemMap(Gpa, PAGE_SIZE, 0, HostPtr);
}


static INTSTATUS
IntUnmapGpaForTranslation(
    _In_ QWORD Gpa,
    _Inout_ _At_(*HostPtr, _Post_null_) void **HostPtr
    )
{
#ifdef USER_MODE
    if (__likely(gGuest.GpaCache))
    {
        return IntGpaCacheRelease(gGuest.GpaCache, Gpa);
    }
#else
    UNREFERENCED_PARAMETER(Gpa);
#endif // USER_MODE

    return IntPhysMemUnmap(HostPtr);
}


static INTSTATUS
IntTranslateVa32(
    _In_ UINT32 Gva,
    _In_ UINT32 Cr3,
    _Out_ VA_TRANSLATION *Translation
    )
{
    INTSTATUS status;

    UINT32 pdi = PD32_INDEX(Gva);
    UINT32 pti = PT32_INDEX(Gva);

    UINT32 pde, pte, pf;
    UINT32 *pd, *pt;

    pd = pt = NULL;
    pde = pte = 0;

    Translation->Pointer64 = FALSE;

    //
    // Fetch and handle PD entry.
    //
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS32(Cr3), &pd);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit;
    }

    pde = pd[pdi];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS32(Cr3) + 4ull * pdi;
    Translation->MappingsEntries[Translation->MappingsCount] = pde;
    Translation->MappingsCount++;

    if ((pde & PD_P) == 0)
    {
        status = INT_STATUS_NO_MAPPING_STRUCTURES;
        goto cleanup_and_exit;
    }

    Translation->IsUser = Translation->IsUser && !!(pde & PT_US);
    Translation->IsWritable = Translation->IsWritable && !!(pde & PT_RW);

    // Check if this is a 4 MB page.
    if (0 != (pde & PD_PS))
    {
        // This is a 4 MB page, next table is in fact the physical address of the page
        Translation->Flags = pde;
        Translation->PageSize = PAGE_SIZE_4M;

        pf = (pde & 0xFFC00000) | ((pde & 0x003FE000) << 19);
        pf += (Gva & 0x3FFFFF);

        goto using_4m_page;
    }


    // Fetch & handle PT entry.
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS32(pde), &pt);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit;
    }

    pte = pt[pti];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS32(pde) + 4ull * pti;
    Translation->MappingsEntries[Translation->MappingsCount] = pte;
    Translation->MappingsCount++;

    Translation->Flags = pte;
    Translation->PageSize = PAGE_SIZE_4K;

    if ((pte & PD_P) == 0)
    {
        pf = 0;
    }
    else
    {
        Translation->IsUser = Translation->IsUser && !!(pte & PT_US);
        Translation->IsWritable = Translation->IsWritable && !!(pte & PT_RW);

        pf = CLEAN_PHYS_ADDRESS32(pte);
        pf += (Gva & 0xFFF);
    }

using_4m_page:
    Translation->PhysicalAddress = pf;

    status = INT_STATUS_SUCCESS;

cleanup_and_exit:
    if (NULL != pd)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS32(Cr3), &pd);
    }

    if (NULL != pt)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS32(pde), &pt);
    }

    return status;
}


static INTSTATUS
IntTranslateVa32Pae(
    _In_ UINT64 Gva,
    _In_ UINT64 Cr3,
    _Out_ VA_TRANSLATION *Translation
    )
{
    INTSTATUS status;

    UINT32 pdpi = PDPPAE_INDEX(Gva);
    UINT32 pdi  = PDPAE_INDEX(Gva);
    UINT32 pti  = PTPAE_INDEX(Gva);

    UINT64 pdpe, pde, pte, pf;
    UINT64 *pdp, *pd, *pt;

    pdp = pd = pt = NULL;
    pdpe = pde = pte = 0;

    Translation->Pointer64 = TRUE;

    //
    // Fetch and handle the PDP entry.
    //
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS32PAE(Cr3), &pdp);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit_pae;
    }

    // CR3 is aligned only to 32 bytes, not 4096 bytes.
    pdp = (PQWORD)(((PBYTE)pdp) + (Cr3 & 0xFE0));

    // We get the PDPE from the PDP, which points to a PD
    pdpe = pdp[pdpi];

    Translation->MappingsTrace[Translation->MappingsCount] = (CLEAN_PHYS_ADDRESS32PAE_ROOT(Cr3)) + 8ull * pdpi;
    Translation->MappingsEntries[Translation->MappingsCount] = pdpe;
    Translation->MappingsCount++;

    pdp = (PQWORD)(((PBYTE)pdp) - (Cr3 & 0xFE0));

    if ((pdpe & PDP_P) == 0)
    {
        status = INT_STATUS_NO_MAPPING_STRUCTURES;
        goto cleanup_and_exit_pae;
    }

    //
    // Fetch and handle PD entry.
    //
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS32PAE(pdpe), &pd);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit_pae;
    }

    pde = pd[pdi];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS32PAE(pdpe) + 8ull * pdi;
    Translation->MappingsEntries[Translation->MappingsCount] = pde;
    Translation->MappingsCount++;

    if ((pde & PD_P) == 0)
    {
        status = INT_STATUS_NO_MAPPING_STRUCTURES;
        goto cleanup_and_exit_pae;
    }

    Translation->IsUser = Translation->IsUser && !!(pde & PT_US);
    Translation->IsWritable = Translation->IsWritable && !!(pde & PT_RW);
    Translation->IsExecutable = Translation->IsExecutable && !(pde & PT_XD);

    // Check if this is a 2 MB page.
    if (pde & PD_PS)
    {
        // This is a 2 MB page, next table is in fact the physical address of the page
        Translation->Flags = pde;
        Translation->PageSize = PAGE_SIZE_2M;

        pf = CLEAN_PHYS_ADDRESS32PAE(pde) & (~0x1FFFFFULL);
        pf += (Gva & 0x1FFFFF);

        goto using_2m_page_pae;
    }

    //
    // Fetch and handle PT entry.
    //
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS32PAE(pde), &pt);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit_pae;
    }

    pte = pt[pti];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS32PAE(pde) + 8ull * pti;
    Translation->MappingsEntries[Translation->MappingsCount] = pte;
    Translation->MappingsCount++;

    Translation->Flags = pte;
    Translation->PageSize = PAGE_SIZE_4K;

    if ((pte & PT_P) == 0)
    {
        pf = 0;
    }
    else
    {
        Translation->IsUser = Translation->IsUser && !!(pte & PT_US);
        Translation->IsWritable = Translation->IsWritable && !!(pte & PT_RW);
        Translation->IsExecutable = Translation->IsExecutable && !(pte & PT_XD);

        pf = CLEAN_PHYS_ADDRESS32PAE(pte);
        pf += (Gva & 0xFFF);
    }

using_2m_page_pae:
    Translation->PhysicalAddress = pf;

    status = INT_STATUS_SUCCESS;

cleanup_and_exit_pae:
    if (NULL != pdp)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS32PAE(Cr3), &pdp);
    }

    if (NULL != pd)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS32PAE(pdpe), &pd);
    }

    if (NULL != pt)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS32PAE(pde), &pt);
    }

    return status;
}


static INTSTATUS
IntTranslateVa64(
    _In_ UINT64 Gva,
    _In_ UINT64 Cr3,
    _Out_ VA_TRANSLATION *Translation
    )
{
    INTSTATUS status;

    UINT32 pml4i = PML4_INDEX(Gva);
    UINT32 pdpi  = PDP_INDEX (Gva);
    UINT32 pdi   = PD_INDEX  (Gva);
    UINT32 pti   = PT_INDEX  (Gva);

    UINT64 pml4e, pdpe, pde, pte, pf;   // entries values
    UINT64 *pml4, *pdp, *pd, *pt;       // mapped pages

    pml4 = pdp = pd = pt = NULL;
    pml4e = pdpe = pde = pte = 0;

    Translation->Pointer64 = TRUE;

    // Fetch and handle the PML4 entry.
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS64(Cr3), &pml4);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit_4_level;
    }

    pml4e = pml4[pml4i];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS64(Cr3) + 8ull * pml4i;
    Translation->MappingsEntries[Translation->MappingsCount] = pml4e;
    Translation->MappingsCount++;

    if ((pml4e & PD_P) == 0)
    {
        status = INT_STATUS_NO_MAPPING_STRUCTURES;
        goto cleanup_and_exit_4_level;
    }

    Translation->IsUser = Translation->IsUser && !!(pml4e & PT_US);
    Translation->IsWritable = Translation->IsWritable && !!(pml4e & PT_RW);
    Translation->IsExecutable = Translation->IsExecutable && !(pml4e & PT_XD);

    // Fetch and handle the PDP entry.
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pml4e), &pdp);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit_4_level;
    }

    pdpe = pdp[pdpi];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS64(pml4e) + 8ull * pdpi;
    Translation->MappingsEntries[Translation->MappingsCount] = pdpe;
    Translation->MappingsCount++;

    if ((pdpe & PD_P) == 0)
    {
        status = INT_STATUS_NO_MAPPING_STRUCTURES;
        goto cleanup_and_exit_4_level;
    }

    Translation->IsUser = Translation->IsUser && !!(pdpe & PT_US);
    Translation->IsWritable = Translation->IsWritable && !!(pdpe & PT_RW);
    Translation->IsExecutable = Translation->IsExecutable && !(pdpe & PT_XD);

    // Check if this is a 1 GB page.
    if (pdpe & PDP_PS)
    {
        // This is a 1 GB page, next table is in fact the physical address of the page
        Translation->Flags = pdpe;
        Translation->PageSize = PAGE_SIZE_1G;

        pf = CLEAN_PHYS_ADDRESS64(pdpe) & (~0x3FFFFFFFULL);
        pf += (Gva & 0x3FFFFFFF);

        goto using_1g_page_4_level;
    }

    // Fetch and handle the PD entry.
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pdpe), &pd);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit_4_level;
    }

    pde = pd[pdi];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS64(pdpe) + 8ull * pdi;
    Translation->MappingsEntries[Translation->MappingsCount] = pde;
    Translation->MappingsCount++;

    if ((pde & PD_P) == 0)
    {
        status = INT_STATUS_NO_MAPPING_STRUCTURES;
        goto cleanup_and_exit_4_level;
    }

    Translation->IsUser = Translation->IsUser && !!(pde & PT_US);
    Translation->IsWritable = Translation->IsWritable && !!(pde & PT_RW);
    Translation->IsExecutable = Translation->IsExecutable && !(pde & PT_XD);

    // Check if this is a 2 MB page.
    if (pde & PD_PS)
    {
        // This is a 2 MB page, next table is in fact the physical address of the page
        Translation->Flags = pde;
        Translation->PageSize = PAGE_SIZE_2M;

        pf = CLEAN_PHYS_ADDRESS64(pde) & (~0x1FFFFFULL);
        pf += (Gva & 0x1FFFFF);

        goto using_2m_page_4_level;
    }

    // Fetch and handle the PT entry.
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pde), &pt);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit_4_level;
    }

    pte = pt[pti];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS64(pde) + 8ull * pti;
    Translation->MappingsEntries[Translation->MappingsCount] = pte;
    Translation->MappingsCount++;

    Translation->Flags = pte;
    Translation->PageSize = PAGE_SIZE_4K;

    if ((pte & PD_P) == 0)
    {
        // If the last page is not present, we will return success; The caller must check if the page is
        // present or not.
        pf = 0;
    }
    else
    {
        Translation->IsUser = Translation->IsUser && !!(pte & PT_US);
        Translation->IsWritable = Translation->IsWritable && !!(pte & PT_RW);
        Translation->IsExecutable = Translation->IsExecutable && !(pte & PT_XD);

        pf = CLEAN_PHYS_ADDRESS64(pte);
        pf += (Gva & 0xFFF);
    }

using_1g_page_4_level:
using_2m_page_4_level:
    Translation->PhysicalAddress = pf;

    status = INT_STATUS_SUCCESS;

cleanup_and_exit_4_level:
    if (NULL != pml4)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS64(Cr3), &pml4);
    }

    if (NULL != pdp)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pml4e), &pdp);
    }

    if (NULL != pd)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pdpe), &pd);
    }

    if (NULL != pt)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pde), &pt);
    }

    return status;
}


static INTSTATUS
IntTranslateVa64La57(
    _In_ UINT64 Gva,
    _In_ UINT64 Cr3,
    _Out_ VA_TRANSLATION *Translation
    )
{
    INTSTATUS status;

    UINT32 pml5i = PML5_INDEX(Gva);
    UINT32 pml4i = PML4_INDEX(Gva);
    UINT32 pdpi = PDP_INDEX(Gva);
    UINT32 pdi = PD_INDEX(Gva);
    UINT32 pti = PT_INDEX(Gva);

    UINT64 pml5e, pml4e, pdpe, pde, pte, pf;    // entries values
    UINT64 *pml5, *pml4, *pdp, *pd, *pt;        // mapped pages

    pml5 = pml4 = pdp = pd = pt = NULL;
    pml5e = pml4e = pdpe = pde = pte = 0;

    Translation->Pointer64 = TRUE;

    // Fetch and handle PML5 entry.
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS64(Cr3), &pml5);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit_5_level;
    }

    pml5e = pml5[pml5i];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS64(Cr3) + 8ull * pml5i;
    Translation->MappingsEntries[Translation->MappingsCount] = pml5e;
    Translation->MappingsCount++;

    if ((pml5e & PD_P) == 0)
    {
        status = INT_STATUS_NO_MAPPING_STRUCTURES;
        goto cleanup_and_exit_5_level;
    }

    // Fetch and handle PML4 entry.
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pml5e), &pml4);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit_5_level;
    }

    pml4e = pml4[pml4i];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS64(Cr3) + 8ull * pml4i;
    Translation->MappingsEntries[Translation->MappingsCount] = pml4e;
    Translation->MappingsCount++;

    if ((pml4e & PD_P) == 0)
    {
        status = INT_STATUS_NO_MAPPING_STRUCTURES;
        goto cleanup_and_exit_5_level;
    }

    Translation->IsUser = Translation->IsUser && !!(pml4e & PT_US);
    Translation->IsWritable = Translation->IsWritable && !!(pml4e & PT_RW);
    Translation->IsExecutable = Translation->IsExecutable && !(pml4e & PT_XD);

    // Fetch and handle the PDP entry.
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pml4e), &pdp);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit_5_level;
    }

    pdpe = pdp[pdpi];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS64(pml4e) + 8ull * pdpi;
    Translation->MappingsEntries[Translation->MappingsCount] = pdpe;
    Translation->MappingsCount++;

    if ((pdpe & PD_P) == 0)
    {
        status = INT_STATUS_NO_MAPPING_STRUCTURES;
        goto cleanup_and_exit_5_level;
    }

    Translation->IsUser = Translation->IsUser && !!(pdpe & PT_US);
    Translation->IsWritable = Translation->IsWritable && !!(pdpe & PT_RW);
    Translation->IsExecutable = Translation->IsExecutable && !(pdpe & PT_XD);

    // Check if this is a 1 GB page.
    if (pdpe & PDP_PS)
    {
        // This is a 1 GB page, next table is in fact the physical address of the page
        Translation->Flags = pdpe;
        Translation->PageSize = PAGE_SIZE_1G;

        pf = CLEAN_PHYS_ADDRESS64(pdpe) & (~0x3FFFFFFFULL);
        pf += (Gva & 0x3FFFFFFF);

        goto using_1g_page_5_level;
    }

    // Fetch and handle the PD entry.
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pdpe), &pd);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit_5_level;
    }

    pde = pd[pdi];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS64(pdpe) + 8ull * pdi;
    Translation->MappingsEntries[Translation->MappingsCount] = pde;
    Translation->MappingsCount++;

    if ((pde & PD_P) == 0)
    {
        status = INT_STATUS_NO_MAPPING_STRUCTURES;
        goto cleanup_and_exit_5_level;
    }

    Translation->IsUser = Translation->IsUser && !!(pde & PT_US);
    Translation->IsWritable = Translation->IsWritable && !!(pde & PT_RW);
    Translation->IsExecutable = Translation->IsExecutable && !(pde & PT_XD);

    // Check if this is a 2 MB page.
    if (pde & PD_PS)
    {
        // This is a 2 MB page, next table is in fact the physical address of the page
        Translation->Flags = pde;
        Translation->PageSize = PAGE_SIZE_2M;

        pf = CLEAN_PHYS_ADDRESS64(pde) & (~0x1FFFFFULL);
        pf += (Gva & 0x1FFFFF);

        goto using_2m_page_5_level;
    }

    // Fetch and handle the PT entry.
    status = IntMapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pde), &pt);
    if (!INT_SUCCESS(status))
    {
        goto cleanup_and_exit_5_level;
    }

    pte = pt[pti];

    Translation->MappingsTrace[Translation->MappingsCount] = CLEAN_PHYS_ADDRESS64(pde) + 8ull * pti;
    Translation->MappingsEntries[Translation->MappingsCount] = pte;
    Translation->MappingsCount++;

    Translation->Flags = pte;
    Translation->PageSize = PAGE_SIZE_4K;

    if ((pte & PD_P) == 0)
    {
        // If the last page is not present, we will return success; The caller must check if the page is
        // present or not.
        pf = 0;
    }
    else
    {
        Translation->IsUser = Translation->IsUser && !!(pte & PT_US);
        Translation->IsWritable = Translation->IsWritable && !!(pte & PT_RW);
        Translation->IsExecutable = Translation->IsExecutable && !(pte & PT_XD);

        pf = CLEAN_PHYS_ADDRESS64(pte);
        pf += (Gva & 0xFFF);
    }

using_1g_page_5_level:
using_2m_page_5_level:
    Translation->PhysicalAddress = pf;

    status = INT_STATUS_SUCCESS;

cleanup_and_exit_5_level:
    if (NULL != pml5)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS64(Cr3), &pml5);
    }

    if (NULL != pml4)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pml5e), &pml4);
    }

    if (NULL != pdp)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pml4e), &pdp);
    }

    if (NULL != pd)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pdpe), &pd);
    }

    if (NULL != pt)
    {
        IntUnmapGpaForTranslation(CLEAN_PHYS_ADDRESS64(pde), &pt);
    }

    return status;
}


INTSTATUS
IntTranslateVirtualAddressEx(
    _In_ QWORD Gva,
    _In_ QWORD Cr3,
    _In_ DWORD Flags,
    _Out_ VA_TRANSLATION *Translation
    )
{
    INTSTATUS status;
    BYTE pagingMode;

    if (Translation == NULL)
    {
        return INT_STATUS_INVALID_PARAMETER_4;
    }

    memzero(Translation, sizeof(*Translation));

    if (__unlikely(0 != (Flags & TRFLG_MODE_MASK)))
    {
        switch (Flags & TRFLG_MODE_MASK)
        {
        case TRFLG_NORMAL_MODE:
            pagingMode = PAGING_NORMAL_MODE;
            break;
        case TRFLG_PAE_MODE:
            pagingMode = PAGING_PAE_MODE;
            break;
        case TRFLG_4_LEVEL_MODE:
            pagingMode = PAGING_4_LEVEL_MODE;
            break;
        case TRFLG_5_LEVEL_MODE:
            pagingMode = PAGING_5_LEVEL_MODE;
            break;
        default:
            return INT_STATUS_INVALID_PARAMETER_2;
        }
    }
    else
    {
        if (__unlikely(gVcpu && !(gVcpu->Regs.Cr0 & CR0_PG)))
        {
            pagingMode = PAGING_NONE;
        }
        else
        {
            pagingMode = gGuest.Mm.Mode;
        }
    }

    Translation->VirtualAddress = Gva;
    Translation->Cr3 = Cr3;
    Translation->IsUser = Translation->IsWritable = Translation->IsExecutable = TRUE;
    Translation->PagingMode = pagingMode;

    if (pagingMode == PAGING_5_LEVEL_MODE)
    {
        status = IntTranslateVa64La57(Gva, Cr3, Translation);
    }
    else if (pagingMode == PAGING_4_LEVEL_MODE)
    {
        status = IntTranslateVa64(Gva, Cr3, Translation);
    }
    else if (pagingMode == PAGING_PAE_MODE)
    {
        status = IntTranslateVa32Pae(Gva, Cr3, Translation);
    }
    else if (pagingMode == PAGING_NORMAL_MODE)
    {
        status = IntTranslateVa32((UINT32)Gva, (UINT32)Cr3, Translation);
    }
    else
    {
        // The CPU is probably in real mode/without paging for the moment.
        Translation->PhysicalAddress = Gva;

        status = INT_STATUS_SUCCESS;
    }

    // Validate the caching attributes, if required.
    if (__unlikely((INT_STATUS_SUCCESS == status) && (0 != (Flags & TRFLG_CACHING_ATTR))))
    {
        // For now, we assume PAT is ALWAYS present & used.

        // Get the per-page caching attributes of this VA page.
        BYTE patIndex;
        IG_QUERY_MSR msr = {0};

        patIndex = 0;

        // Compute the PAT index.
        patIndex |= (Translation->Flags & PT_PAT) >> 5;
        patIndex |= (Translation->Flags & PT_PCD) >> 3;
        patIndex |= (Translation->Flags & PT_PWT) >> 3;

        // Read the PAT MSR.
        msr.MsrId = IG_IA32_PAT;

        status = IntQueryGuestInfo(IG_QUERY_INFO_CLASS_READ_MSR,
                                   (void *)(size_t)IG_CURRENT_VCPU,
                                   &msr,
                                   sizeof(IG_QUERY_MSR));
        if (!INT_SUCCESS(status))
        {
            Translation->CachingAttribute = IG_MEM_UNKNOWN;
        }
        else
        {
            Translation->CachingAttribute = (msr.Value >> (patIndex * 8)) & 0x7;
        }
    }

    return status;
}


INTSTATUS
IntTranslateVirtualAddress(
    _In_ QWORD Gva,
    _In_opt_ QWORD Cr3,
    _Out_ QWORD *PhysicalAddress
    )
{
    INTSTATUS status;
    VA_TRANSLATION translation = { 0 };

    if (NULL == PhysicalAddress)
    {
        return INT_STATUS_INVALID_PARAMETER_3;
    }

    if (0 == Cr3)
    {
        status = IntCr3Read(IG_CURRENT_VCPU, &Cr3);
        if (!INT_SUCCESS(status))
        {
            ERROR("[ERROR] IntCr3Read failed: 0x%08x\n", status);
            return status;
        }
    }

    status = IntTranslateVirtualAddressEx(Gva, Cr3, TRFLG_NONE, &translation);
    if (!INT_SUCCESS(status))
    {
        return status;
    }

    if (0 == (translation.Flags & PT_P) && PAGING_NONE != translation.PagingMode)
    {
        return INT_STATUS_PAGE_NOT_PRESENT;
    }

    *PhysicalAddress = translation.PhysicalAddress;

    return INT_STATUS_SUCCESS;
}


__forceinline static INTSTATUS
IntVirtMemMapMultiPage(
    _In_ QWORD GuestVirtualAddress,
    _In_ DWORD Length,
    _In_opt_ QWORD Cr3,
    _Outptr_result_bytebuffer_(Length) void **HostPtr
    )
{
    INTSTATUS status;
    void *pAlloc, *pAlignedAlloc;
    DWORD readSize = 0;
    MULTI_PAGE_MAP *pPage;

    pAlloc = HpAllocWithTag((size_t)Length + PAGE_SIZE, IC_TAG_MLMP);
    if (NULL == pAlloc)
    {
        return INT_STATUS_INSUFFICIENT_RESOURCES;
    }

    pAlignedAlloc = (void *)ALIGN_UP((SIZE_T)pAlloc, PAGE_SIZE_4K);

    // This will trigger a recursive call, but it's safe since form IntVirtMemRead it will
    // only map one page per call, and will not re-enter this function.
    status = IntVirtMemRead(GuestVirtualAddress, Length, Cr3, pAlignedAlloc, &readSize);
    if (!INT_SUCCESS(status))
    {
        HpFreeAndNullWithTag(&pAlloc, IC_TAG_MLMP);

        return status;
    }
    else if (readSize != Length)
    {
        // Something went wrong without an error. IntKernVirtMemRead always returns an error when
        // a map fails, so this isn't the case.
        HpFreeAndNullWithTag(&pAlloc, IC_TAG_MLMP);

        return INT_STATUS_INVALID_INTERNAL_STATE;
    }

    pPage = HpAllocWithTag(sizeof(*pPage), IC_TAG_MLMP);
    if (NULL == pPage)
    {
        HpFreeAndNullWithTag(&pAlloc, IC_TAG_MLMP);
        return INT_STATUS_INSUFFICIENT_RESOURCES;
    }

    pPage->Gva = GuestVirtualAddress;
    pPage->Length = Length;
    pPage->HostPtr = pAlignedAlloc;
    pPage->OrigAlloc = pAlloc;

    InsertTailList(&gMultiPageMaps, &pPage->Link);

    *HostPtr = pAlignedAlloc;

    return INT_STATUS_SUCCESS;
}


__must_check
INTSTATUS
IntVirtMemMap(
    _In_ QWORD Gva,
    _In_ DWORD Length,
    _In_opt_ QWORD Cr3,
    _In_ DWORD Flags,
    _Outptr_result_bytebuffer_(Length) void **HostPtr
    )
{
    INTSTATUS status;
    VA_TRANSLATION translation;
    DWORD flags;

    UNREFERENCED_PARAMETER(Flags);

    if (0 == Length)
    {
        return INT_STATUS_INVALID_PARAMETER_2;
    }

    if (0 == Cr3)
    {
        status = IntCr3Read(IG_CURRENT_VCPU, &Cr3);
        if (!INT_SUCCESS(status))
        {
            return status;
        }
    }

    if (__unlikely(PAGE_COUNT_4K(Gva, Length) > 1))
    {
        return IntVirtMemMapMultiPage(Gva, Length, Cr3, HostPtr);
    }

    status = IntTranslateVirtualAddressEx(Gva, Cr3, TRFLG_NONE, &translation);
    if (!INT_SUCCESS(status))
    {
        return status;
    }

    if (0 == (translation.Flags & PT_P) && PAGING_NONE != translation.PagingMode)
    {
        return INT_STATUS_PAGE_NOT_PRESENT;
    }

    flags = (translation.PageSize != PAGE_SIZE_4K) ? IG_PHYSMAP_NO_CACHE : 0;

    return IntPhysMemMap(translation.PhysicalAddress, Length, flags, HostPtr);
}


static BOOLEAN
IntVirtMemUnmapMultiPage(
    _Inout_ _At_(*HostPtr, _Post_null_) void **HostPtr
    )
{
    list_for_each(gMultiPageMaps, MULTI_PAGE_MAP, pPage)
    {
        if (__unlikely(pPage->HostPtr == *HostPtr))
        {
            HpFreeAndNullWithTag(&pPage->OrigAlloc, IC_TAG_MLMP);

            RemoveEntryList(&pPage->Link);

            HpFreeAndNullWithTag(&pPage, IC_TAG_MLMP);

            *HostPtr = NULL;

            return TRUE;
        }
    }

    return FALSE;
}


INTSTATUS
IntVirtMemUnmap(
    _Inout_ _At_(*HostPtr, _Post_null_) void **HostPtr
    )
{
    INTSTATUS status;

    if (__unlikely(IntVirtMemUnmapMultiPage(HostPtr)))
    {
        return INT_STATUS_SUCCESS;
    }

    status = IntPhysMemUnmap(HostPtr);
    if (!INT_SUCCESS(status))
    {
        ERROR("[ERROR] IntPhysMemUnmap failed for (%p %p): 0x%08x\n", HostPtr, *HostPtr, status);
    }

    return status;
}


INTSTATUS
IntInjectExceptionInGuest(
    _In_ BYTE Vector,
    _In_ QWORD Cr2,
    _In_ DWORD ErrorCode,
    _In_ DWORD CpuNumber
    )
{
    INTSTATUS status;

    if (CpuNumber >= gGuest.CpuCount)
    {
        return INT_STATUS_INVALID_PARAMETER_4;
    }

    if (gGuest.VcpuArray[CpuNumber].Exception.Valid)
    {
        ERROR("[ERROR] An exception injection is already pending on CPU %d: vector %d, CR2 0x%016llx, "
              "error code 0x%08x",
              CpuNumber, gGuest.VcpuArray[CpuNumber].Exception.Vector, gGuest.VcpuArray[CpuNumber].Exception.Cr2,
              gGuest.VcpuArray[CpuNumber].Exception.ErrorCode);
        return INT_STATUS_ALREADY_INITIALIZED;
    }

    status = IntInjectTrap(CpuNumber, Vector, ErrorCode, Cr2);
    if (!INT_SUCCESS(status))
    {
        return status;
    }

    gGuest.VcpuArray[CpuNumber].Exception.Valid = TRUE;
    gGuest.VcpuArray[CpuNumber].Exception.Vector = Vector;
    gGuest.VcpuArray[CpuNumber].Exception.Cr2 = Cr2;
    gGuest.VcpuArray[CpuNumber].Exception.ErrorCode = ErrorCode;

    return INT_STATUS_SUCCESS;
}


INTSTATUS
IntPauseVcpus(
    void
    )
{
    INTSTATUS status;

    if (gInsideDebugger)
    {
        return INT_STATUS_SUCCESS;
    }

    status = GluePauseVcpus();
    if (!INT_SUCCESS(status))
    {
        ERROR("[ERROR] GluePauseVcpus failed: 0x%08x\n", status);
        IntDbgEnterDebugger();
    }

    return status;
}


INTSTATUS
IntResumeVcpus(
    void
    )
{
    INTSTATUS status;

    if (gInsideDebugger)
    {
        return INT_STATUS_SUCCESS;
    }

    status = GlueResumeVcpus();
    if (!INT_SUCCESS(status))
    {
        ERROR("[ERROR] GlueResumeVcpus failed: 0x%08x\n", status);
        IntDbgEnterDebugger();
    }

    return status;
}


void
IntEnterDebugger2(
    _In_ PCHAR File,
    _In_ DWORD Line
    )
{
    if (NULL != File)
    {
        LOG("[DEBUGGER] IntEnterDebugger called from %s:%d\n", File, Line);
    }

    GlueEnterDebugger();
}


void
IntDbgEnterDebugger2(
    _In_ PCHAR File,
    _In_ DWORD Line
    )
{
    const char *commands[] =
    {
        "!processes",                               // Dump the processes
        "!stvad",
        "!modules_intro",                           // Dump the introspection drivers
        "!modules_guest",                           // Dump the guest modules
        "!hooks_gpa",                               // Dump the GPA hooks
        "!hooks_gva",                               // Dump the GVA hooks
        "!pts_dump",                                // Dump the PTS tree
        "!pfnlocks",                                // Dump the PFN locks
        "!transactions",                            // Dump the swap-in transactions
        "!stats",                                   // Dump the performance stats.
        "!icache",                                  // Dump the instruction cache content.
    };

    LOG("[CRITICAL] IntDbgEnterDebugger called from %s:%d\n", File, Line);
    LOG("Bug check generated! Dumps follow:\n");

    for (DWORD i = 0; i < ARRAYSIZE(commands); i++)
    {
        IntDbgProcessCommand(1, &commands[i]);
    }

    LOG("Bug check dump complete!\n");

    GlueEnterDebugger();
}


BOOLEAN
IntMatchPatternUtf8(
    _In_z_ const CHAR *Pattern,
    _In_z_ const CHAR *String,
    _In_ DWORD Flags
    )
{
    CHAR pat[255] = {0};

    if (Flags & INTRO_MATCH_TRUNCATED)
    {
        char *wild = strchr(Pattern, '*');

        if (wild && ((size_t)(wild - Pattern + 1) < sizeof(pat) - 1))
        {
            memcpy(pat, Pattern, (size_t)(wild - Pattern + 1));

            Pattern = pat;
        }
    }

    return glob_match_utf8(Pattern, String, TRUE, FALSE);
}


BOOLEAN
IntMatchPatternUtf16(
    _In_z_ const WCHAR *Pattern,
    _In_z_ const WCHAR *String,
    _In_ DWORD Flags
    )
{
    CHAR pat[255];
    CHAR str[255];

    utf16toutf8(pat, Pattern, sizeof(pat));
    utf16toutf8(str, String, sizeof(str));

    return IntMatchPatternUtf8(pat, str, Flags);
}


INTSTATUS
IntGuestUninitOnBugcheck(
    _In_ void const  *Detour
    )
{
    const char *func;

    UNREFERENCED_PARAMETER(Detour);

    if (!(gGuest.CoreOptions.Current & INTRO_OPT_BUGCHECK_CLEANUP))
    {
        return INT_STATUS_NOT_NEEDED_HINT;
    }

    func = gGuest.OSType == introGuestWindows ? "KiDisplayBlueScreen" : "panic";

    LOG("[INFO] Guest reached %s handler. Will attempt to uninit!\n", func);

    TRACE("[INFO] Will dump RIPs for all VCPUs\n");
    for (DWORD i = 0; i < gGuest.CpuCount; i++)
    {
        TRACE("[INFO][CPU %d] RIP @ 0x%016llx\n", i, gGuest.VcpuArray[i].Regs.Rip);
    }

    gGuest.BugCheckInProgress = TRUE;

    // Need this because, if we uninit on the same exit as this one, we'll end up calling
    // IntDetUninit -> we'll remove the jump back for the function and end up making the
    // current vcpu execute garbage code.
    return INT_STATUS_REMOVE_DETOUR_AND_SET_RIP;
}


BOOLEAN
IntPolicyProcIsBeta(
    _In_opt_ const void *Process,
    _In_ QWORD Flag
    )
{
    if (Process == NULL)
    {
        return FALSE;
    }

    if (gGuest.OSType == introGuestWindows)
    {
        return IntWinProcPolicyIsBeta(Process, Flag);
    }
    else if (gGuest.OSType == introGuestLinux)
    {
        return IntLixProcPolicyIsBeta(Process, Flag);
    }

    return FALSE;
}


BOOLEAN
IntPolicyCoreIsOptionBeta(
    _In_ QWORD Flag
    )
{
    if ((Flag & POLICY_KM_BETA_FLAGS) && gGuest.KernelBetaDetections)
    {
        return TRUE;
    }

    return (gGuest.CoreOptions.Beta & Flag) != 0;
}


BOOLEAN
IntPolicyProcIsFeedback(
    _In_opt_ const void *Process,
    _In_ QWORD Flag
    )
{
    if (Process == NULL)
    {
        return FALSE;
    }

    if (gGuest.OSType == introGuestWindows)
    {
        return IntWinProcPolicyIsFeedback(Process, Flag);
    }
    else if (gGuest.OSType == introGuestLinux)
    {
        return IntLixProcPolicyIsFeedback(Process, Flag);
    }

    return FALSE;
}


QWORD
IntPolicyGetProcProt(
    _In_opt_ const void *Process
    )
{
    if (NULL == Process)
    {
        return 0;
    }

    if (gGuest.OSType == introGuestWindows)
    {
        return IntWinProcGetProtOption(Process);
    }
    else if (gGuest.OSType == introGuestLinux)
    {
        return IntLixProcGetProtOption(Process);
    }

    return 0;
}


BOOLEAN
IntPolicyCoreTakeAction(
    _In_ QWORD Flag,
    _Inout_ INTRO_ACTION *Action,
    _Inout_ INTRO_ACTION_REASON *Reason
    )
{
    if (gGuest.CoreOptions.Feedback & Flag)
    {
        *Action = introGuestAllowed;
        *Reason = introReasonAllowedFeedback;

        return TRUE;
    }

    if (*Action == introGuestNotAllowed && (IntPolicyCoreIsOptionBeta(Flag)))
    {
        //
        // Don't change the action here, since in the alert we must know the original action & reason. The
        // only thing we do, is return a TRUE so the alert gets sent and the ALERT_FLAG_BETA will be set
        // inside it.
        //
        return TRUE;
    }

    return *Action == introGuestNotAllowed || *Reason == introReasonAllowedFeedback;
}


BOOLEAN
IntPolicyProcTakeAction(
    _In_ QWORD Flag,
    _In_ void const *Process,
    _Inout_ INTRO_ACTION *Action,
    _Inout_ INTRO_ACTION_REASON *Reason
    )
{
    if ((gGuest.CoreOptions.Feedback & IntPolicyGetProcProt(Process)) || IntPolicyProcIsFeedback(Process, Flag))
    {
        *Action = introGuestAllowed;
        *Reason = introReasonAllowedFeedback;

        return TRUE;
    }

    if (*Action == introGuestNotAllowed && (IntPolicyProcIsBeta(Process, Flag)))
    {
        // Don't change the action here, since in the alert we must know the original action & reason. The
        // only thing we do, is return a TRUE so the alert gets sent and the ALERT_FLAG_BETA will be set
        // inside it.
        return TRUE;
    }

    return *Action == introGuestNotAllowed || *Reason == introReasonAllowedFeedback;
}


BOOLEAN
IntPolicyProcForceBetaIfNeeded(
    _In_ QWORD Flag,
    _In_ void *Process,
    _Inout_ INTRO_ACTION *Action
    )
{
    if (*Action == introGuestNotAllowed && IntPolicyProcIsBeta(Process, Flag))
    {
        *Action = introGuestAllowed;

        return TRUE;
    }

    return FALSE;
}


BOOLEAN
IntPolicyCoreForceBetaIfNeeded(
    _In_ QWORD Flag,
    _Inout_ INTRO_ACTION *Action
    )
{
    if (*Action == introGuestNotAllowed && (IntPolicyCoreIsOptionBeta(Flag)))
    {
        *Action = introGuestAllowed;

        return TRUE;
    }

    return FALSE;
}


BOOLEAN
IntPolicyIsCoreOptionFeedback(
    _In_ QWORD Flag
    )
{
    return (gGuest.CoreOptions.Feedback & Flag) != 0;
}


char *
utf16_for_log(
    _In_z_ const WCHAR *WString
    )
{
    static char gLogBuffer[8][ONE_KILOBYTE];
    char *c;

    if (__unlikely(gCurLogBuffer >= ARRAYSIZE(gLogBuffer)))
    {
        ERROR("[ERROR] gCurLogBuffer = %d/%ld\n", gCurLogBuffer, ARRAYSIZE(gLogBuffer));
        return "";
    }

    c = utf16toutf8(gLogBuffer[gCurLogBuffer], WString, sizeof(gLogBuffer[0]));

    gCurLogBuffer++;

    return c;
}


INTSTATUS
IntReadString(
    _In_ QWORD StrGva,
    _In_ DWORD MinimumLength,
    _In_ BOOLEAN AnsiOnly,
    _Inout_ char **String,
    _Out_opt_ DWORD *StringLength
    )
{
    INTSTATUS status;
    DWORD strLen;
    QWORD gva;
    BOOLEAN found;
    char *str;

    if (introGuestLinux == gGuest.OSType && !IS_KERNEL_POINTER_LIX(StrGva))
    {
        return INT_STATUS_INVALID_PARAMETER_1;
    }
    else if (introGuestWindows == gGuest.OSType && !IS_KERNEL_POINTER_WIN(gGuest.Guest64, StrGva))
    {
        return INT_STATUS_INVALID_PARAMETER_1;
    }

    if (0 == MinimumLength || PAGE_SIZE < MinimumLength)
    {
        return INT_STATUS_INVALID_PARAMETER_2;
    }

    str = NULL;

    if (NULL != StringLength)
    {
        *StringLength = 0;
    }

    // In case the string splits in two pages
    strLen = 0;
    gva = StrGva;
    found = FALSE;
    do
    {
        int i = 0;
        DWORD remaining = PAGE_REMAINING(gva);
        char *p, *pStr;

        status = IntVirtMemMap(gva, remaining, gGuest.Mm.SystemCr3, 0, &pStr);
        if (!INT_SUCCESS(status))
        {
            ERROR("[ERROR] IntVirtMemMap failed for GVA 0x%016llx with size %x: 0x%08x\n",
                  gva, remaining, status);
            return status;
        }

        p = pStr;

        if (AnsiOnly)
        {
            i = is_str_ansi(p, remaining, MinimumLength);
        }
        else
        {
            while ((DWORD)i < remaining && *p)
            {
                i++;
                p++;
            }

            if ((DWORD)i < MinimumLength)
            {
                i = -1;
            }
        }

        IntVirtMemUnmap(&pStr);

        // this is no string
        if (-1 == i)
        {
            break;
        }

        strLen += i;

        // found in this page
        if (i < (int)remaining && i > 0)
        {
            found = TRUE;
            break;
        }

        // go to the next page (will be aligned from now on)
        gva += remaining;
    } while ((gva & PAGE_MASK) - (StrGva & PAGE_MASK) == PAGE_SIZE);

    if (!found)
    {
        return INT_STATUS_NOT_FOUND;
    }

    // From now on we include the NULL terminator.
    strLen++;

    if (NULL == *String)
    {
        str = HpAllocWithTag(strLen, IC_TAG_NAME);
        if (NULL == str)
        {
            return INT_STATUS_INSUFFICIENT_RESOURCES;
        }
    }
    else
    {
        str = *String;
    }

    // This is fine, because we check in the beginning that the StrGva is a kernel pointer.
    status = IntKernVirtMemRead(StrGva, strLen, str, NULL);
    if (!INT_SUCCESS(status))
    {
        ERROR("[ERROR] IntKernVirtMemRead failed for gva 0x%016llx, length %x: 0x%08x\n", StrGva, strLen, status);
        goto _clean_leave;
    }

    // We just allocated strLen + 1, so we are guaranteed to have space for the NULL terminator.
    str[strLen - 1] = 0;

    *String = str;

    if (NULL != StringLength)
    {
        *StringLength = strLen;
    }

_clean_leave:
    if (!INT_SUCCESS(status))
    {
        if (NULL != StringLength)
        {
            *StringLength = 0;
        }

        if (NULL != str)
        {
            HpFreeAndNullWithTag(&str, IC_TAG_NAME);
        }
    }

    return status;
}