VirtualAllocExNuma

Home
Back To Tips Page

The example given in the MSDN in the article titled "Allocating Memory from a NUMA Node", which is linked to from the VirtualAllocExNuma description, represents several extremely bad coding practices.  It looks like a beginner wrote it.  This project cleans up the code and makes it look like it was written by a programmer with more than six weeks' experience.  For critique of the original code, see this article.

The output from the program is shown below.  This represents one test on one snapshot of one Vista machine.

I changed the specs a bit; the original wanted a command line argument of the number of bytes to allocate, expressed in hexadecimal.  This certainly made no sense as a user interface.  I changed it to take a quantity of pages expressed as a decimal number.

The subroutines shown were adapted to work in my NUMA Explorer.

Note that although there was a desire to allocate 4096 pages for CPU1 on its preferred node (node 0) in fact only 2959 pages could be allocated on that node; the remainder were allocated on node 1.  I have seen traces where the blocks were often under 10 pages and there was interleaving for hundreds of blocks that were piecewise on different nodes.  These are far too long and boring to paste into a Web page in their entirety. But here's part of one

Compare the two side-by-side.  Note that to reduce the column width, a few comments and long lines of both sources have been re-wrapped.

Original Code Rewritten Code
#include <windows.h>
#include <stdlib.h>
#include <stdio.h>
#include <psapi.h>

SIZE_T AllocationSize;
DWORD  PageSize;



void DumpNumaNodeInfo (PVOID Buffer, SIZE_T Size);



































































void __cdecl wmain (int argc, const wchar_t* argv[])
{
    ULONG HighestNodeNumber;
    ULONG NumberOfProcessors;













    if (argc > 1)
    {
        (void)swscanf (argv[1], L"%Ix", &AllocationSize);
    }

    if (AllocationSize == 0)
    {
        AllocationSize = 16*1024*1024;
    }

    //
    // Get the number of processors and system page size.
    //

    SYSTEM_INFO SystemInfo;
    GetSystemInfo (&SystemInfo);
    NumberOfProcessors = SystemInfo.dwNumberOfProcessors;
    PageSize = SystemInfo.dwPageSize;





    //
    // Get the highest node number.
    //








    if (TRUE != GetNumaHighestNodeNumber (&HighestNodeNumber))
    {
        printf ("GetNumaHighestNodeNumber failed: 0x%x\r\n", GetLastError());
        goto Exit;
    }


    if (HighestNodeNumber == 0)
    {
        puts ("Not a NUMA system - exiting");
        goto Exit;                           
    }

    //
    // Allocate array of pointers to memory blocks.
    //

    PVOID* Buffers = (PVOID*) malloc (sizeof(PVOID)*NumberOfProcessors);

    if (Buffers == NULL)                            
    {                                               
        puts ("Allocating array of buffers failed");
        goto Exit;                                  
    }                                               

    ZeroMemory (Buffers, sizeof(PVOID)*NumberOfProcessors);

    //
    // For each processor, get its associated NUMA node 
    // and allocate some memory from it.
    //

    for (UCHAR i = 0; i < NumberOfProcessors; i++)
    {
        UCHAR NodeNumber;

        if (TRUE != GetNumaProcessorNode (i, &NodeNumber))                   
        {                                                                    
            printf ("GetNumaProcessorNode failed: 0x%x\r\n", GetLastError());
            goto Exit;                                                       
        }                                                                    




        printf ("CPU %u: node %u\r\n", (ULONG)i, NodeNumber);

        PCHAR Buffer = (PCHAR)VirtualAllocExNuma(
            GetCurrentProcess(),
            NULL,
            AllocationSize,
            MEM_RESERVE | MEM_COMMIT,
            PAGE_READWRITE,
            NodeNumber
        );

        if (Buffer == NULL)
        {
            printf ("VirtualAllocExNuma failed: 0x%x, node %u\r\n",  
                                         GetLastError(), NodeNumber);
            goto Exit;                                               
        }


        PCHAR BufferEnd = Buffer + AllocationSize - 1;
        SIZE_T NumPages = ((SIZE_T)BufferEnd)/PageSize 
                              - ((SIZE_T)Buffer)/PageSize + 1;

        puts ("Allocated virtual memory:");                    
        printf ("%p - %p (%6Iu pages), preferred node %u\r\n", 
                      Buffer, BufferEnd, NumPages, NodeNumber);


        Buffers[i] = Buffer;

        //
        // At this point, virtual pages are allocated but no valid physical
        // pages are associated with them yet.
        //
        // The FillMemory call below will touch every page in the buffer, 
        // faulting them into our working set. When this happens physical 
        // pages will be allocated from the preferred node we specified 
        // in VirtualAllocExNuma, or any node
        // if the preferred one is out of pages.
        //

        FillMemory (Buffer, AllocationSize, 'x');

        //
        // Check the actual node number for the physical pages that are still valid
        // (if system is low on physical memory, 
        // some pages could have been trimmed already).
        //

        DumpNumaNodeInfo (Buffer, AllocationSize);

        puts("");
    }

Exit:
    if (Buffers != NULL)                                 
    {                                                    
        for (UINT i = 0; i < NumberOfProcessors; i++)    
        {                                                
            if (Buffers[i] != NULL)                      
            {                                            
                VirtualFree (Buffers[i], 0, MEM_RELEASE);
            }                                            
        }                                                
                                                         
        free (Buffers);                                  
    }                                                    

}














void DumpRegion (PVOID StartPtr, PVOID EndPtr, BOOL Valid, DWORD Node)
{
    DWORD_PTR StartPage = ((DWORD_PTR)StartPtr)/PageSize;
    DWORD_PTR EndPage   = ((DWORD_PTR)EndPtr)/PageSize;
    DWORD_PTR NumPages  = (EndPage - StartPage) + 1;

    if (Valid == TRUE)                                      
    {                                                       
        printf ("%p - %p (%6Iu pages): node %u\r\n",        
                StartPtr, EndPtr, NumPages, Node);          
    }                                                       
    else                                                    
    {                                                       
        printf ("%p - %p (%6Iu pages): no valid pages\r\n", 
                StartPtr, EndPtr, NumPages);                
    }                                                       
}












void DumpNumaNodeInfo (PVOID Buffer, SIZE_T Size)
{
    DWORD_PTR StartPage = ((DWORD_PTR)Buffer)/PageSize;
    DWORD_PTR EndPage   = ((DWORD_PTR)Buffer + Size - 1)/PageSize;
    DWORD_PTR NumPages  = (EndPage - StartPage) + 1;

    PCHAR StartPtr = (PCHAR)(PageSize*StartPage);

    puts ("Checking NUMA node:");

    PPSAPI_WORKING_SET_EX_INFORMATION WsInfo = (PPSAPI_WORKING_SET_EX_INFORMATION)
        malloc (NumPages*sizeof(PSAPI_WORKING_SET_EX_INFORMATION));               

    if (WsInfo == NULL)                                                                  
    {                                                                                    
        puts ("Could not allocate array of PSAPI_WORKING_SET_EX_INFORMATION structures");
        return;                                                                          
    }                                                                                    

    for (DWORD_PTR i = 0; i < NumPages; i++)
    {
        WsInfo[i].VirtualAddress = StartPtr + i*PageSize;
    }

    BOOL bResult = QueryWorkingSetEx(
        GetCurrentProcess(),
        WsInfo,
        NumPages*sizeof(PSAPI_WORKING_SET_EX_INFORMATION)
    );

    if (bResult != TRUE)                                              
    {                                                                 
        printf ("QueryWorkingSetEx failed: 0x%x\r\n", GetLastError());
        free (WsInfo);                                                
        return;                                                       
    }                                                                 

    PCHAR RegionStart;
    BOOL  RegionIsValid;
    DWORD RegionNode;

    for (DWORD_PTR i = 0; i < NumPages; i++)
    {
        PCHAR Address = (PCHAR)WsInfo[i].VirtualAddress;
        BOOL  IsValid = WsInfo[i].u1.VirtualAttributes.Valid;
        DWORD Node    = WsInfo[i].u1.VirtualAttributes.Node;

        if (i == 0)
        {
            RegionStart   = Address;
            RegionIsValid = IsValid;
            RegionNode    = Node;
        }

        if (IsValid != RegionIsValid || Node != RegionNode)
        {
            DumpRegion (RegionStart, Address - 1, RegionIsValid, RegionNode);

            RegionStart   = Address;
            RegionIsValid = IsValid;
            RegionNode    = Node;
        }

        if (i == (NumPages - 1))
        {
            DumpRegion (RegionStart, Address + PageSize - 1, IsValid, Node);
        }
    }

    free (WsInfo);
}
#include "stdafx.h"                                         
// Moved all standard includes to stdafx.h                  
#pragma comment(lib, "psapi.lib")                           
// Make sure it links by putting the lib specifications here


/* pseudo-const */ DWORD  PageSize;  // written once at startup and never changed
// Mark the above as write-once; it is the only global variable 
// that could possibly make sense

void DumpNumaNodeInfo (PVOID Buffer, SIZE_T Size);

/****************************************************************************
*                                 ReportError                                
* Inputs:                                                                    
*       LPCTSTR msg: First part of message                                   
*       DWORD err: Error code                                                
* Result: void                                                               
*                                                                            
* Effect:                                                                    
*       Issues the error message                                             
****************************************************************************/
// All functions get a header so the code is readable and well-documented    
static void ReportError(LPCTSTR msg, DWORD err)                                     
    {                                                                               
     LPTSTR errmsg;                                                                 
     if(::FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
                        NULL, // source                                             
                        err,  // Message ID                                         
                        0,    // Default language ID                                
                        (LPTSTR)&errmsg, // buffer                                  
                        0,    // size                                               
                        NULL) == 0) // argument list                                
        { /* No message code found */                                               
         _tprintf(_T("%s: Error %d (0x%08x)\n"), msg, err, err);                    
         return;                                                                    
        } /* No message code found */                                               
     else                                                                           
        { /* has message */                                                         
         _tprintf(_T("%s: %s\n"), msg, errmsg);                                     
         LocalFree(errmsg);                                                         
         return;                                                                    
        } /* has message */                                                         
    } // ReportError                                                                
// The above code is so simple that it should be hyperlinked from every example     
// that needs to display an error                                                   

/****************************************************************************
*                              class BufferInfo                              
****************************************************************************/

class BufferInfo {                                                           
    public:                                                                  
       BufferInfo() { Buffer = NULL; }                                       
       virtual ~BufferInfo() {                                               
                              if(Buffer != NULL)                             
                                 ::VirtualFree(Buffer, 0, MEM_RELEASE);      
                             }                                               
    PVOID Buffer;                                                            
};                                                                           
// There is no need to zero fields and have deletion loops when C++          
// makes this so easy!  Note the example was written in C++!                 

/****************************************************************************
*                                   _tmain                                   
* Inputs:                                                                    
*       int argc: Number of pages to allocate                                
*       _TCHAR * argv[]: Array of arguments                                  
* Result: int                                                                
*       0 if successful                                                      
*       1 if error                                                           
* Effect:                                                                    
*       The program                                                          
* Argv:                                                                      
*       [1] If present, the number of pages to allocate, as a decimal number 
*           If argc < 2, defaults to 4096 pages                              
****************************************************************************/

int _tmain(int argc, _TCHAR* argv[])     
// This is the correct prototype for main
   {
    ULONG NumberOfProcessors; 
    SIZE_T AllocationSize = 0;                                  
    // Since this is not used except in this function, there was
    // no reason to make it a global variable                   

    //
    // Get the number of processors and system page size.
    //

    SYSTEM_INFO SystemInfo;
    GetSystemInfo (&SystemInfo);
    NumberOfProcessors = SystemInfo.dwNumberOfProcessors;
    PageSize = SystemInfo.dwPageSize;

    if (argc > 1)
       {
        AllocationSize = _ttoi(argv[1]) * PageSize;
       }

    if (AllocationSize == 0)
       {
        // If not specified, default to 4096 pages
        AllocationSize = 4096 * PageSize;         
       }













    //****************************************************************
    // Get the highest node number.
    //****************************************************************

    ULONG HighestNodeNumber;
    // Since this variable is not needed until the next line  
    // it makes no sense to have declared it at the block head

    if (!GetNumaHighestNodeNumber (&HighestNodeNumber))         
       {
        DWORD err = GetLastError();                             
        ReportError(_T("GetNumaHighestNodeNumber failed"), err);
        return 1;                                               
       }

    if (HighestNodeNumber == 0)
       {
        _putts (_T("Not a NUMA system - exiting\n"));
        return 1;                                    
       }

    //****************************************************************
    // Allocate array of pointers to memory blocks.
    //****************************************************************

    std::vector Buffers;
    Buffers.resize(NumberOfProcessors);

    // Avoid any need to delete objects by letting C++ delete them
    // This also guarantees that the error caused by the goto     
    // bypassing initialization cannot occur                      




    //****************************************************************
    // For each processor, get its associated NUMA node 
    // and allocate some memory from it.
    //****************************************************************

    for (UCHAR i = 0; i < NumberOfProcessors; i++)
       { /* Per-processor allocation */
        UCHAR NodeNumber;

        if (!GetNumaProcessorNode (i, &NodeNumber))             
           {                                                    
            DWORD err = ::GetLastError();                       
            ReportError(_T("GetNumaProcessorNode failed"), err);
            return 1;                                           
           }                                                    

        _tprintf (
          _T("----------------------------------------------------------------\n"));
        _tprintf (_T("CPU %u: node %u\n"), (ULONG)i, NodeNumber);                   

        PCHAR Buffer = (PCHAR)VirtualAllocExNuma(
           GetCurrentProcess(),
           NULL,
           AllocationSize,
           MEM_RESERVE | MEM_COMMIT,
           PAGE_READWRITE,
           NodeNumber
                                                );

        if (Buffer == NULL)
           {
            DWORD err = ::GetLastError();                     
            _tprintf(_T("Node %u, "), NodeNumber);            
            ReportError(_T("VirtualAllocExNuma failed"), err);
            return 1;                                         
           }

        PCHAR BufferEnd = Buffer + AllocationSize - 1;
        SIZE_T NumPages = ((SIZE_T)BufferEnd)/PageSize 
                              - ((SIZE_T)Buffer)/PageSize + 1;

        _tprintf (
           _T("Allocated virtual memory: %p - %p (%6Iu pages), preferred node %u\n"), 
                              Buffer, BufferEnd, NumPages, NodeNumber);


        Buffers[i].Buffer = Buffer;

        //****************************************************************
        // At this point, virtual pages are allocated but no valid physical
        // pages are associated with them yet.
        //
        // The FillMemory call below will touch every page in the buffer, 
        // faulting them into our working set. When this happens physical 
        // pages will be allocated from the preferred node we specified 
        // in VirtualAllocExNuma, or any node
        // if the preferred one is out of pages.
        //****************************************************************

        FillMemory (Buffers[i].Buffer, AllocationSize, 'x');

        //****************************************************************
        // Check the actual node number for the physical pages that are still valid
        // (if system is low on physical memory, 
        // some pages could have been trimmed already).
        //****************************************************************

        DumpNumaNodeInfo (Buffers[i].Buffer, AllocationSize);


       } /* Per-processor allocation */














    return 0;
   } // _tmain

/****************************************************************************
*                                 DumpRegion                                 
* Inputs:                                                                    
*       PVOID StartPtr: Starting address of the region                       
*       PVOID EndPtr: Ending address of the region                           
*       BOOL Valid: TRUE if the region is valid, FALSE otherwise             
*       DWORD Node: NUMA node this applies to                                
* Result: void                                                               
*                                                                            
* Effect:                                                                    
*       Prints out the message                                               
****************************************************************************/

void DumpRegion (PVOID StartPtr, PVOID EndPtr, BOOL Valid, DWORD Node)
   {
    DWORD_PTR StartPage = ((DWORD_PTR)StartPtr)/PageSize;
    DWORD_PTR EndPage   = ((DWORD_PTR)EndPtr)/PageSize;
    DWORD_PTR NumPages  = (EndPage - StartPage) + 1;

    if (Valid)                                                  
       {                                                        
        _tprintf (_T("%p - %p (%6Iu pages): node %u\n"),        
                  StartPtr, EndPtr, NumPages, Node);            
       }                                                        
    else                                                        
       {                                                        
        _tprintf (_T("%p - %p (%6Iu pages): no valid pages\n"), 
                  StartPtr, EndPtr, NumPages);                  
       }                                                        
   } // DumpRegion

/****************************************************************************
*                              DumpNumaNodeInfo                              
* Inputs:                                                                    
*       PVOID Buffer: A pointer to a block of memory                         
*       SIZE_T Size: The size of the block of memory, in bytes               
* Result: void                                                               
*                                                                            
* Effect:                                                                    
*       Displays how the memory is spread across the NUMA nodes              
****************************************************************************/

void DumpNumaNodeInfo (PVOID Buffer, SIZE_T Size)
   {
    DWORD_PTR StartPage = ((DWORD_PTR)Buffer)/PageSize;
    DWORD_PTR EndPage   = ((DWORD_PTR)Buffer + Size - 1)/PageSize;
    DWORD_PTR NumPages  = (EndPage - StartPage) + 1;

    PCHAR StartPtr = (PCHAR)(PageSize*StartPage);

    _tprintf(_T("Checking NUMA node for %d pages:\n"), NumPages);

    std::vector WsInfo;     
    WsInfo.resize(NumPages);







    for (DWORD_PTR i = 0; i < NumPages; i++)
       {
        WsInfo[i].VirtualAddress = StartPtr + (i * PageSize);
       }

    BOOL bResult = QueryWorkingSetEx(
        GetCurrentProcess(),
        &WsInfo[0],
        NumPages * sizeof(PSAPI_WORKING_SET_EX_INFORMATION)
    );

    if (!bResult)                                         
       {                                                  
        DWORD err = ::GetLastError();                     
        ReportError (_T("QueryWorkingSetEx failed"), err);
        return;                                           
       }                                                  

    PCHAR RegionStart;
    BOOL  RegionIsValid;
    DWORD RegionNode;

    for (DWORD_PTR i = 0; i < NumPages; i++)
       { /* Show runs */
        PCHAR Address = (PCHAR)WsInfo[i].VirtualAddress;
        BOOL  IsValid = WsInfo[i].VirtualAttributes.Valid;
        DWORD Node    = WsInfo[i].VirtualAttributes.Node;

        if (i == 0)
           { /* first run */
            RegionStart   = Address;
            RegionIsValid = IsValid;
            RegionNode    = Node;
           } /* first run */

        if (IsValid != RegionIsValid || Node != RegionNode)
           { /* run change */
            DumpRegion (RegionStart, Address - 1, RegionIsValid, RegionNode);

            RegionStart   = Address;
            RegionIsValid = IsValid;
            RegionNode    = Node;
           } /* run change */

        if (i == (NumPages - 1))
           { /* end of runs */
            DumpRegion (RegionStart, Address + PageSize - 1, IsValid, Node);
           } /* end of runs */
       } /* Show runs */


   } // DumpNumaNodeInfo

download.gif (1234 bytes) VS2008 source code for the Microsoft example (rewritten to look like professional code)

 

[Dividing Line Image]

The views expressed in these essays are those of the author, and in no way represent, nor are they endorsed by, Microsoft.

Send mail to newcomer@flounder.com with questions or comments about this web site.
Copyright 2009, Joseph M. Newcomer, All Rights Reserved.
Last modified: May 14, 2011