When calling WinExec and passing the two variables to it (as per MSDN):

UINT WinExec(
  [in] LPCSTR lpCmdLine,
  [in] UINT   uCmdShow
);

I can’t seem to get a correct value returned in RAX and the command doesn’t execute – it comes back zero’d and the program fails. I am building this in Visual Studio 2022 using MASM and the entry point is main. Everything compiles fine but I think it might be the way the values are being passed.

    extrn ExitProcess: PROC
    extrn GetProcAddress: PROC
    extrn LoadLibraryA: PROC
    
    .data
        winExec db 'WinExec', 0
        kernel32 db 'kernel32.dll', 0
        cmd db "C:WindowsSystem32calc.exe", 0
        kernelAddr dq 0
        winExecAddr dq 0

    .code

    main PROC
        ; Setup the stack
        sub rsp, 28h   
        push rbp
        mov rbp, rsp

        ; Get a handle to Kernel32.dll
        lea rcx, kernel32                               ; Load the address of "kernel32.dll"
        call LoadLibraryA                               ; Load the DLL
        test rax, rax                                   ; Check if LoadLibraryA succeeded
        jz exit                                         ; Exit if it failed
        mov kernelAddr, rax                             ; Store address of Kernel32 in kernelAddr
    
        ; GetProcAddress of WinExec
        mov rcx, kernelAddr                             ; Load handle to kernel32.dll into rcx
        lea rdx, winExec                                ; Load the address of winExec string 'WinExec'
        call GetProcAddress                             ; Get the address of WinExec
        test rax, rax                                   ; Check if GetProcAddress succeeded
        jz exit                                         ; Exit if it failed
        mov winExecAddr, rax                            ; Store address of WinExec in winExecAddr
    
        ; Execute WinExec
        ; Call WinExec to execute the command
        ; UINT result = WinExec(cmd, SW_SHOWNORMAL);
        mov rdx, 1                                      ; uCmdShow = 1 (SW_SHOWNORMAL)
        lea rcx, [cmd]                                  ; Command line string (first parameter)
        call qword ptr [winExecAddr]                    ; Indirect call to WinExec
    
    exit:
        ; Clean up and return
        mov rsp, rbp                                    ; Restore original stack pointer
        pop rbp                                         ; Restore original base pointer
        add rsp, 28h                                    ; Clean up stack space reserved earlier
        ret


    
    main ENDP
    END

I have looked at other examples and I can’t see how mine differs. I have confirmed the address of everything is correct (Kernel32 and WinExec).

I built out the same functionality in C++ successfully and compared my WinExec call with that:

My WinExec call: N.B : I also tried using mov rcx, qword ptr [cmd] with no luck

    ; Call WinExec to execute the command
    ; UINT result = WinExec(cmd, SW_SHOWNORMAL);
    mov      rdx, 1                                       ; uCmdShow = 1 (SW_SHOWNORMAL)
    lea      rcx, [cmd]                                   ; Command line string (first parameter)
    call     qword ptr [winExecAddr]                      ; Indirect call to WinExec

The disassembled call in C++:

    // Call WinExec to execute the command
    UINT result = WinExec(cmd, SW_SHOWNORMAL);
    mov         edx,1  
    mov         rcx,qword ptr [cmd]  
    call        qword ptr [__imp_WinExec (07FF6C9011000h)]  
    mov         dword ptr [result],eax