Question

I have a GUI C# app that uses a console created with AllocConsole. It works under normal conditions, but when the app is started in debug mode under Visual Studio, all output ends up in the Visual Studio Output window. How do I stop that?

I am using C# 3.5 and Visual Studio Pro 2010.
The process hosting option is off.

Answer 1

My solution was to reset the standard streams to the newly created console myself. Here is the code:

public static class ConsoleHelper
{
    /// <summary>
    /// Allocates a console and resets the standard stream handles.
    /// </summary>
    public static void Alloc()
    {
        if (!AllocConsole())
            throw new Win32Exception();
        SetStdHandle(StdHandle.Output, GetConsoleStandardOutput());
        SetStdHandle(StdHandle.Input,  GetConsoleStandardInput());
    }

    private static IntPtr GetConsoleStandardInput()
    {
        var handle = CreateFile
            ( "CONIN$"
            , DesiredAccess.GenericRead | DesiredAccess.GenericWrite
            , FileShare.ReadWrite
            , IntPtr.Zero
            , FileMode.Open
            , FileAttributes.Normal
            , IntPtr.Zero
            );
        if (handle == InvalidHandleValue)
            throw new Win32Exception();
        return handle;
    }

    private static IntPtr GetConsoleStandardOutput()
    {
        var handle = CreateFile
            ( "CONOUT$"
            , DesiredAccess.GenericWrite | DesiredAccess.GenericWrite
            , FileShare.ReadWrite
            , IntPtr.Zero
            , FileMode.Open
            , FileAttributes.Normal
            , IntPtr.Zero
            );
        if (handle == InvalidHandleValue)
            throw new Win32Exception();
        return handle;
    }

    [DllImport("kernel32.dll", SetLastError = true)]
    private static extern bool AllocConsole();

    [DllImport("kernel32.dll")]
    private static extern bool SetStdHandle(StdHandle nStdHandle, IntPtr hHandle);

    [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)]
    private static extern IntPtr CreateFile
        (                               string         lpFileName
        , [MarshalAs(UnmanagedType.U4)] DesiredAccess  dwDesiredAccess
        , [MarshalAs(UnmanagedType.U4)] FileShare      dwShareMode
        ,                               IntPtr         lpSecurityAttributes
        , [MarshalAs(UnmanagedType.U4)] FileMode       dwCreationDisposition
        , [MarshalAs(UnmanagedType.U4)] FileAttributes dwFlagsAndAttributes
        ,                               IntPtr         hTemplateFile
        );

    [Flags]
    enum DesiredAccess : uint
    {
        GenericRead    = 0x80000000,
        GenericWrite   = 0x40000000,
        GenericExecute = 0x20000000,
        GenericAll     = 0x10000000
    }

    private enum StdHandle : int
    {
        Input  = -10,
        Output = -11,
        Error  = -12
    }

    private static readonly IntPtr InvalidHandleValue = new IntPtr(-1);
}