C#•4mo ago

Help with LibraryImport and custom marshalling

What am I doing wrong here? I am getting a System.AccessViolationException when it tries to convert the first string (DeviceName) from unmanaged to managed.

[LibraryImport("user32.dll", EntryPoint = "EnumDisplayDevicesW", StringMarshalling = StringMarshalling.Utf16)]
[return: MarshalAs(UnmanagedType.Bool)]
public static partial bool EnumDisplayDevices(string? lpDevice, uint iDevNum, [MarshalUsing(typeof(DISPLAY_DEVICEMarshaller))] ref DISPLAY_DEVICE lpDisplayDevice, uint dwFlags);

[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
public struct DISPLAY_DEVICE
{
    [MarshalAs(UnmanagedType.U4)]
    public uint cb;
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 32)]
    public string? DeviceName;
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 128)]
    public string? DeviceString;
    [MarshalAs(UnmanagedType.U4)]
    public DisplayDeviceStateFlags StateFlags;
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 128)]
    public string? DeviceID;
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 128)]
    public string? DeviceKey;

    public static DISPLAY_DEVICE Create() => new DISPLAY_DEVICE { cb = (uint)Marshal.SizeOf<DISPLAY_DEVICE>() };
}

[Flags]
public enum DisplayDeviceStateFlags : uint
{
    AttachedToDesktop = 0x1,
    MultiDriver = 0x2,
    PrimaryDevice = 0x4,
    MirroringDriver = 0x8,
    VGACompatible = 0x10,
    Removable = 0x20,
    ModesPruned = 0x8000000,
    Remote = 0x4000000,
    Disconnect = 0x2000000
}

[CustomMarshaller(typeof(DISPLAY_DEVICE), MarshalMode.ManagedToUnmanagedRef, typeof(DISPLAY_DEVICEMarshaller))]
public static unsafe class DISPLAY_DEVICEMarshaller
{
    internal struct DISPLAY_DEVICE_Unmanaged
    {
        public uint cb;
        public ushort* DeviceName;
        public ushort* DeviceString;
        public uint StateFlags;
        public ushort* DeviceID;
        public ushort* DeviceKey;
    }

    public static DISPLAY_DEVICE ConvertToManaged(DISPLAY_DEVICE_Unmanaged unmanaged)
    {
        return new DISPLAY_DEVICE
        {
            cb = unmanaged.cb,
            DeviceName = Utf16StringMarshaller.ConvertToManaged(unmanaged.DeviceName),
            DeviceString = Utf16StringMarshaller.ConvertToManaged(unmanaged.DeviceString),
            StateFlags = (DisplayDeviceStateFlags)unmanaged.StateFlags,
            DeviceID = Utf16StringMarshaller.ConvertToManaged(unmanaged.DeviceID),
            DeviceKey = Utf16StringMarshaller.ConvertToManaged(unmanaged.DeviceKey)
        };
    }

    public static DISPLAY_DEVICE_Unmanaged ConvertToUnmanaged(DISPLAY_DEVICE managed)
    {
        return new DISPLAY_DEVICE_Unmanaged
        {
            cb = managed.cb,
            DeviceName = Utf16StringMarshaller.ConvertToUnmanaged(managed.DeviceName),
            DeviceString = Utf16StringMarshaller.ConvertToUnmanaged(managed.DeviceString),
            StateFlags = (uint)managed.StateFlags,
            DeviceID = Utf16StringMarshaller.ConvertToUnmanaged(managed.DeviceID),
            DeviceKey = Utf16StringMarshaller.ConvertToUnmanaged(managed.DeviceKey)
        };
    }

    public static unsafe void Free(DISPLAY_DEVICE_Unmanaged unmanaged)
    {
        Utf16StringMarshaller.Free(unmanaged.DeviceName);
        Utf16StringMarshaller.Free(unmanaged.DeviceString);
        Utf16StringMarshaller.Free(unmanaged.DeviceID);
        Utf16StringMarshaller.Free(unmanaged.DeviceKey);
    }
}

[LibraryImport("user32.dll", EntryPoint = "EnumDisplayDevicesW", StringMarshalling = StringMarshalling.Utf16)]
[return: MarshalAs(UnmanagedType.Bool)]
public static partial bool EnumDisplayDevices(string? lpDevice, uint iDevNum, [MarshalUsing(typeof(DISPLAY_DEVICEMarshaller))] ref DISPLAY_DEVICE lpDisplayDevice, uint dwFlags);

[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
public struct DISPLAY_DEVICE
{
    [MarshalAs(UnmanagedType.U4)]
    public uint cb;
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 32)]
    public string? DeviceName;
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 128)]
    public string? DeviceString;
    [MarshalAs(UnmanagedType.U4)]
    public DisplayDeviceStateFlags StateFlags;
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 128)]
    public string? DeviceID;
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 128)]
    public string? DeviceKey;

    public static DISPLAY_DEVICE Create() => new DISPLAY_DEVICE { cb = (uint)Marshal.SizeOf<DISPLAY_DEVICE>() };
}

[Flags]
public enum DisplayDeviceStateFlags : uint
{
    AttachedToDesktop = 0x1,
    MultiDriver = 0x2,
    PrimaryDevice = 0x4,
    MirroringDriver = 0x8,
    VGACompatible = 0x10,
    Removable = 0x20,
    ModesPruned = 0x8000000,
    Remote = 0x4000000,
    Disconnect = 0x2000000
}

[CustomMarshaller(typeof(DISPLAY_DEVICE), MarshalMode.ManagedToUnmanagedRef, typeof(DISPLAY_DEVICEMarshaller))]
public static unsafe class DISPLAY_DEVICEMarshaller
{
    internal struct DISPLAY_DEVICE_Unmanaged
    {
        public uint cb;
        public ushort* DeviceName;
        public ushort* DeviceString;
        public uint StateFlags;
        public ushort* DeviceID;
        public ushort* DeviceKey;
    }

    public static DISPLAY_DEVICE ConvertToManaged(DISPLAY_DEVICE_Unmanaged unmanaged)
    {
        return new DISPLAY_DEVICE
        {
            cb = unmanaged.cb,
            DeviceName = Utf16StringMarshaller.ConvertToManaged(unmanaged.DeviceName),
            DeviceString = Utf16StringMarshaller.ConvertToManaged(unmanaged.DeviceString),
            StateFlags = (DisplayDeviceStateFlags)unmanaged.StateFlags,
            DeviceID = Utf16StringMarshaller.ConvertToManaged(unmanaged.DeviceID),
            DeviceKey = Utf16StringMarshaller.ConvertToManaged(unmanaged.DeviceKey)
        };
    }

    public static DISPLAY_DEVICE_Unmanaged ConvertToUnmanaged(DISPLAY_DEVICE managed)
    {
        return new DISPLAY_DEVICE_Unmanaged
        {
            cb = managed.cb,
            DeviceName = Utf16StringMarshaller.ConvertToUnmanaged(managed.DeviceName),
            DeviceString = Utf16StringMarshaller.ConvertToUnmanaged(managed.DeviceString),
            StateFlags = (uint)managed.StateFlags,
            DeviceID = Utf16StringMarshaller.ConvertToUnmanaged(managed.DeviceID),
            DeviceKey = Utf16StringMarshaller.ConvertToUnmanaged(managed.DeviceKey)
        };
    }

    public static unsafe void Free(DISPLAY_DEVICE_Unmanaged unmanaged)
    {
        Utf16StringMarshaller.Free(unmanaged.DeviceName);
        Utf16StringMarshaller.Free(unmanaged.DeviceString);
        Utf16StringMarshaller.Free(unmanaged.DeviceID);
        Utf16StringMarshaller.Free(unmanaged.DeviceKey);
    }
}

Calling code:

public static DISPLAY_DEVICE GetPrimaryDisplay()
{
    uint id = 0;
    while (true)
    {
        DISPLAY_DEVICE d = DISPLAY_DEVICE.Create();

        bool done = EnumDisplayDevices(null, id, ref d, 0);

        if ((d.StateFlags & DisplayDeviceStateFlags.PrimaryDevice) == DisplayDeviceStateFlags.PrimaryDevice)
        {
            return d;
        }

        if (done)
            break;

        id++;
    }

    return default;
}

public static DISPLAY_DEVICE GetPrimaryDisplay()
{
    uint id = 0;
    while (true)
    {
        DISPLAY_DEVICE d = DISPLAY_DEVICE.Create();

        bool done = EnumDisplayDevices(null, id, ref d, 0);

        if ((d.StateFlags & DisplayDeviceStateFlags.PrimaryDevice) == DisplayDeviceStateFlags.PrimaryDevice)
        {
            return d;
        }

        if (done)
            break;

        id++;
    }

    return default;
}

5 Replies

sibber•4mo ago

why the custom marshaller

reflectronic•4mo ago

because LibraryImport does not generate marshallers for structs yet

    public static DISPLAY_DEVICE Create() => new DISPLAY_DEVICE { cb = (uint)Marshal.SizeOf<DISPLAY_DEVICE>() };

    public static DISPLAY_DEVICE Create() => new DISPLAY_DEVICE { cb = (uint)Marshal.SizeOf<DISPLAY_DEVICE>() };

this is not correct, you must not mix Marshal.SizeOf (or any other Marshal APIs) with LibraryImport. you want to use sizeof(DISPLAY_DEVICE_Unmanaged) your DISPLAY_DEVICE_Unmanaged is not correct either. DISPLAY_DEVICE_Unmanaged is what gets sent to unmanaged code, but doesn't match the unmanaged definition of DISPLAY_DEVICE--the C definition is using an inline array (that's what the UnmanagedType.ByValTStr, SizeConst = 32 means) but you are just using a pointer so you would need to have public fixed ushort DeviceName[16];. and then copy the string into that buffer directly. the LibraryImport marshallers are not useful for that i would consider not using LibraryImport for this tbh. for complex structs with members that need marshalling it is a lot of work at the moment. you can keep using it for other methods but this one it'd be easier to keep as DllImport

sibber•4mo ago

id just use DllImport then lol rather than having to deal with all this why?

reflectronic•4mo ago

because Marshal uses the built-in marshalling system it will give different answers than what you need if you are using LibraryImport it only 'works' in this case because there are MarshalAs on the struct (which LibraryImport does not use yet) and even then, in this case, it does not give the size of the struct that LibraryImport marshals, because the struct that it actually marshals to (DISPLAY_DEVICE_Unmanaged) is declared differently (incorrectly for this API, but nonetheless)

Petris•4mo ago

You can find a correct definition without marshalling here for reference https://github.com/terrafx/terrafx.interop.windows/blob/39a0cf1a35f6e17e8c9f50f926a03596c702b5e1/sources/Interop/Windows/Windows/um/WinUser/Windows.cs#L2915

Gaming

Programming

Help with LibraryImport and custom marshalling