Worse performance with pointers?

[StructLayout(LayoutKind.Explicit, Size = 16)]
public readonly partial struct MyCustomStruct
{

    [FieldOffset(0)]
    private readonly ulong Ulong0;
    [FieldOffset(8)]
    private readonly ulong Ulong1;

    private MyCustomStruct(in ulong ulong0, in ulong ulong1)
    {
        this.Ulong0 = ulong0;
        this.Ulong1 = ulong1;
    }

    public unsafe MyCustomStruct SetBitUnsafe(in int bitPosition)
    {
        ulong mask = 1UL << (bitPosition % 64);

        // copy to new val so we do not modify original
        MyCustomStruct returnVal = this;

        ulong* ptr = &(returnVal.Ulong0);

        ptr[(bitPosition / 64)] |= mask;

        return returnVal;
    }

    public MyCustomStruct SetBitSafe(in int bitPosition)
    {
        int ulongIndex = bitPosition / 64;
        int bitIndex = bitPosition % 64;
        ulong mask = (1UL << bitIndex);
        ulong ulong0 = Ulong0;
        ulong ulong1 = Ulong1;

        switch (ulongIndex)
        {
            case 0:
                ulong0 |= mask;
                break;
            case 1:
                ulong1 |= mask;
                break;
            default:
                throw new ArgumentOutOfRangeException(nameof(bitPosition), "Bit position is out of range.");
        }

        return new MyCustomStruct(in ulong0, in ulong1);
    }
}


//Benchmarks
[Benchmark]
public MyCustomStruct SetBitSafe_Test()
{
    MyCustomStruct returnVal = new();
    for (int i = 0; i < 10000; i++)
    {
        returnVal = new MyCustomStruct().SetBitSafe(67);
    }

    return returnVal;
}

[Benchmark]
public MyCustomStruct SetBitUnsafe_Test()
{
    MyCustomStruct returnVal = new();
    for (int i = 0; i < 10000; i++)
    {
        returnVal = new MyCustomStruct().SetBitUnsafe(67);
    }

    return returnVal;
}