Builder Pattern as Ref Struct

public struct MyBuilder {
  public required int Id { get; set; }
  public string? Name { get; set; }
  public My Build() => new(this);
}
public static class MyBuilderExtensions {
  public ref MyBuilder WithId(this ref MyBuilder b, int id) {
    b.Id = id;
    return ref b;
  }
  public ref MyBuilder WithName(this ref MyBuilder b, string? name) {
    b.Name = name;
    return ref b;
  }
}
public static class Usage {
  public static My Works() {
    var builder = new MyBuilder() { Id = 1 };
    return builder.WithName("Test").Build();
  }
  public static My DoesNotWork() {
    // CS1510: A ref or out value must be an assignable variable
    return new MyBuilder() { Id = 1 }.WithName("Test").Build();
  }
}
public partial class My
{
  public required int Id { get => throw null; init => throw null; }
  public string Name { get => throw null; init => throw null; }
  // only if set by builder/init
  public bool TryGetName(out string name) => throw null;;
  [SetsRequiredMembers]
  public My(int id) => throw null;
  [SetsRequiredMembers]
  public My(MyBuilder b) => throw null;
}