Table column loses data when sorting by relationship

Tables\Columns\TextColumn::make('dayparts.0.date')
    ->label('Datum')
    ->date('d-m-Y')
    ->sortable(
        query: fn ($query, $direction) => $query
            ->join('dayparts', function ($join) {
                $join->on('dayparts.booking_id', '=', 'bookings.id')
                    ->on('dayparts.id', DB::raw("(SELECT min(id) FROM dayparts WHERE dayparts.booking_id = bookings.id)"));
            })
            ->orderBy('date', $direction)
    ),