Optimising read-and-process loop

await foreach (var res in GetSerializedUnoptimisedResponses()) {
    if (res.resp is null) continue;
}

List<string> serialisedKeys = new(4000000);
await foreach (var res in GetSerializedUnoptimisedResponses()) {
    if (res.resp is null) continue;
    serialisedKeys.Add(res.key);
    if (serialisedKeys.Count % 1000 == 0) _ = Console.Out.WriteAsync($"{serialisedKeys.Count}\r");
}

var chunkSize = serialisedKeys.Count / Environment.ProcessorCount;
var chunks = serialisedKeys.Chunk(chunkSize+1).Select(x => (x.First(), x.Length)).ToList();
Console.WriteLine($"Got {chunks.Count} chunks:");
foreach (var chunk in chunks) {
    Console.WriteLine($"Chunk {chunk.Item1} with length {chunk.Length}");
}

private async Task<List<string>> GetSerializedUnoptimisedKeysParallel(string start = "init") {
    ConcurrentDictionary<string, string> pairs = [];
    var unoptimisedKeys = (await storageProvider.GetAllKeysAsync()).Where(static x => !x.StartsWith("old/")).ToFrozenSet();
    await Parallel.ForEachAsync(unoptimisedKeys, async (key, _) => {
        var data = await storageProvider.LoadObjectAsync<SyncResponse>(key, SyncResponseSerializerContext.Default.SyncResponse);
        if (data is null) return;
        pairs.TryAdd(key, data.NextBatch);
    });

    var serializedKeys = new List<string>();
    var currentKey = start;
    while (pairs.TryGetValue(currentKey, out var nextKey)) {
        serializedKeys.Add(currentKey);
        currentKey = nextKey;
    }

    return serializedKeys;
}

fail: ModerationClient.ViewModels.ClientViewModel[0]
      Error running client view model.
      System.AggregateException: One or more errors occurred. ('s' is an invalid start of a value. Path: $ | LineNumber: 0 | BytePositionInLine: 0.)
       ---> System.Text.Json.JsonException: 's' is an invalid start of a value. Path: $ | LineNumber: 0 | BytePositionInLine: 0.
       ---> System.Text.Json.JsonReaderException: 's' is an invalid start of a value. LineNumber: 0 | BytePositionInLine: 0.
         at System.Text.Json.ThrowHelper.ThrowJsonReaderException(Utf8JsonReader& json, ExceptionResource resource, Byte nextByte, ReadOnlySpan`1 bytes)
         at System.Text.Json.Utf8JsonReader.ConsumeValue(Byte marker)
         at System.Text.Json.Utf8JsonReader.ReadFirstToken(Byte first)
         at System.Text.Json.Utf8JsonReader.ReadSingleSegment()
         at System.Text.Json.Serialization.JsonConverter`1.ReadCore(Utf8JsonReader& reader, T& value, JsonSerializerOptions options, ReadStack& state)
         --- End of inner exception stack trace ---
         at System.Text.Json.ThrowHelper.ReThrowWithPath(ReadStack& state, JsonReaderException ex)
         at System.Text.Json.Serialization.JsonConverter`1.ReadCore(Utf8JsonReader& reader, T& value, JsonSerializerOptions options, ReadStack& state)
         at System.Text.Json.Serialization.Metadata.JsonTypeInfo`1.Deserialize(Stream utf8Json)
         at ModerationClient.Services.FileStorageProvider.LoadObjectAsync[T](String key, JsonTypeInfo`1 jsonTypeInfo) in /home/Rory/git/matrix/ModerationClient/ModerationClient/Services/FileStorageProvider.cs:line 64
         at LibMatrix.Helpers.SyncStateResolver.<>c__DisplayClass31_0.<<GetSerializedUnoptimisedKeysParallel>b__0>d.MoveNext() in /home/Rory/git/matrix/ModerationClient/LibMatrix/LibMatrix/Helpers/SyncStateResolver.cs:line 85
      --- End of stack trace from previous location ---
         at System.Threading.Tasks.Parallel.<>c__53`1.<<ForEachAsync>b__53_0>d.MoveNext()
      --- End of stack trace from previous location ---
         at LibMatrix.Helpers.SyncStateResolver.GetSerializedUnoptimisedKeysParallel(String start) in /home/Rory/git/matrix/ModerationClient/LibMatrix/LibMatrix/Helpers/SyncStateResolver.cs:line 84
         at LibMatrix.Helpers.SyncStateResolver.OptimiseStore(Action`2 progressCallback) in /home/Rory/git/matrix/ModerationClient/LibMatrix/LibMatrix/Helpers/SyncStateResolver.cs:line 124
         at ModerationClient.ViewModels.ClientViewModel.Run() in /home/Rory/git/matrix/ModerationClient/ModerationClient/ViewModels/ClientViewModel.cs:line 99
         --- End of inner exception stack trace ---

private async Task<List<string>> GetSerializedUnoptimisedKeysParallel(string start = "init") {
    ConcurrentDictionary<string, string> pairs = [];
    var unoptimisedKeys = (await storageProvider.GetAllKeysAsync()).Where(static x => !x.Contains('/')).ToFrozenSet();
    await Parallel.ForEachAsync(unoptimisedKeys, async (key, _) => {
        try {
            var data = await storageProvider.LoadObjectAsync<SyncResponse>(key, SyncResponseSerializerContext.Default.SyncResponse);
            if (data is null) return;
            pairs.TryAdd(key, data.NextBatch);
        }
        catch (Exception e) {
            Console.WriteLine($"Failed to read {key}:");
            throw;
        }
    });

    var serializedKeys = new List<string>();
    var currentKey = start;
    while (pairs.TryGetValue(currentKey, out var nextKey)) {
        serializedKeys.Add(currentKey);
        currentKey = nextKey;
    }

    return serializedKeys;
}

// Usage:
List<string> serialisedKeys = await GetSerializedUnoptimisedKeysParallel();

var chunkSize = serialisedKeys.Count / Environment.ProcessorCount;
var chunks = serialisedKeys.Chunk(chunkSize+1).Select(x => (x.First(), x.Length)).ToList();
Console.WriteLine($"Got {chunks.Count} chunks:");
foreach (var chunk in chunks) {
    Console.WriteLine($"Chunk {chunk.Item1} with length {chunk.Length}");
}