Как исправить ошибку "Specified value has invalid Control characters" при скачивании файла через HttpClient?

Question

Есть следующий код:

                static readonly HttpClient client = new HttpClient(handler);
        private async Task RequestAsync(Guid ActId, string FileId, string FileName, string FileFolderName, string Description, string UploaderName, string UploadDate)
        {
            client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.5");
            client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate");
            client.DefaultRequestHeaders.Add("Accept", "text/html, application/xhtml+xml, image/jxr, */*");
            try
            {
                var url = FileLinkIBMFileNet.Replace("{file_id}", FileId).Replace("{file_name}", FileName);
                var authToken = Encoding.ASCII.GetBytes($"{LoginIBMFilenet}:{PasswordIBMFileNet}");
                client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic",
                Convert.ToBase64String(authToken));
                byte[] tmpFileByte = await client.GetByteArrayAsync(url);
                InsertFile(ActId, FileName, FileFolderName, tmpFileByte, Description, UploaderName, UploadDate, FileId);
        }
        catch (Exception ex) {
            string message = string.Format("Error in RequestAsync\nMessage ---\n{0} \nSource ---\n{1} \nStackTrace ---\n{2} \nActId ---\n{3} \nFileId ---\n{4} \nFilename ---\n{5}", ex.Message, ex.Source, ex.StackTrace, ActId, FileId, FileName);
            _logger.Error(message, ex);
            throw ex;
        }
        }

Который через Fiddler при запросах всегда получает 200 ОК. Но в коде у большинство файлов при получении ответа происходит следующая ошибка в методе

await client.GetByteArrayAsync(url);

Ошибка:

Сервер нарушил протокол. Section=ResponseHeader Detail=За возвратом каретки должен следовать перевод строки
Specified value has invalid Control characters. (Parameter 'value') 
Source ---
System.Net.WebHeaderCollection 
StackTrace ---
   at System.Net.HttpValidationHelpers.CheckBadHeaderValueChars(String value)
   at System.Net.WebHeaderCollection.Set(String name, String value)
   at System.Net.HttpWebResponse.get_Headers()
   at ITRusPostISO.ITMetasonicMigrationFile.ITMetasonicMigrationFileService.RequestAsync(Guid ActId, String FileId, String FileName, String FileFolderName, String Description, String UploaderName, String UploadDate) 

Также прикрепляю скриншот Request/Response из Fiddler файла который нормально скачивается и которые падает в ошибку.

Response InternetExplorer при переходе по ссылке и скачивании файлов.

` Вот старый код на HttpWebRequest

private async Task RequestAsync(Guid ActId, string FileId, string FileName, string FileFolderName, string Description, string UploaderName, string UploadDate)
        {
            try{
                string downloadLink = FileLinkIBMFileNet.Replace("{file_id}", FileId).Replace("{file_name}", FileName);
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(downloadLink);
                request.Credentials = new NetworkCredential(LoginIBMFilenet, PasswordIBMFileNet);
                request.Timeout = 180000;
                request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
                HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync();
            WebHeaderCollection headers = response.Headers;
            int streamSize = int.Parse(response.Headers["Content-Disposition"].Split(';')[2].Replace(" size=", ""));
            byte[] tmpFileByte = new byte[streamSize];
            using (Stream stream = response.GetResponseStream())
            {
                using (BinaryReader reader = new BinaryReader(stream))
                {
                    reader.Read(tmpFileByte, 0, streamSize);
                }
            }
            InsertFile(ActId, FileName, FileFolderName, tmpFileByte, Description, UploaderName, UploadDate, FileId);
            response.Close();
        }
        catch(Exception ex) {
            string message = string.Format("Error in RequestAsync\nMessage ---\n{0} \nSource ---\n{1} \nStackTrace ---\n{2} \nActId ---\n{3} \nFileId ---\n{4} \nFilename ---\n{5}", ex.Message, ex.Source, ex.StackTrace, ActId, FileId, FileName);
            _logger.Error(message, ex);
            throw ex;
        }
    }

Решение использовать tcpClient так как сервер шлет всегда битый response

Answer 1

Я читал бы ответ от сервера примерно вот так:

private readonly char[] _colon = new[] { ':' };
public async Task RequestAsync(Guid ActId, string FileId, string FileName, string FileFolderName, string Description, string UploaderName, string UploadDate)
{
    try
    {
        byte[] body;
        Uri url = new Uri(FileLinkIBMFileNet.Replace("{file_id}", FileId).Replace("{file_name}", FileName));
        using (var tcp = new TcpClient(url.Host, 9080))
        using (var tcpStream = tcp.GetStream())
        {
            tcp.SendTimeout = 1000;
            tcp.ReceiveTimeout = 1800;
            StringBuilder builder = new StringBuilder();
            builder.Append("GET ").Append(url.AbsoluteUri).AppendLine(" HTTP/1.1");
            builder.Append("Host: ").AppendLine(url.Authority);
            builder.AppendLine("Connection: Keep-Alive");
            builder.AppendLine("Accept-Encoding: gzip");
            builder.Append("Authorization: Basic ").AppendLine(Convert.ToBase64String(Encoding.ASCII.GetBytes($"{LoginIBMFilenet}:{PasswordIBMFileNet}")));
            builder.AppendLine();
            byte[] header = Encoding.ASCII.GetBytes(builder.ToString());
            await tcpStream.WriteAsync(header, 0, header.Length).ConfigureAwait(false);
        Dictionary<string, string> headers = new Dictionary<string, string>();
        string line;
        while ((line = tcpStream.ReadLine())?.Length > 0)
        {
            string[] tokens = line.Split(_colon, 2);
            if (tokens.Length > 1)
                headers.Add(tokens[0], tokens[1].Trim());
            else
                headers.Add("Response", tokens[0]);
        }

        using (MemoryStream memory = new MemoryStream())
        {
            bool FirstChunk = true;
            if (headers.TryGetValue("Transfer-Encoding", out string chk) && chk.Contains("chunked"))
            {
                int chunkSize = 0;
                while (true)
                {
                    if (!FirstChunk)
                        tcpStream.ReadLine();
                    else
                        FirstChunk = false;
                    chunkSize = int.Parse(tcpStream.ReadLine(), NumberStyles.HexNumber);
                    if (chunkSize == 0) break;
                    await tcpStream.CopyBytesAsync(memory, chunkSize).ConfigureAwait(false);
                }
            }
            else
                await tcpStream.CopyToAsync(memory).ConfigureAwait(false);

            memory.Position = 0;

            if (headers.TryGetValue("Content-Encoding", out string enc) && enc.Contains("gzip"))
                using (Stream gzipStream = new GZipStream(memory, CompressionMode.Decompress, true))
                using (MemoryStream ms = new MemoryStream())
                {
                    await gzipStream.CopyToAsync(ms).ConfigureAwait(false);
                    body = ms.ToArray();
                }
            else
                body = memory.ToArray();
        }
    }
    InsertFile(ActId, FileName, FileFolderName, body, Description, UploaderName, UploadDate, FileId);
}
catch (Exception ex)
{
    throw ex;
}

}

И вот такие экстеншн методы для NetworkStream понадобятся.

public static class NetworkStreamExtensions
{
    public static string ReadLine(this NetworkStream stream)
    {
        List<byte> bytes = new List<byte>();
        int b;
        while ((b = stream.ReadByte()) != -1 && b != '\n')
            bytes.Add((byte)b);
        if (bytes.Count > 0 && bytes.Last() == '\r')
            bytes.RemoveAt(bytes.Count - 1);
        return b == -1 && bytes.Count == 0 ? null : Encoding.ASCII.GetString(bytes.ToArray());
    }
public static async Task CopyBytesAsync(this NetworkStream stream, Stream outputStream, int bytesToCopy)
{
    int toCopy = bytesToCopy;
    const int bufferSize = 32768;
    byte[] buffer = new byte[bufferSize];
    while (true)
    {
        int toRead = toCopy &gt; bufferSize ? bufferSize : toCopy;
        if (toRead == 0) break;
        int bytesRead = await stream.ReadAsync(buffer, 0, toRead).ConfigureAwait(false);
        if (bytesRead == 0) break;
        await outputStream.WriteAsync(buffer, 0, bytesRead).ConfigureAwait(false);
        toCopy -= bytesRead;
    }
}

}