HttpWebRequest and WebResponse To HttpClient

NGitLab.Tests/Docker/GitLabTestContextRequestOptions.cs

@@ -30,86 +30,31 @@
         UserAgent = "NGitLab.Tests/1.0.0";
     }
 
-    public override WebResponse GetResponse(HttpWebRequest request)
+    public override void ProcessGitLabRequestResult(GitLabRequestResult result)
     {
+        var request = result.Request;
         lock (_allRequests)
         {
             _allRequests.Add(request);
         }
 
-        WebResponse response = null;
-
-        // GitLab is unstable, so let's make sure we don't overload it with many concurrent requests
-        s_semaphoreSlim.Wait();
+        WebResponse response = result.Response;
         try
         {
-            try
-            {
-                response = base.GetResponse(request);
-            }
-            catch (WebException exception)
+            if (result.Exception is WebException exception)
             {
                 response = exception.Response;
                 if (response is HttpWebResponse webResponse)
                 {
                     response = new LoggableHttpWebResponse(webResponse);
-                    throw new WebException(exception.Message, exception, exception.Status, response);
+                    result.Exception = new WebException(exception.Message, exception, exception.Status, response);
                 }
-
-                throw;
-            }
-            finally
-            {
-                response = LogRequest(request, response);
             }
         }
         finally
         {
-            s_semaphoreSlim.Release();
+            result.Response = LogRequest(request, response);
         }
-
-        return response;
-    }
-
-    public override async Task<WebResponse> GetResponseAsync(HttpWebRequest request, CancellationToken cancellationToken)
-    {
-        lock (_allRequests)
-        {
-            _allRequests.Add(request);
-        }
-
-        WebResponse response = null;
-
-        // GitLab is unstable, so let's make sure we don't overload it with many concurrent requests
-        await s_semaphoreSlim.WaitAsync(cancellationToken).ConfigureAwait(false);
-        try
-        {
-            try
-            {
-                response = await base.GetResponseAsync(request, cancellationToken).ConfigureAwait(false);
-            }
-            catch (WebException exception)
-            {
-                response = exception.Response;
-                if (response is HttpWebResponse webResponse)
-                {
-                    response = new LoggableHttpWebResponse(webResponse);
-                    throw new WebException(exception.Message, exception, exception.Status, response);
-                }
-
-                throw;
-            }
-            finally
-            {
-                response = LogRequest(request, response);
-            }
-        }
-        finally
-        {
-            s_semaphoreSlim.Release();
-        }
-
-        return response;
     }
 
     private WebResponse LogRequest(HttpWebRequest request, WebResponse response)
@@ -178,7 +123,7 @@
         return response;
     }
 
     internal override Stream GetRequestStream(HttpWebRequest request)
     {
         var stream = new LoggableRequestStream(request.GetRequestStream());
         _pendingRequest.AddOrUpdate(request, stream, (_, _) => stream);

NGitLab.Tests/HttpRequestorTests.cs

@@ -166,11 +166,12 @@ public override bool ShouldRetry(Exception ex, int retryNumber)
             return base.ShouldRetry(ex, retryNumber);
         }
 
-        public override WebResponse GetResponse(HttpWebRequest request)
+        public override void ProcessGitLabRequestResult(GitLabRequestResult result)
         {
+            var request = result.Request;
             HttpRequestSudoHeader = request.Headers["Sudo"];
             HandledRequests.Add(request);
-            throw new GitLabException { StatusCode = HttpStatusCode.InternalServerError };
+            result.Exception = new GitLabException { StatusCode = HttpStatusCode.InternalServerError };
         }
     }
 }

NGitLab/GitLabRequestResult.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Net.Http;
+
+namespace NGitLab;
+
+public sealed class GitLabRequestResult
+{
+    public HttpRequestMessage Request { get; set; }
+
+    public HttpResponseMessage   Response { get; set; }
+
+    public Exception Exception { get; set; }
+}

NGitLab/Impl/HttpRequestor.GitLabRequest.cs

@@ -76,61 +76,67 @@
             }
         }
 
-        public WebResponse GetResponse(RequestOptions options)
+        public HttpResponseMessage GetResponse(RequestOptions options)
         {
-            Func<WebResponse> getResponseImpl = () => GetResponseImpl(options);
+            Func<HttpResponseMessage> getResponseImpl = () => GetResponseImpl(options);
 
             return getResponseImpl.Retry(options.ShouldRetry,
                 options.RetryInterval,
                 options.RetryCount,
                 options.IsIncremental);
         }
 
-        public Task<WebResponse> GetResponseAsync(RequestOptions options, CancellationToken cancellationToken)
+        public Task<HttpResponseMessage> GetResponseAsync(RequestOptions options, CancellationToken cancellationToken)
         {
-            Func<Task<WebResponse>> getResponseImpl = () => GetResponseImplAsync(options, cancellationToken);
+            Func<Task<HttpResponseMessage>> getResponseImpl = () => GetResponseImplAsync(options, cancellationToken);
 
             return getResponseImpl.RetryAsync(options.ShouldRetry,
                 options.RetryInterval,
                 options.RetryCount,
                 options.IsIncremental);
         }
 
-        private WebResponse GetResponseImpl(RequestOptions options)
+        private HttpResponseMessage GetResponseImpl(RequestOptions options)
         {
+            var result = new GitLabRequestResult();
             try
             {
-                var request = CreateRequest(options);
-                return options.GetResponse(request);
+                result.Request = CreateRequest(options);
+                result.Response = _httpClient.SendAsync(result.Request).GetAwaiter().GetResult();
             }
-            catch (WebException wex)
+            catch (Exception ex)
             {
-                if (wex.Response == null)
-                    throw;
-
-                HandleWebException(wex);
-                throw;
+                result.Exception = ex is WebException wex && wex.Response is not null ?
+                                   HandleWebException(wex) :
+                                   ex;
             }
+
+            options.ProcessGitLabRequestResult(result);
+
+            return result.Exception is not null ? throw result.Exception : result.Response;
         }
 
-        private async Task<WebResponse> GetResponseImplAsync(RequestOptions options, CancellationToken cancellationToken)
+        private async Task<HttpResponseMessage> GetResponseImplAsync(RequestOptions options, CancellationToken cancellationToken)
         {
+            var result = new GitLabRequestResult();
             try
             {
-                var request = CreateRequest(options);
-                return await options.GetResponseAsync(request, cancellationToken).ConfigureAwait(false);
+                result.Request = CreateRequest(options);
+                result.Response = await _httpClient.SendAsync(result.Request, cancellationToken).ConfigureAwait(false);
             }
-            catch (WebException wex)
+            catch (Exception ex)
             {
-                if (wex.Response == null)
-                    throw;
-
-                HandleWebException(wex);
-                throw;
+                result.Exception = ex is WebException wex && wex.Response is not null ?
+                                   HandleWebException(wex) :
+                                   ex;
             }
+
+            options.ProcessGitLabRequestResult(result);
+
+            return result.Exception is not null ? throw result.Exception : result.Response;
         }
 
-        private void HandleWebException(WebException ex)
+        private GitLabException HandleWebException(WebException ex)
         {
             using var errorResponse = (HttpWebResponse)ex.Response;
             string jsonString;
@@ -149,7 +155,7 @@
                 exceptionMessage += $". With data {JsonData}";
             }
 
-            throw new GitLabException(exceptionMessage)
+            return new GitLabException(exceptionMessage)
             {
                 OriginalCall = Url,
                 ErrorObject = errorDetails,
@@ -159,20 +165,22 @@
             };
         }
 
-        private HttpWebRequest CreateRequest(RequestOptions options)
+        private static HttpClient _httpClient = null;
+
+        private HttpRequestMessage CreateRequest(RequestOptions options)
         {
-            var request = WebRequest.CreateHttp(Url);
-            request.Method = Method.ToString().ToUpperInvariant();
-            request.Accept = "application/json";
-            request.Headers = Headers;
-            request.AutomaticDecompression = DecompressionMethods.GZip;
-            request.Timeout = (int)options.HttpClientTimeout.TotalMilliseconds;
-            request.ReadWriteTimeout = (int)options.HttpClientTimeout.TotalMilliseconds;
-            if (options.Proxy != null)
+            if (_httpClient != null)
             {
-                request.Proxy = options.Proxy;
+                var handler = new HttpClientHandler
+                {
+                    AutomaticDecompression = DecompressionMethods.GZip,
+                    Proxy = options.Proxy,
+                    UseProxy = options.Proxy != null,
+                };
+                _httpClient = new HttpClient(handler);
+                _httpClient.Timeout = options.HttpClientTimeout;
             }
-
+            var request = new HttpRequestMessage(new HttpMethod(Method.ToString().ToUpperInvariant()), Url);
             if (HasOutput)
             {
                 if (FormData != null)
@@ -190,29 +198,27 @@
             }
             else if (Method == MethodType.Put)
             {
-                request.ContentLength = 0;
+                //request.ContentLength = 0;
             }
 
             return request;
         }
 
-        private void AddJsonData(HttpWebRequest request, RequestOptions options)
+        private void AddJsonData(HttpRequestMessage request, RequestOptions options)
         {
-            request.ContentType = "application/json";
-
+            request.Content.Headers.ContentType.MediaType = "application/json";
             using var writer = new StreamWriter(options.GetRequestStream(request));
             writer.Write(JsonData);
             writer.Flush();
             writer.Close();
         }
 
-        public void AddFileData(HttpWebRequest request, RequestOptions options)
+        public void AddFileData(HttpRequestMessage request, RequestOptions options)
         {
             var boundary = $"--------------------------{DateTime.UtcNow.Ticks.ToStringInvariant()}";
             if (Data is not FormDataContent formData)
                 return;
-            request.ContentType = "multipart/form-data; boundary=" + boundary;
-
+            request.Content.Headers.ContentType.MediaType = "multipart/form-data; boundary=" + boundary;
             using var uploadContent = new MultipartFormDataContent(boundary)
             {
                 { new StreamContent(formData.Stream), "file", formData.Name },
@@ -221,10 +227,9 @@
             uploadContent.CopyToAsync(options.GetRequestStream(request)).Wait();
         }
 
-        public void AddUrlEncodedData(HttpWebRequest request, RequestOptions options)
+        public void AddUrlEncodedData(HttpRequestMessage request, RequestOptions options)
         {
-            request.ContentType = "application/x-www-form-urlencoded";
-
+            request.Content.Headers.ContentType.MediaType = "application/x-www-form-urlencoded";
             using var content = new FormUrlEncodedContent(UrlEncodedData.Values);
             content.CopyToAsync(options.GetRequestStream(request)).Wait();
         }

NGitLab/Impl/HttpRequestor.cs

@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using System.Net;
+using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using NGitLab.Impl.Json;
@@ -144,7 +145,7 @@
         using var response = request.GetResponse(_options);
         if (parser != null)
         {
-            using var stream = response.GetResponseStream();
+            using var stream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult();
             parser(stream, new WebHeadersDictionaryAdaptor(response.Headers));
         }
     }
@@ -159,7 +160,7 @@
         using var response = await request.GetResponseAsync(_options, cancellationToken).ConfigureAwait(false);
         if (parser != null)
         {
-            using var stream = response.GetResponseStream();
+            using var stream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult();
             await parser(stream, new WebHeadersDictionaryAdaptor(response.Headers)).ConfigureAwait(false);
         }
     }
@@ -213,8 +214,7 @@
                 using (var response = await request.GetResponseAsync(_options, cancellationToken).ConfigureAwait(false))
                 {
                     nextUrlToLoad = GetNextPageUrl(response);
-
-                    using var stream = response.GetResponseStream();
+                    using var stream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult();
                     responseText = await ReadTextAsync(stream).ConfigureAwait(false);
                 }
 
@@ -234,8 +234,7 @@
                 using (var response = request.GetResponse(_options))
                 {
                     nextUrlToLoad = GetNextPageUrl(response);
-
-                    using var stream = response.GetResponseStream();
+                    using var stream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult();
                     responseText = ReadText(stream);
                 }
 
@@ -245,19 +244,17 @@
             }
         }
 
-        private static Uri GetNextPageUrl(WebResponse response)
+        private static Uri GetNextPageUrl(HttpResponseMessage response)
         {
             // <http://localhost:1080/api/v3/projects?page=2&per_page=0>; rel="next", <http://localhost:1080/api/v3/projects?page=1&per_page=0>; rel="first", <http://localhost:1080/api/v3/projects?page=2&per_page=0>; rel="last"
-            var link = response.Headers["Link"] ?? response.Headers["Links"];
-
+            var link = response.Headers.FirstOrDefault(f => f.Key == "Link").Value.FirstOrDefault();
             string[] nextLink = null;
             if (!string.IsNullOrEmpty(link))
             {
                 nextLink = link.Split(',')
                    .Select(l => l.Split(';'))
                    .FirstOrDefault(pair => pair[1].Contains("next"));
             }
-
             return nextLink != null ? new Uri(nextLink[0].Trim('<', '>', ' ')) : null;
         }
     }

NGitLab/Impl/WebHeadersDictionaryAdaptor.cs

@@ -3,6 +3,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
+using System.Net.Http.Headers;
 
 namespace NGitLab.Impl;
 
@@ -20,6 +21,19 @@ public WebHeadersDictionaryAdaptor(WebHeaderCollection headers)
         _headers = headers ?? throw new ArgumentNullException(nameof(headers));
     }
 
+    public WebHeadersDictionaryAdaptor(HttpResponseHeaders _headers)
+    {
+        var _tmp = _headers ?? throw new ArgumentNullException(nameof(_headers));
+         _headers.Clear();
+        _tmp.ToList().ForEach(header =>
+        {
+            foreach (var value in header.Value)
+            {
+                this._headers.Add(header.Key, value);
+            }
+        });
+    }
+
     public int Count => _headers.Count;
 
     public IEnumerable<string> this[string key] => _headers.GetValues(key) ?? throw new InvalidOperationException("Header not found");