proxy format conflict

resolve conflict between proxy format: HTTPX and Requests

Author: ma-ji

scholarly/_proxy_generator.py

@@ -136,7 +136,8 @@ def _check_proxy(self, proxies) -> bool:
         :rtype: {bool}
         """
         with requests.Session() as session:
-            session.proxies = proxies
+            # Reformat proxy for requests. Requests and HTTPX use different proxy format.
+            session.proxies = {'http':proxies['http://'], 'https':proxies['https://']}
             try:
                 resp = session.get("http://httpbin.org/ip", timeout=self._TIMEOUT)
                 if resp.status_code == 200:
@@ -189,6 +190,7 @@ def _use_proxy(self, http: str, https: str = None) -> bool:
         :returns: whether or not the proxy was set up successfully
         :rtype: {bool}
         """
+        # Reformat proxy for HTTPX
         if http[:4] not in ("http", "sock"):
             http = "http://" + http
         if https is None:
@@ -521,6 +523,7 @@ def _fp_coroutine(self, timeout=1, wait_time=120):
             proxies = {'http://': proxy, 'https://': proxy}
             proxy_works = self._check_proxy(proxies)
             if proxy_works:
+                print(proxies)
                 dirty_proxy = (yield proxy)
                 t1 = time.time()
             else: