opening two instances of chrome webdriver(with and without proxies) with selenium python

huangapple
go评论91阅读模式
英文:

opening two instances of chrome webdriver(with and without proxies) with selenium python

问题

我正在尝试使用Python的Selenium创建两个Chrome WebDriver的实例。第一个实例使用代理，而第二个实例没有代理。

我面临的问题是，第二个实例打开的URL无法加载（因为它使用代理）。我已经实现了两个不同的实例，但仍然遇到此错误。以下是我迄今为止编写的代码：

  1. import os, tempfile
  2. from selenium.common.exceptions import TimeoutException
  3. from selenium.webdriver.support.ui import WebDriverWait
  4. from selenium.webdriver.common.proxy import Proxy, ProxyType
  5. from selenium import webdriver
  7. # 使用代理和隐身模式设置Selenium
  8. def setup_selenium_with_proxy(proxy):
  9.     proxy_config = Proxy()
  10.     proxy_config.proxy_type = ProxyType.MANUAL
  11.     proxy_config.http_proxy = f"{proxy['ip']}:{proxy['port']}"
  12.     proxy_config.ssl_proxy = f"{proxy['ip']}:{proxy['port']}"
  14.     capabilities = webdriver.DesiredCapabilities.CHROME
  15.     proxy_config.add_to_capabilities(capabilities)
  17.     options = webdriver.ChromeOptions()
  18.     options.add_argument("--incognito")
  19.     options.add_argument("--disable-dev-shm-usage")
  20.     options.add_argument("--no-sandbox")
  21.     options.add_argument("--log-level=3")
  22.     options.add_experimental_option("excludeSwitches", ["enable-automation", "enable-logging"])
  24.     # 为代理浏览器创建单独的用户数据目录
  25.     user_data_dir_proxy = os.path.join(tempfile.gettempdir(), "selenium_proxy_profile")
  26.     if not os.path.exists(user_data_dir_proxy):
  27.         os.makedirs(user_data_dir_proxy)
  28.     options.add_argument(f"--user-data-dir={user_data_dir_proxy}")
  30.     driver = webdriver.Chrome(options=options, desired_capabilities=capabilities)
  31.     return driver
  33. # 不使用代理和隐身模式设置Selenium
  34. def setup_selenium_without_proxy():
  35.     options = webdriver.ChromeOptions()
  36.     options.add_argument("--incognito")
  37.     options.add_argument("--disable-dev-shm-usage")
  38.     options.add_argument("--no-sandbox")
  39.     options.add_argument("--log-level=3")
  40.     options.add_experimental_option("excludeSwitches", ["enable-automation", "enable-logging"])
  42.     # 为非代理浏览器创建单独的用户数据目录
  43.     user_data_dir_no_proxy = os.path.join(tempfile.gettempdir(), "selenium_no_proxy_profile")
  44.     if not os.path.exists(user_data_dir_no_proxy):
  45.         os.makedirs(user_data_dir_no_proxy)
  46.     options.add_argument(f"--user-data-dir={user_data_dir_no_proxy}")
  48.     driver = webdriver.Chrome(options=options)
  49.     return driver

我的主函数首先使用代理驱动程序工作，当我调用第二个无代理的驱动程序时，网站无法工作并显示网站无法访问 net::ERR_TUNNEL_CONNECTION_FAILED。

  1. def main():
  2.     url = "http://example.com"
  3.     
  4.     # 从 'http.txt' 文件中读取代理
  5.     with open('http.txt', 'r') as file:
  6.         proxy_list = [{'ip': proxy.split(':')[0], 'port': proxy.split(':')[1].strip()} for proxy in file.readlines()]
  8.     at_least_one_working_proxy = False
  9.     proxy_index = 0
  11.     while proxy_index < len(proxy_list):
  12.         proxy = proxy_list[proxy_index]
  13.         driver = setup_selenium_with_proxy(proxy)
  14.         driver.set_page_load_timeout(20)
  15.         try:
  16.             driver.get(url)
  17.             
  18.             # 代码
  19. 			
  20.             new_driver = setup_selenium_without_proxy()
  21.             new_driver.set_page_load_timeout(20)
  22.         except TimeoutException:
  23.             print(f"Failed to open url using proxy {proxy['ip']}:{proxy['port']} within 20 seconds")
  24.         except WebDriverException as e:
  25.             print(f"WebDriverException encountered while using proxy {proxy['ip']}:{proxy['port']}: {e}")
英文:

I am trying to create two instances of chrome webdriver with selenium python. the first one uses proxies and the second one is proxyless.
The problem I am facing is that the URL opened in the second instance doesn't load(because it uses proxies). I've implemented two different instances, but I still get this error. This is what I've written so far:

  1. import os, tempfile
  2. from selenium.common.exceptions import TimeoutException
  3. from selenium.webdriver.support.ui import WebDriverWait
  4. from selenium.webdriver.common.proxy import Proxy, ProxyType
  5. from selenium import webdriver
  6. # Set up Selenium with proxy and incognito mode
  7. def setup_selenium_with_proxy(proxy):
  8. proxy_config = Proxy()
  9. proxy_config.proxy_type = ProxyType.MANUAL
  10. proxy_config.http_proxy = f&quot;{proxy[&#39;ip&#39;]}:{proxy[&#39;port&#39;]}&quot;
  11. proxy_config.ssl_proxy = f&quot;{proxy[&#39;ip&#39;]}:{proxy[&#39;port&#39;]}&quot;
  12. capabilities = webdriver.DesiredCapabilities.CHROME
  13. proxy_config.add_to_capabilities(capabilities)
  14. options = webdriver.ChromeOptions()
  15. options.add_argument(&quot;--incognito&quot;)
  16. options.add_argument(&quot;--disable-dev-shm-usage&quot;)
  17. options.add_argument(&quot;--no-sandbox&quot;)
  18. options.add_argument(&quot;--log-level=3&quot;)
  19. options.add_experimental_option(&quot;excludeSwitches&quot;, [&quot;enable-automation&quot;, &quot;enable-logging&quot;])
  20. # Create a separate user data directory for the proxy browser
  21. user_data_dir_proxy = os.path.join(tempfile.gettempdir(), &quot;selenium_proxy_profile&quot;)
  22. if not os.path.exists(user_data_dir_proxy):
  23. os.makedirs(user_data_dir_proxy)
  24. options.add_argument(f&quot;--user-data-dir={user_data_dir_proxy}&quot;)
  25. driver = webdriver.Chrome(options=options, desired_capabilities=capabilities)
  26. return driver
  27. # Set up Selenium without proxy and incognito mode
  28. def setup_selenium_without_proxy():
  29. options = webdriver.ChromeOptions()
  30. options.add_argument(&quot;--incognito&quot;)
  31. options.add_argument(&quot;--disable-dev-shm-usage&quot;)
  32. options.add_argument(&quot;--no-sandbox&quot;)
  33. options.add_argument(&quot;--log-level=3&quot;)
  34. options.add_experimental_option(&quot;excludeSwitches&quot;, [&quot;enable-automation&quot;, &quot;enable-logging&quot;])
  35. # Create a separate user data directory for the non-proxy browser
  36. user_data_dir_no_proxy = os.path.join(tempfile.gettempdir(), &quot;selenium_no_proxy_profile&quot;)
  37. if not os.path.exists(user_data_dir_no_proxy):
  38. os.makedirs(user_data_dir_no_proxy)
  39. options.add_argument(f&quot;--user-data-dir={user_data_dir_no_proxy}&quot;)
  40. driver = webdriver.Chrome(options=options)
  41. return driver

my main function works with the proxy driver first, and when I call the second proxyless driver, the website doesn't work and shows website cannot be reached net::ERR_TUNNEL_CONNECTION_FAILED.

  1. def main():
  2. url = &quot;http://example.com&quot;
  3. # Read proxies from &#39;http.txt&#39; file
  4. with open(&#39;http.txt&#39;, &#39;r&#39;) as file:
  5. proxy_list = [{&#39;ip&#39;: proxy.split(&#39;:&#39;)[0], &#39;port&#39;: proxy.split(&#39;:&#39;)[1].strip()} for proxy in file.readlines()]
  6. at_least_one_working_proxy = False
  7. proxy_index = 0
  8. while proxy_index &lt; len(proxy_list):
  9. proxy = proxy_list[proxy_index]
  10. driver = setup_selenium_with_proxy(proxy)
  11. driver.set_page_load_timeout(20)
  12. try:
  13. driver.get(url)
  14. # code
  15. new_driver = setup_selenium_without_proxy()
  16. new_driver.set_page_load_timeout(20)
  17. except TimeoutException:
  18. print(f&quot;Failed to open url using proxy {proxy[&#39;ip&#39;]}:{proxy[&#39;port&#39;]} within 20 seconds&quot;)
  19. except WebDriverException as e:
  20. print(f&quot;WebDriverException encountered while using proxy {proxy[&#39;ip&#39;]}:{proxy[&#39;port&#39;]}: {e}&quot;)

答案1

得分: 0

将以下行添加到您的without_proxy函数中：

  1. options.add_argument('--no-proxy-server')
英文:

add the following line to your without_proxy function:

  1. options.add_argument(&#39;--no-proxy-server&#39;)

huangapple
  • 本文由 发表于 2023年4月7日 05:41:45
  • 转载请务必保留本文链接：https://go.coder-hub.com/75953983.html
  • python
  • selenium-webdriver
  • web-scraping
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定