ScrollDown in table on a website usinig Selenium

huangapple go评论71阅读模式
英文:

ScrollDown in table on a website usinig Selenium

问题

我尝试使用以下代码在网站上的表格中向下滚动:

import os
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from webdriver_manager.chrome import ChromeDriverManager

if __name__ == '__main__':
    WAIT = 1
    print("Checking Browser driver...")
    os.environ['WDM_LOG'] = '0'
    options = Options()
    options.add_argument("start-maximized")
    options.add_experimental_option("prefs", {"profile.default_content_setting_values.notifications": 1})
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option('excludeSwitches', ['enable-logging'])
    options.add_experimental_option('useAutomationExtension', False)
    options.add_argument('--disable-blink-features=AutomationControlled')
    srv = Service(ChromeDriverManager().install())
    driver = webdriver.Chrome(service=srv, options=options)
    waitWD = WebDriverWait(driver, 10)

    link = "https://www.appliancepartspros.com/ge-dryer-timer-knob-we1m654-ap3995088.html"
    driver.get(link)
    driver.execute_script("arguments[0].scrollIntoView(true);", waitWD.until(EC.presence_of_element_located((By.XPATH, '//h2[text()="Cross Reference and Model Information"]'))))
    tmpBODY = driver.find_element(By.XPATH, '//div[@class="m-bsc"]/a[@name="crossref"]')
    for _ in range(5):
        tmpBODY.send_keys(Keys.PAGE_DOWN)

但我始终收到以下错误消息:

(selenium) C:\DEV\Fiverr\TRY\biglaundrystore>python try.py
Checking Browser driver...
Traceback (most recent call last):
  File "C:\DEV\Fiverr\TRY\biglaundrystore\try.py", line 31, in <module>
    tmpBODY.send_keys(Keys.PAGE_DOWN)
  File "C:\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\remote\webelement.py", line 231, in send_keys
    self._execute(
  File "C:\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\remote\webelement.py", line 404, in _execute
    return self._parent.execute(command, params)
  File "C:\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 440, in execute
    self.error_handler.check_response(response)
  File "C:\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 245, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
  (Session info: chrome=113.0.5672.127)

这是我想要向下滚动的表格 - 如您所见,页面向下滚动并不像预期那样工作:

ScrollDown in table on a website usinig Selenium

英文:

i try to scroll down in a table on a website with the following code:

import os
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from webdriver_manager.chrome import ChromeDriverManager

if __name__ == &#39;__main__&#39;:  
  WAIT = 1
  print(f&quot;Checking Browser driver...&quot;)
  os.environ[&#39;WDM_LOG&#39;] = &#39;0&#39; 
  options = Options()
  options.add_argument(&quot;start-maximized&quot;)
  options.add_experimental_option(&quot;prefs&quot;, {&quot;profile.default_content_setting_values.notifications&quot;: 1})    
  options.add_experimental_option(&quot;excludeSwitches&quot;, [&quot;enable-automation&quot;])
  options.add_experimental_option(&#39;excludeSwitches&#39;, [&#39;enable-logging&#39;])
  options.add_experimental_option(&#39;useAutomationExtension&#39;, False)
  options.add_argument(&#39;--disable-blink-features=AutomationControlled&#39;) 
  srv=Service(ChromeDriverManager().install())
  driver = webdriver.Chrome (service=srv, options=options)    
  waitWD = WebDriverWait (driver, 10)         
  
  link = &quot;https://www.appliancepartspros.com/ge-dryer-timer-knob-we1m654-ap3995088.html&quot;
  driver.get (link)     
  driver.execute_script(&quot;arguments[0].scrollIntoView(true);&quot;, waitWD.until(EC.presence_of_element_located((By.XPATH,&#39;//h2[text()=&quot;Cross Reference and Model Information&quot;]&#39;))))         
  tmpBODY = driver.find_element(By.XPATH, &#39;//div[@class=&quot;m-bsc&quot;]/a[@name=&quot;crossref&quot;]&#39;)
  for _ in range(5):
    tmpBODY.send_keys (Keys.PAGE_DOWN)

But i allways get this error-message:

(selenium) C:\DEV\Fiverr\TRY\biglaundrystore&gt;python try.py
Checking Browser driver...
Traceback (most recent call last):
  File &quot;C:\DEV\Fiverr\TRY\biglaundrystore\try.py&quot;, line 31, in &lt;module&gt;
    tmpBODY.send_keys (Keys.PAGE_DOWN)
  File &quot;C:\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\remote\webelement.py&quot;, line 231, in send_keys       
    self._execute(
  File &quot;C:\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\remote\webelement.py&quot;, line 404, in _execute        
    return self._parent.execute(command, params)
  File &quot;C:\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\remote\webdriver.py&quot;, line 440, in execute
    self.error_handler.check_response(response)
  File &quot;C:\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\remote\errorhandler.py&quot;, line 245, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
  (Session info: chrome=113.0.5672.127)

This is the table i would like to scrolldown - as you can see the page-downs are not working as expected:

ScrollDown in table on a website usinig Selenium

答案1

得分: 0

也许这是一个 iframe,你可以在页面源代码中搜索 "iframe",你应该首先进入 iframe,然后向下滚动。

你可以使用滚动来向下滚动。

from selenium.webdriver.common.actions.wheel_input import ScrollOrigin
from selenium.webdriver import ActionChains
from selenium import webdriver

scroll_origin = ScrollOrigin.from_viewport(0, 0)
ActionChains(driver).scroll_from_origin(scroll_origin, 0, kkk_y).perform()

这个答案可能会有帮助。

https://stackoverflow.com/a/76598852/17601704

查看文档:

https://www.selenium.dev/documentation/webdriver/actions_api/wheel/#scroll-to-element
https://www.selenium.dev/documentation/webdriver/interactions/frames/

英文:

Maybe it is a iframe and you can search "iframe" in page source, you should go to iframe firstly, then scroll down<br>

and you can use scroll to go down<br>

from selenium.webdriver.common.actions.wheel_input import ScrollOrigin
from selenium.webdriver import ActionChains
from selenium import webdriver
scroll_origin = ScrollOrigin.from_viewport(0, 0)
ActionChains(driver).scroll_from_origin(scroll_origin, 0, kkk_y).perform()

this answer maybe helpful

https://stackoverflow.com/a/76598852/17601704

view the document

https://www.selenium.dev/documentation/webdriver/actions_api/wheel/#scroll-to-element
https://www.selenium.dev/documentation/webdriver/interactions/frames/

答案2

得分: 0

我认为我找到了使用这段代码的解决方案:

(可能不是最优雅的解决方案,但似乎在最后做我需要的事情)

import os
import time
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from webdriver_manager.chrome import ChromeDriverManager

if __name__ == '__main__':
    WAIT = 1
    print("Checking Browser driver...")
    os.environ['WDM_LOG'] = '0'
    options = Options()
    options.add_argument("start-maximized")
    options.add_experimental_option("prefs", {"profile.default_content_setting_values.notifications": 1})
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option('excludeSwitches', ['enable-logging'])
    options.add_experimental_option('useAutomationExtension', False)
    options.add_argument('--disable-blink-features=AutomationControlled')
    srv=Service(ChromeDriverManager().install())
    driver = webdriver.Chrome (service=srv, options=options)
    waitWD = WebDriverWait (driver, 10)

    link = "https://www.appliancepartspros.com/ge-dryer-timer-knob-we1m654-ap3995088.html"
    driver.get (link)
    lastCounts = 0
    while True:
        print(f"Reading elements found {lastCounts}")
        driver.execute_script("arguments[0].scrollIntoView(true);", waitWD.until(EC.presence_of_element_located((By.XPATH,'//div[@id="xmodellist"]//tr[last()]'))))
        time.sleep(WAIT)
        soup = BeautifulSoup (driver.page_source, 'lxml')
        worker = soup.find("div", {"id": "xmodellist"})
        worker = worker.find_all("tr")
        if len(worker) == lastCounts:
            break
        lastCounts = len(worker)
英文:

I think i found a solution using this code:

(probably not the most elegant solution - but it seems its doing what i need at the end)

import os
import time
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from webdriver_manager.chrome import ChromeDriverManager

if __name__ == &#39;__main__&#39;:  
  WAIT = 1
  print(f&quot;Checking Browser driver...&quot;)
  os.environ[&#39;WDM_LOG&#39;] = &#39;0&#39; 
  options = Options()
  options.add_argument(&quot;start-maximized&quot;)
  options.add_experimental_option(&quot;prefs&quot;, {&quot;profile.default_content_setting_values.notifications&quot;: 1})    
  options.add_experimental_option(&quot;excludeSwitches&quot;, [&quot;enable-automation&quot;])
  options.add_experimental_option(&#39;excludeSwitches&#39;, [&#39;enable-logging&#39;])
  options.add_experimental_option(&#39;useAutomationExtension&#39;, False)
  options.add_argument(&#39;--disable-blink-features=AutomationControlled&#39;) 
  srv=Service(ChromeDriverManager().install())
  driver = webdriver.Chrome (service=srv, options=options)    
  waitWD = WebDriverWait (driver, 10)         
  
  link = &quot;https://www.appliancepartspros.com/ge-dryer-timer-knob-we1m654-ap3995088.html&quot;
  driver.get (link)  
  lastCounts = 0   
  while True:   
    print(f&quot;Reading elements found {lastCounts}&quot;)
    driver.execute_script(&quot;arguments[0].scrollIntoView(true);&quot;, waitWD.until(EC.presence_of_element_located((By.XPATH,&#39;//div[@id=&quot;xmodellist&quot;]//tr[last()]&#39;))))      
    time.sleep(WAIT)
    soup = BeautifulSoup (driver.page_source, &#39;lxml&#39;)  
    worker = soup.find(&quot;div&quot;, {&quot;id&quot;: &quot;xmodellist&quot;})  
    worker = worker.find_all(&quot;tr&quot;)
    if len(worker) == lastCounts:
      break
    lastCounts = len(worker)

huangapple
  • 本文由 发表于 2023年7月3日 17:33:45
  • 转载请务必保留本文链接:https://go.coder-hub.com/76603505.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定