from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import pandas as pd
url = 'https://www.zaubacorp.com/company-list'
# Set up Selenium options
options = Options()
options.add_argument('--headless')
# Create a new instance of the Chrome driver
driver = webdriver.Chrome(options=options)
# Navigate to the webpage
driver.get(url)
# Wait for the page to load
driver.implicitly_wait(10)
# Find all table elements on the page using the 'tag_name' locator strategy
tables = driver.find_elements('tag name', 'table')
# Iterate through the tables to find the one you need
table = None
for t in tables:
    if 'list-group-item' in t.get_attribute('class'):
        table = t
        break
if table:
    # Extract the table data
    data = []
    for row in table.find_elements('tag name', 'tr'):
        rowData = []
        for cell in row.find_elements('tag name', 'td'):
            rowData.append(cell.text)
        data.append(rowData)
    # Store the table data in a DataFrame
    results = pd.DataFrame(data)
    # Print the results
    print(results)
else:
    print('Table not found.')
# Close the Selenium driver
driver.quit()

import requests
import pandas as pd
from bs4 import BeautifulSoup
import re
url = 'https://www.zaubacorp.com/company-list/p-1-company.html'
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')
last_page = soup.find_all('a', text=lambda text: text and '>>' in text)[0]['href']
match = int(re.search(r'p-(\d+)',last_page).group(1))
dfs = []
tot = match
for page in range(1, match+1):
    url = f'https://www.zaubacorp.com/company-list/p-{page}-company.html'
    print(f'Page: {page} of {tot}')
    dfs.append(pd.read_html(url)[0])
    
df = pd.concat(dfs)

tables = driver.find_elements(By.TAG_NAME, 'table')