How can I fix a 403 Forbidden error when attempting web-scraping in R?

403 Forbidden error message when trying to access Congressional Record

I tried this code:

library(tidyverse)
library(rvest)
library(httr)

# Functions
get_urls_in_table <- function(master_link) {
  with_user_agent <- GET(master_link, add_headers('user-agent' = 'name@xxx.edu'))
  links <- read_html(with_user_agent) %>% html_nodes("td a") %>% html_attr('href') %>% as.data.frame()
  return(links)
}

get_content <- function(x) {
  with_user_agent <- GET(x, add_headers('user-agent' = 'name@xxx.edu'))
  content <- read_html(with_user_agent) %>% html_nodes(".styled") %>% html_text()
  print("getting content")
  Sys.sleep(30) # Here add 30 sec between each scraping attempt to prevent the queries from becoming too frequent
  return(content)
}

# Layer 1
senate_pages <- get_urls_in_table("https://www.congress.gov/congressional-record/108th-congress/browse-by-date")

senate_pages <- senate_pages %>%
  mutate(real_url = paste("https://www.congress.gov", ., sep = "")) %>%
  filter(grepl("senate",real_url))

# Layer 2
senate_articles <- lapply(senate_pages$real_url[1:2],get_urls_in_table) 
# Here I only ran the first two issues, but this should work for all if you get rid of [1:2]

senate_articles <- as.data.frame(do.call(rbind, senate_articles)) %>% # Take data frames out of lists
  mutate(real_url = paste("https://www.congress.gov", ., sep = "")) %>%
  filter(grepl("article",real_url))

# Layer 3
senate_content <- lapply(senate_articles$real_url[1:3],get_content)
# Again only running the first few, but should work on a larger scale
senate_content <- as.data.frame(do.call(rbind, senate_content)) # Data clean should follow this

And got the 403 Forbidden error message. I have tried to change the user agent to different names without any success, including my actual email address (and not the name@xxx.edu used in the code above), the name of my thesis, my name, and my name with spaces before my email address. Nothing seems to work.

That 403 response actually includes a Cloudflare javascript-based challange (screenshot forom Chromote session) and it's quite effective with blocking static scrapers and HTTP request tools like httr(2) and revest.

But it also detects curl requests with fully configured headers (i.e. Copy as cURL call from browser's dev tools; and there we have user agent, cookies and other headers cloned from our fully functional browser session).

Though Chromote, once it's unique user-agent is updated, is currently able to get through (with default user-agent it would still trigger that js challenge). In the example bellow I only included Chromote session handling and replaced httr::GET() calls with Chromote calls. And I wasn't so keen on waiting 30 seconds between requests... BTW, nice reproducible example there, no additional changes were required!

library(tidyverse)
library(rvest)

### update /
library(chromote)
# new sessio, set userAgent, value grabbed from current Chrome wof Windows
b <- ChromoteSession$new()
b$Network$setUserAgentOverride(userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36")
#> named list()
### / update 

# Functions
get_urls_in_table <- function(master_link) {
  ### update /
  {
    b$Page$navigate(master_link)
    b$Page$loadEventFired()
  } 
  with_user_agent <- b$Runtime$evaluate("document.querySelector('html').outerHTML")$result$value
  ### / update
  links <- read_html(with_user_agent) %>% html_nodes("td a") %>% html_attr('href') %>% as.data.frame()
  return(links)
}

get_content <- function(x) {
  ### update /  
  {
    b$Page$navigate(x)
    b$Page$loadEventFired()
  } 
  with_user_agent <- b$Runtime$evaluate("document.querySelector('html').outerHTML")$result$value
  ### / update
  content <- read_html(with_user_agent) %>% html_nodes(".styled") %>% html_text()
  print("getting content")
  # Sys.sleep(30) # Here add 30 sec between each scraping attempt to prevent the queries from becoming too frequent
  return(content)
}

# Layer 1
senate_pages <- get_urls_in_table("https://www.congress.gov/congressional-record/108th-congress/browse-by-date")

senate_pages <- senate_pages %>%
  mutate(real_url = paste("https://www.congress.gov", ., sep = "")) %>%
  filter(grepl("senate",real_url))

# Layer 2
senate_articles <- lapply(senate_pages$real_url[1:2],get_urls_in_table) 
# Here I only ran the first two issues, but this should work for all if you get rid of [1:2]

senate_articles <- as.data.frame(do.call(rbind, senate_articles)) %>% # Take data frames out of lists
  mutate(real_url = paste("https://www.congress.gov", ., sep = "")) %>%
  filter(grepl("article",real_url))

# Layer 3
senate_content <- lapply(senate_articles$real_url[1:3],get_content)
#> [1] "getting content"
#> [1] "getting content"
#> [1] "getting content"
# Again only running the first few, but should work on a larger scale
senate_content <- as.data.frame(do.call(rbind, senate_content)) # Data clean should follow this

### update /
# close Chromote session
b$close()
#> [1] TRUE

Result:

# senate_content:
as_tibble(senate_content)
#> # A tibble: 3 × 1
#>   V1                                                                            
#>   <chr>                                                                         
#> 1 "\n[Pages S12089-S12092]\nFrom the Congressional Record Online through the Go…
#> 2 "\n[Page S12092]\nFrom the Congressional Record Online through the Government…
#> 3 "\n[Pages S12092-S12093]\nFrom the Congressional Record Online through the Go…

# beginning of the 1st article:
cat(substr(senate_content[1,1],1,400))
#> 
#> [Pages S12089-S12092]
#> From the Congressional Record Online through the Government Publishing Office [www.gpo.gov]
#> 
#> 
#> 
#> 
#>                           OFFSHORE OUTSOURCING
#> 
#>   Mr. LIEBERMAN. Mr. President, on December 15, 2004, my office 
#> released a white paper entitled ``Data Dearth in Offshore Outsourcing: 
#> Policymaking Requires Facts.'' This white paper is closely linked to a 
#> previous white paper ent

^{Created on 2023-05-22 with reprex v2.0.2}