rvest: Adapt webscraping function to continue if it comes across no data

The following function works well for scraping the Begin: and End: dates for different datasets from this dataset repository. However, not every dataset's metadata includes Begin: and End: which causes the function to stop and throw an error message. Can the function be adapted to skip to the next URL (package_id) rather than stop?

library(tidyverse)
library(rvest)
library(janitor)

# Function for scraping begin and end dates from different metadata URLs

scraper <- function(package_id) {
  cat("Scraping", package_id, "\n")
  data <- str_c("https://portal.edirepository.org/nis/metadataviewer?packageid=",
                package_id) %>%
    read_html() %>%
    html_elements(".subgroup.onehundred_percent") %>%
    pluck(1) %>%
    html_elements(".roweven") %>%
    html_text2() 
  
  tibble(begin = pluck(data, 1), 
         end = pluck(data, 1))
}

# Grabs package_id

good_page <-
  "http://portal.edirepository.org:80/nis/simpleSearch?defType=edismax&q=*:*&fq=-scope:ecotrends&fq=-scope:lter-landsat*&fq=scope:(knb-lter-kbs)&fl=id,packageid,title,author,organization,pubdate,coordinates&debug=false&start=0&rows=3" %>%
  read_html()

bad_page <-
  "http://portal.edirepository.org:80/nis/simpleSearch?defType=edismax&q=*:*&fq=-scope:ecotrends&fq=-scope:lter-landsat*&fq=scope:(knb-lter-kbs)&fl=id,packageid,title,author,organization,pubdate,coordinates&debug=false&start=40&rows=4" %>%
  read_html()

# Works as it should, returns publication_date, package_id, begin, and end

dat1 <- good_page %>%
  html_table() %>%
  pluck(4) %>%
  clean_names() %>%
  mutate(across(title, ~ str_squish(str_remove_all(., "\\n")))) %>%
  mutate(date = map(package_id, scraper)) %>% 
  unnest(date) %>%
  select(!c(title,creators))
dat1

# Breaks because knb-lter-kbs.141.1 does not include begin or end dates
# Can the function above be amended to skip to the next package_id rather than stop?

dat2 <- bad_page %>%
  html_table() %>%
  pluck(4) %>%
  clean_names() %>%
  mutate(across(title, ~ str_squish(str_remove_all(., "\\n")))) %>%
  mutate(date = map(package_id, scraper)) %>% 
  unnest(date) %>%
  select(!c(title,creators))

Using the purrr::possibly() function operator, you can define a version of your function that returns NA values when an error occurs:

library(tidyverse)
library(rvest)
library(janitor)

safe_scraper <- possibly(
  scraper, 
  otherwise = tibble(begin = NA_character_, end = NA_character_)
)

bad_page %>%
  html_table() %>%
  pluck(4) %>%
  clean_names() %>%
  mutate(
    title = str_squish(str_remove_all(title, "\\n")),
    date = map(package_id, safe_scraper)
  ) %>% 
  unnest(date) %>%
  select(!c(title, creators))

# A tibble: 4 × 4
  publication_date package_id          begin                                               end                                          
             <int> <chr>               <chr>                                               <chr>                                        
1             2018 knb-lter-kbs.139.5  2008-01-01                                          2008-01-01                                   
2             2018 knb-lter-kbs.140.5  NA                                                  NA                                           
3             2016 knb-lter-kbs.141.1  Information Manager (LTER Network Office) [ email ] Information Manager (LTER Network Office) [ …
4             2018 knb-lter-kbs.162.79 1950-01-01                                          1950-01-01

Judging by row 3, though, looks like there may be another issue in your workflow.

Changed from html approach to XML for more consistent data extraction

library(httr2)
library(tidyverse)
library(janitor)

bad_page <-
  "http://portal.edirepository.org:80/nis/simpleSearch?defType=edismax&q=*:*&fq=-scope:ecotrends&fq=-scope:lter-landsat*&fq=scope:(knb-lter-kbs)&fl=id,packageid,title,author,organization,pubdate,coordinates&debug=false&start=40&rows=4" %>%
  read_html()

scraper <- function(package_id) {
  cat("Scraping", package_id, "\n")
  data <- str_c(
    "https://portal.edirepository.org/nis/metadataviewer?packageid=",
    package_id, 
    "&contentType=application/xml"
  ) %>%
    request() %>%
    req_perform() %>%
    resp_body_xml() %>%
    xml_find_all(".//calendarDate") %>%
    xml_text()
  
  tibble(begin = pluck(data, 1),
         end =   pluck(data, 1))
}

data <- bad_page %>%
  html_table() %>%
  pluck(4) %>%
  clean_names() %>%
  mutate(across(title, ~ str_squish(str_remove_all(., "\\n")))) %>%
  mutate(date = map(package_id, possibly(
    scraper, otherwise = tibble(begin = NA, end = NA)
  ))) %>%
  unnest(date, keep_empty = TRUE) %>%
  select(!c(title, creators))

The result:

# A tibble: 4 × 4
  publication_date package_id          begin      end       
             <int> <chr>               <chr>      <chr>     
1             2018 knb-lter-kbs.139.5  2008-01-01 2008-01-01
2             2018 knb-lter-kbs.140.5  NA         NA        
3             2016 knb-lter-kbs.141.1  NA         NA        
4             2018 knb-lter-kbs.162.79 1950-01-01 1950-01-01