Downloading images from web and its attributes in R.

huangapple
go评论103阅读模式
英文:

Downloading images from web and its attributes in R

问题

我要下载此网站 https://moweek.com.uy/ 中的图像，具体来说，那些属于类 = "expandedCategory" 的图像: "VESTIMENTA", "CALZADO", "ACCESORIOS" (data-category-id="1", data-category-id="2" 和 data-category-id="3")。例如，进入"VESTIMENTA"下的子类别（<div id="expandedCategoryContainer">），除了"Ver todo"之外，像"Activewear"一样点击所有图像并获取数据集中的所有信息以及图像。

我尝试过这样做，但没有成功。

英文:

I want to download the images of this site https://moweek.com.uy/ in R, specifically, those under the class = "expandedCategory": "VESTIMENTA", "CALZADO", "ACCESORIOS" (data-category-id="1", data-category-id="2", and data-category-id="3"). For example, go into the sub categories (<div id="expandedCategoryContainer">) under "VESTIMENTA" (except from "Ver todo") like "Activewear" (<a data-category-id="57" href="/vestimenta/activewear/1" class=" categoryLevelTwoTitle selected'>Activewear</a> == $0) clicking in all the images and obtain all of the information in a dataset as well as the images.

I've trying to do this but I am failing.

  1. pacman::p_load(tidyverse, rvest, httr)
  3. url &lt;- &quot;https://moweek.com.uy/&quot;
  4. html_content &lt;- GET(url)
  5. webpage &lt;- read_html(content(html_content, as = &quot;text&quot;))
  7. category_nodes &lt;- html_nodes(webpage, &quot;.expandedCategory&quot;)
  9. category_urls &lt;- lapply(category_nodes, function(node) html_nodes(node, &quot;a&quot;) %&gt;% 
  10.                           html_attr(&quot;href&quot;)) %&gt;% 
  11.   unlist() %&gt;% 
  12.   str_subset(&quot;/vestimenta/|/calzado/|/accesorios/&quot;)
  13.                  
  14. image_data &lt;- data.frame()
  15. for (url in category_urls) {
  16.   cat_url &lt;- paste0(&quot;https://moweek.com.uy&quot;, url)
  17.   cat_content &lt;- GET(cat_url)
  18.   cat_page &lt;- read_html(content(cat_content, as = &quot;text&quot;))
  19.   
  20.   # Extract the category name and the subcategory name from the page title
  21.   cat_name &lt;- html_text(html_node(cat_page, &quot;title&quot;)) %&gt;%
  22.     str_replace(&quot;- MoWeek&quot;, &quot;&quot;) %&gt;%
  23.     str_to_title()
  24.   subcat_name &lt;- html_text(html_node(cat_page, &quot;.categoryLevelTwoTitle&quot;)) %&gt;%
  25.     str_to_title()
  26.   
  27.   # Extract the image information from the page and store it in a data frame
  28.   image_tags &lt;- html_nodes(cat_page, &quot;.productItem&quot;)
  29.   image_info &lt;- data.frame()
  30.   for (tag in image_tags) {
  31.     name &lt;- html_text(html_node(tag, &quot;.productTitle&quot;))
  32.     price &lt;- html_text(html_node(tag, &quot;.priceText&quot;))
  33.     img_url &lt;- html_attr(html_node(tag, &quot;img&quot;), &quot;src&quot;)
  34.     image_info &lt;- image_info %&gt;%
  35.       add_row(Category = cat_name,
  36.               Subcategory = subcat_name,
  37.               Name = name,
  38.               Price = price,
  39.               Image_URL = img_url)
  40.   }
  41.   image_data &lt;- image_data %&gt;% bind_rows(image_info)
  42. }
  44. for (i in 1:nrow(image_data)) {
  45.   img &lt;- GET(image_data$Image_URL[i])
  46.   category &lt;- image_data$Category[i]
  47.   subcategory &lt;- image_data$Subcategory[i]
  48.   category_folder &lt;- gsub(&quot; &quot;, &quot;_&quot;, category)
  49.   subcategory_folder &lt;- gsub(&quot; &quot;, &quot;_&quot;, subcategory)
  50.   
  51.   if (!dir.exists(category_folder)) {
  52.     dir.create(category_folder)
  53.   }
  54.   if (!dir.exists(paste0(category_folder, &quot;/&quot;, subcategory_folder))) {
  55.     dir.create(paste0(category_folder, &quot;/&quot;, subcategory_folder))
  56.   }
  57.   
  58.   file_name &lt;- paste
  59. }

答案1

得分: 1

这是您提供的代码的翻译：

  1. library(tidyverse)
  2. library(rvest)
  3. library(httr)
  5. url <- "https://moweek.com.uy/"
  6. html_content <- GET(url)
  7. webpage <- read_html(content(html_content, as = "text"))
  9. category_nodes <- html_nodes(webpage, ".expandedCategory")
  11. category_urls <- lapply(category_nodes, function(node) html_nodes(node, "a") %>%
  12.                           html_attr("href")) %>%
  13.   unlist() %>%
  14.   str_subset("/vestimenta/|/calzado/|/accesorios/")
  16. image_data <- tibble()
  17. for (url in category_urls) {
  18.   cat_url <- paste0("https://moweek.com.uy", url)
  19.   # 因为我们发送了很多请求，有些可能会失败
  20.   cat_content <- RETRY("GET", cat_url)
  21.   cat_page <- read_html(content(cat_content, as = "text"))
  23.   # 从页面标题中提取类别名称和子类别名称
  24.   cat_name <- html_text(html_node(cat_page, "title")) %>%
  25.     str_replace("- MoWeek", "") %>%
  26.     str_to_title()
  27.   subcat_name <- html_text(html_node(cat_page, ".categoryLevelTwoTitle")) %>%
  28.     str_to_title()
  30.   # 从页面中提取图像信息并将其存储在数据框中
  31.   image_tags <- html_nodes(cat_page, ".productViewContainer")
  32.   # 初始化列以便 bind_rows() 正常工作
  33.   image_info <- tibble(
  34.     Category = character(),
  35.     Subcategory = character(),
  36.     Name = character(),
  37.     Price = character(),
  38.     Image_URL = character()
  39.   )
  40.   for (tag in image_tags) {
  41.     name <- html_text(html_node(tag, ".productViewName"))
  42.     price <- html_text(html_node(tag, ".productViewPrice"))
  43.     img_url <- html_attr(html_node(tag, ".productViewTop"), "data-hover-image")
  44.     image_info <- image_info %>%
  45.       add_row(Category = cat_name,
  46.               Subcategory = subcat_name,
  47.               Name = name,
  48.               Price = price,
  49.               Image_URL = img_url)
  50.   }
  51.   image_data <- image_data %>% bind_rows(image_info)
  52. }
  54. # 清理数据
  55. image_data <- image_data %>%
  56.   mutate(
  57.     Name = str_trim(Name),
  58.     Price = str_trim(Price)
  59.   )
  61. # 获取每个图像的文件名
  62. file_names <- str_match(image_data$Image_URL, ".+(/.*?$)")[,2] %>%
  63.   str_sub(start = 2L)
  65. for (i in seq_len(nrow(image_data))) {
  66.   url <- image_data$Image_URL[i]
  67.   category <- image_data$Category[i]
  68.   subcategory <- image_data$Subcategory[i]
  69.   category_folder <- gsub(" ", "_", category)
  70.   subcategory_folder <- gsub(" ", "_", subcategory)
  72.   if (!dir.exists(category_folder)) {
  73.     dir.create(category_folder)
  74.   }
  75.   if (!dir.exists(paste0(category_folder, "/", subcategory_folder))) {
  76.     dir.create(paste0(category_folder, "/", subcategory_folder))
  77.   }
  79.   # 下载并将图像存储在正确的目录中
  80.   dir_name <- paste0(category_folder, "/", subcategory_folder, "/")
  81.   file_name <- file_names[i]
  82.   download.file(url, paste0(dir_name, file_name))
  83. }
英文:

This code seemed to work for me.

  1. library(tidyverse) 
  2. library(rvest)
  3. library(httr)
  5. url &lt;- &quot;https://moweek.com.uy/&quot;
  6. html_content &lt;- GET(url)
  7. webpage &lt;- read_html(content(html_content, as = &quot;text&quot;))
  9. category_nodes &lt;- html_nodes(webpage, &quot;.expandedCategory&quot;)
  11. category_urls &lt;- lapply(category_nodes, function(node) html_nodes(node, &quot;a&quot;) %&gt;% 
  12.                           html_attr(&quot;href&quot;)) %&gt;% 
  13.   unlist() %&gt;% 
  14.   str_subset(&quot;/vestimenta/|/calzado/|/accesorios/&quot;)
  16. image_data &lt;- tibble()
  17. for (url in category_urls) {
  18.   cat_url &lt;- paste0(&quot;https://moweek.com.uy&quot;, url)
  19.   # Because we are sending so many requests, some are likely to fail
  20.   cat_content &lt;- RETRY(&quot;GET&quot;, cat_url)
  21.   cat_page &lt;- read_html(content(cat_content, as = &quot;text&quot;))
  22.   
  23.   # Extract the category name and the subcategory name from the page title
  24.   cat_name &lt;- html_text(html_node(cat_page, &quot;title&quot;)) %&gt;%
  25.     str_replace(&quot;- MoWeek&quot;, &quot;&quot;) %&gt;%
  26.     str_to_title()
  27.   subcat_name &lt;- html_text(html_node(cat_page, &quot;.categoryLevelTwoTitle&quot;)) %&gt;%
  28.     str_to_title()
  29.   
  30.   # Extract the image information from the page and store it in a data frame
  31.   image_tags &lt;- html_nodes(cat_page, &quot;.productViewContainer&quot;)
  32.   # Initialise columns so bind_rows() works
  33.   image_info &lt;- tibble(
  34.     Category = character(),
  35.     Subcategory = character(),
  36.     Name = character(),
  37.     Price = character(),
  38.     Image_URL = character()
  39.   )
  40.   for (tag in image_tags) {
  41.     name &lt;- html_text(html_node(tag, &quot;.productViewName&quot;))
  42.     price &lt;- html_text(html_node(tag, &quot;.productViewPrice&quot;))
  43.     img_url &lt;- html_attr(html_node(tag, &quot;.productViewTop&quot;), &quot;data-hover-image&quot;)
  44.     image_info &lt;- image_info %&gt;%
  45.       add_row(Category = cat_name,
  46.               Subcategory = subcat_name,
  47.               Name = name,
  48.               Price = price,
  49.               Image_URL = img_url)
  50.   }
  51.   image_data &lt;- image_data %&gt;% bind_rows(image_info)
  52. }
  54. # Clean data
  55. image_data &lt;- image_data %&gt;%
  56.   mutate(
  57.     Name = str_trim(Name),
  58.     Price = str_trim(Price)
  59.   )
  61. # Get the file name of each image
  62. file_names &lt;- str_match(image_data$Image_URL, &quot;.+(/.*?$)&quot;)[,2] %&gt;%
  63.   str_sub(start = 2L)
  65. for (i in seq_len(nrow(image_data))) {
  66.   url &lt;- image_data$Image_URL[i]
  67.   category &lt;- image_data$Category[i]
  68.   subcategory &lt;- image_data$Subcategory[i]
  69.   category_folder &lt;- gsub(&quot; &quot;, &quot;_&quot;, category)
  70.   subcategory_folder &lt;- gsub(&quot; &quot;, &quot;_&quot;, subcategory)
  71.   
  72.   if (!dir.exists(category_folder)) {
  73.     dir.create(category_folder)
  74.   }
  75.   if (!dir.exists(paste0(category_folder, &quot;/&quot;, subcategory_folder))) {
  76.     dir.create(paste0(category_folder, &quot;/&quot;, subcategory_folder))
  77.   }
  78.   
  79.   # Download and store image in correct directory
  80.   dir_name &lt;- paste0(category_folder, &quot;/&quot;, subcategory_folder, &quot;/&quot;)
  81.   file_name &lt;- file_names[i]
  82.   download.file(url, paste0(dir_name, file_name))
  83. }

huangapple
  • 本文由 发表于 2023年4月11日 00:21:49
  • 转载请务必保留本文链接：https://go.coder-hub.com/75978803.html
  • httr
  • r
  • rvest
  • web-scraping
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定