I have this code to try and extract the species from the law found here https://laws.justice.gc.ca/fra/lois/S-15.3/TexteComplet.html

However, I’m not able to make the html_nodes find each section

  section <- div_content %>% html_nodes(xpath = paste0("//h2[contains(text(), '", header, "')]/following-sibling::div[contains(@class, 'ProvisionList')]"))

Basically, I can’t find a way to get the text content and match the other sections. I’ve tried to add the “
” tag and find the text for each section, but it doesnt work (get a {xml_nodeset (0)})

I’m trying to get the data found in div with id “425426”, then, within the scheduleLabel, get text from scheduleTitleText. I need another column for SchedHeadL1 (which is the title of the sections with the species) and the text found in BilingualGroupTitleText (stating the group of animal or plants…). Then provide a nested list of species (here I’m seperating the species from french name, latin and english)

library(rvest)
library(dplyr)
library(stringr)

# URL of the webpage
url <- "https://laws.justice.gc.ca/fra/lois/S-15.3/TexteComplet.html"

# Read the webpage content
webpage <- read_html(url)

# Extract the div with id "425426"
div_content <- webpage %>% html_node("#425426")

# Extract the header h2 with class "scheduleTitleText" from the class "scheduleLabel" and id "h-425427"
schedule_label <- div_content %>% html_node("h2.scheduleLabel#h-425427") %>% html_text()

# Extract all h2 headers with class "SchedHeadL1"
headers <- div_content %>% html_nodes("h2.SchedHeadL1") %>% html_text()


# Use str_extract to extract the "PARTIE #" part
partie_numbers <- str_extract(headers, "PARTIE \d+")

# Use str_remove to remove the "PARTIE #" part from the original strings
descriptions <- str_remove(headers, "PARTIE \d+")

# Combine into a data frame
result <- data.frame(Partie = partie_numbers, Description = descriptions, stringsAsFactors = FALSE)

headers_prep = result |> 
  unite(pd, Partie, Description, sep = "<br>") |> pull(pd)

# Initialize lists to store the extracted data
group_titles <- list()
item_first <- list()
item_second <- list()
scientific_names <- list()
latin_names <- list()

# Loop through each header to extract the associated content
for (header in headers) {
  # Extract the section associated with the current header
  section <- div_content %>% html_nodes(xpath = paste0("//h2[contains(text(), '", header, "')]/following-sibling::div[contains(@class, 'ProvisionList')]"))
  
  # Extract BilingualGroupTitleText within the section
  group_title <- section %>% html_nodes(".BilingualGroupTitleText") %>% html_text()
  group_titles <- c(group_titles, group_title)
  
  # Extract BilingualItemFirst within the section
  item_first_section <- section %>% html_nodes(".BilingualItemFirst") %>% html_text()
  item_first <- c(item_first, item_first_section)
  
  # Extract BilingualItemSecond within the section
  item_second_section <- section %>% html_nodes(".BilingualItemSecond") %>% html_text()
  item_second <- c(item_second, item_second_section)
  
  # Extract otherLang (scientific names) within the section
  scientific_name_section <- section %>% html_nodes(".otherLang") %>% html_text()
  scientific_names <- c(scientific_names, scientific_name_section)
  
  # Extract scientific Latin names from BilingualItemFirst
  latin_name_section <- str_extract(item_first_section, "\(([^)]+)\)") %>% str_replace_all("[()]", "")
  latin_names <- c(latin_names, latin_name_section)
}

# Ensure all columns have the same length by repeating the last element if necessary
max_length <- max(length(headers), length(group_titles), length(item_first), length(item_second), length(scientific_names), length(latin_names))

schedule_label <- rep(schedule_label, length.out = max_length)
headers <- rep(headers, length.out = max_length)
group_titles <- rep(group_titles, length.out = max_length)
item_first <- rep(item_first, length.out = max_length)
item_second <- rep(item_second, length.out = max_length)
scientific_names <- rep(scientific_names, length.out = max_length)
latin_names <- rep(latin_names, length.out = max_length)

# Create a data frame
data <- data.frame(
  ScheduleLabel = schedule_label,
  Header = headers,
  GroupTitle = group_titles,
  ItemFirst = item_first,
  ItemSecond = item_second,
  ScientificName = scientific_names,
  LatinName = latin_names,
  stringsAsFactors = FALSE
)