1
2

# Load data from previous post
load(file = "path/to/file/Summary_data_countries_commemorated.rda")

1
2
3
4
5

# Order data by number of records (descending)
cwgc_summary <- cwgc_summary[order(-cwgc_summary$sum),]

# Show the top 10 values
knitr::kable(head(cwgc_summary, 10), row.names = FALSE, caption = "Number of Records by Country and War")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Filter data frame for countries with fewer than 80,000 records
cwgc_set_1 <- cwgc_summary[cwgc_summary$sum < 80000,]

# Discard countries with zero graves
cwgc_set_1 <- cwgc_set_1[cwgc_set_1$sum != 0,]

# Get vector of countries 
cwgc_set_1 <- as.character(cwgc_set_1$commemorated)

# Remove 'Civilian War Dead' category
cwgc_set_1 <- cwgc_set_1[2:length(cwgc_set_1)]

1
2
3
4
5

# Filter data frame for countries with an individual sum > 80,000
cwgc_set_2 <- cwgc_summary[cwgc_summary$sum > 80000,]

# Get vector of countries 
cwgc_set_2 <- as.character(cwgc_set_2$commemorated)

1
2
3
4
5
6
7
8

# URL to be scraped
war_dead_url <- "https://www.cwgc.org/find/find-war-dead"

# Set up HTML session for scraping
pgsession <- html_session(war_dead_url)

# Parse form fields from URL
page_form <- html_form(pgsession)[[3]]

1
2
3
4

# Set form fields with country to be queried
# Not working as expected
filled_form <- set_values(page_form,
                          "countryCommemoratedAt" = 'India')

1
2
3
4
5
6
7

# URL pattern to reproduce
# https://www.cwgc.org/find/find-war-dead/results?country=Germany

# Construct URL
baseurl <- "https://www.cwgc.org/find/find-war-dead/results?country="
mainurl <- paste(baseurl, country, sep = "")
mainurl <- gsub(" ", "%2B", mainurl) # Replace spaces if present

1
2
3
4
5
6
7
8
9

# Read HTML contents
dat <- read_html(mainurl)

# Get contents of HTML element
dat_html <- html_nodes(dat,
                       xpath = "/html/body/main/section[2]/div/div[1]/div/div/div/div/a")

# Get attribute
dat_attr <- html_attr(dat_html, "href")

1
2

# Concatenate URL and relative URL
download_address <- paste0("https://www.cwgc.org", dat_attr)

1
2

# Download file from URL - the curl method removes blank rows in the output
download.file(download_address, method = "curl", "India.csv")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

# Set path for downloaded files
path <- "./Set1/"

# URL to be scraped
war_dead_url <- "https://www.cwgc.org/find/find-war-dead"

# Set up HTML session for scraping
pgsession <- html_session(war_dead_url)

# Loop to step through countries 
for (country in cwgc_set_1){
  
  # Console output
  cat("Working on", country, "\n")
  
  # Construct URL
  baseurl <- "https://www.cwgc.org/find/find-war-dead/results?country="
  mainurl <- paste(baseurl, country, sep = "")
  mainurl <- gsub(" ", "%2B", mainurl)
  
  # Read HTML contents
  dat <- read_html(mainurl)
  
  # Get contents of HTML element with XPath option
  dat_html <- html_nodes(dat, 
                         xpath = "/html/body/main/section[2]/div/div[1]/div/div/div/div/div[2]/a")
  
  # Get href attribute
  dat_attr <- html_attr(dat_html, "href")
  
  # Concatenate URL and relative URL
  download_address <- paste0("https://www.cwgc.org", dat_attr)
  
  # Set up file name for each country
  file_name <- paste0(path, country, ".csv")
  
  # Console output
  cat("Downloading", country, "\n")
  
  # Download file from URL
  # This function's method parameter depends on your OS, so pick accordingly
  download.file(download_address, method = "curl", file_name)
  
  # Wait between 100 and 200 seconds before next page request
  Sys.sleep(sample(seq(100, 200, 1), 1))
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

# Set path for downloaded files
path <- "./Set1/"

# List of *.csv files 
csv_files <- list.files(path, pattern = "\\.csv$")

# Initialise object for output
out_file <- c()

# Loop through each file
for (my_file in csv_files){
  
  # Set up file name
  file_name <- paste0(path, my_file)
  
  # Read in file
  dat <- read.csv(file_name, header = TRUE, sep = ",")
  
  # Console output for progress indicator
  cat(my_file, " \n")
  
  # Bind results
  out_file <- rbind(out_file, dat)
}

# Save output to file
save(out_file, file = "./path/to/file/Set_1_cwgc_graves.rda")

1
2
3

# Graves per country in Set 2
knitr::kable(cwgc_summary[cwgc_summary$commemorate %in% cwgc_set_2,],
             row.names = FALSE, caption = "Records in Set 2 Countries")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Read in date splits
dates <- read.csv(file = "Dates.csv",
                  sep = ",",
                  header = FALSE,
                  stringsAsFactors = FALSE)

# Add column names
colnames(dates) <- c("Country", "Start", "End")

# Show table
knitr::kable(dates, row.names = FALSE, caption = "Set 2 Date Splits")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

# URL structure to replicate
#.../find-war-dead/results?country=Belgium&dateFrom=01-01-1915&dateTo=01-01-1916

# URL to be scraped
war_dead_url <- "https://www.cwgc.org/find/find-war-dead"

# Set up HTML session for scraping
pgsession <- html_session(war_dead_url)

# Read contents of webpage into XML document
webpage <- read_html(pgsession)

# Loop through dates and country
for(i in 1:nrow(dates)){
  
  # Get country
  country <- dates$Country[i]
  
  # Replace spaces with '+' in country name
  country <- gsub(" ", "+", country)
  
  # Get start date
  start_date <- dates$Start[i]
  # Change '/' to '-'
  start_date <- gsub("/", "-", start_date)

  # Get end date
  end_date <- dates$End[i]
  # Change '/' to '-'
  end_date <- gsub("/", "-", end_date)

  # Set path
  path <- "./Set2/"
  
  # Set up file name
  file_name <- paste0(path, country, start_date, end_date, ".csv")
  
  # Construct URL
  part_1 <- "https://www.cwgc.org/find/find-war-dead/results?"
  part_2 <- paste0("country=", country)
  part_3 <- paste0("&dateFrom=", start_date)
  part_4 <- paste0("&dateTo=", end_date)
  request_URL <- paste0(part_1, part_2, part_3, part_4)
  
  # Get contents of webpage
  webpage <- read_html(request_URL)
  
  # Get 'Download Results' element
  dat_html <- html_nodes(webpage, 
                         xpath = "/html/body/main/section[2]/div/div[1]/div/div/div/div/a")
  
  # Get href attribute
  dat_attr <- html_attr(dat_html, "href")
  
  # Concatenate URL and relative URL
  download_address <- paste0("https://www.cwgc.org", dat_attr)
  
  # Download file from URL
  download.file(download_address, file_name)
  
  # Wait between 30 and 60 seconds before next page request
  Sys.sleep(sample(seq(30, 60, 1), 1))
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

# Set path for downloaded files
path <- "./Set2/"

# List of *.csv files 
csv_files <- list.files(path, pattern = "\\.csv$")

# Initialise object for output
out_file <- c()

# Loop through each file
for (my_file in csv_files){
  
  # Set up file name
  file_name <- paste0(path, my_file)
  
  # Read in file
  dat <- read.csv(file_name, header = TRUE, sep = ",")
  
  # Console output for progress indicator
  cat(my_file, " \n")
  
  # Bind results
  out_file <- rbind(out_file, dat)
}

# Save output to file
save(out_file, file = "./path/to/file/Set_2_cwgc_graves.rda")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# Load data from file
load(file = "./path/to/file/Set_1_cwgc_graves.rda")
set1 <- out_file

# Load data from file
load(file = "./path/to/file/Set_2_cwgc_graves.rda")
set2 <- out_file

# Row bind
cwgc_graves <- rbind(set1, set2)

# Save output to file
save(cwgc_graves, file = "./path/to/file/All_cwgc_graves.rda")