Data Masking

Data masking makes data manipulation faster because it requires less typing. In most (but not all1) base R functions you need to refer to variables with $, leading to code that repeats the name of the data frame many times:

starwars
starwars[starwars$homeworld == "Naboo" & starwars$species == "Human", ]

The dplyr equivalent of this code is more concise because data masking allows you to need to type starwars once:

filter(starwars, homeworld == "Naboo" & species == "Human")

Data- and env-variables The key idea behind data masking is that it blurs the line between the two different meanings of the word “variable”:

• env-variables are “programming” variables that live in an environment. They are usually created with <-

• data-variables are “statistical” variables that live in a data frame. They usually come from data files (e.g. .csv, .xls), or are created manipulating existing variables.

To make those definitions a little more concrete, take this piece of code:

df <- data.frame(x = runif(3), y = runif(3))
df$x

It creates a env-variable, df, that contains two data-variables, x and y. Then it extracts the data-variable x out of the env-variable df using $.

Tidy Select

Data masking makes it easy to compute on values within a dataset. Tidy selection is a complementary tool that makes it easy to work with the columns of a dataset. This provides a miniature domain specific language that makes it easy to select columns by name, position, or type. For example:

select(df, 1) # selects the first column 
select(df, last_col()) # selects the last column
select(df, c(x, y)) # selects columns x, and y
select(df, starts_with("x")) # selects all columns whose name starts with “x”
select(df, ends_with("y")) # selects all columns whose name ends with “y”
select(df, where(is.numeric)) # selects all numeric columns

Modify

str_c("hello", "world")
str_c("hello", "world", sep = " ")

str_dup("hello", 5)

str_flatten(c("hello", "world"))
str_flatten(c("hello", "world"), collapse = "_ _")

# Padding
str_pad("hello", width = 8, side = "left", pad = "x")
str_pad("hello", width = 8, side = "right", pad = "y")
str_pad("hello", width = 7, side = "both", pad = "z")
str_pad("hello", 10, pad = c("-", "_", " "))

# Case
str_to_lower("HEllO")
str_to_upper("hello")
str_to_title("hello world")
str_to_sentence("hello world")

# Sorting
str_order(c("world", "hello"))
str_sort(c("world", "hello"))

# Splitting
str_split("hello", "e")
str_split("hello", "e") %>% unlist()

x <- "This string is moderately long"
rbind(
  str_trunc(x, 20, "right"),
  str_trunc(x, 20, "left"),
  str_trunc(x, 20, "center")
)

# Trimming
str_trim("hello")
str_trim("hello ")
str_trim("  hello   world  ")

str_squish("hello")
str_squish("hello ")
str_squish("  hello   world  ")

Count Patterns

str_length("hello")

str_count("hello", "h") # h letter
str_count("hello;", "\\w") # word + numbers
str_count("hello", "\\d") # digits

Detect Patterns

str_locate("hello", "e")
str_locate("hello", "l")

str_detect("hello", "h")
str_detect("hello", "\\w")
str_detect("hello", "\\d")

str_which(c("hello", "world"), "ll")

str_starts("hello", "h")
str_ends("hello", "\\s") # space

Extract Patterns

str_sub("hello", 2, 3)
str_sub("hello world", 4, 8)

str_extract("hello", "el")
str_extract("hello", "lo")
str_extract(c("hello", "world"), "w")
str_extract_all(c("hello", "world"), "w")
str_extract_all(c("hello", "world"), "w") %>% unlist()

Removing & Replacing Patterns

str_remove("hello", "h")
str_remove(c("hello", "world"), "l")
str_remove_all(c("hello world", "hello world"), "l")

str_replace("hello", "h", "c")
str_replace(c("hello world", "hello world"), "l", "t")
str_replace_all(c("hello world", "hello world"), "l", "t")

Testing Patterns

str_view("hello world!", "world")
str_view("hello world!", "\\d")
str_view("hello world!", "\\w")
str_view("hello world!", "\\w*")
str_view("hello world!", "\\w*\\s\\w+")
str_view("hello world!", "\\w{3}")

Base R Functions

grep("e", "hello")
grep("e", c("hello", "world"))
grep("e", c("hello", "world"), value = TRUE)
grepl("e", c("hello", "world"))
gsub("e", "a", "hello")
nchar("hello")
paste("hello", "world")
tolower("HEllO")
toupper("hello")

Count Levels

levels(gss_cat$partyid)

fct_count(gss_cat$partyid)
gss_cat %>% count(partyid)

Order Levels

levels(gss_cat$rincome)

We can use the function fct_relevel() when we need to manually reorder our factor levels. In addition to the factor, you give it a character vector of level names, and specify where you want to move them. It defaults to moving them to the front, but you can move them after another level with the argument after. If you want to move it to the end, you set after equal to Inf.

fct_relevel(gss_cat$rincome, c("Lt $1000", "$1000 to 2999")) %>%
  levels()

It’s often useful to change the order of the factor levels in a visualisation. We can improve visualization by reordering the levels of relig using fct_reorder().

rincome_summary <- gss_cat %>%
  group_by(rincome) %>%
  summarise(
    age = mean(age, na.rm = TRUE),
    tvhours = mean(tvhours, na.rm = TRUE),
    n = n()
  )

# Reorder income by average age  
levels(rincome_summary$rincome)
fct_reorder(rincome_summary$rincome, rincome_summary$age) %>%
  levels()

# unordered chart, difficult to read
rincome_summary %>%
  ggplot(aes(age, rincome)) +
  geom_point()

# ordered chart, easy to read
rincome_summary %>%
  mutate(rincome = fct_reorder(rincome, age)) %>%
  ggplot(aes(age, rincome)) +
  geom_point()

Finally, for bar plots, you can use fct_infreq() to order levels in increasing frequency: this is the simplest type of reordering because it doesn’t need any extra variables. You may want to combine with fct_rev().

fct_infreq(gss_cat$rincome) %>%
  levels()

fct_infreq(gss_cat$rincome) %>%
  fct_rev() %>%
  levels()

gss_cat %>%
  mutate(rincome = rincome %>% fct_infreq() %>% fct_rev()) %>%
  ggplot(aes(rincome)) +
  geom_bar() +
  coord_flip()

Modify Levels

levels(gss_cat$partyid)

More powerful than changing the orders of the levels is changing their values. This allows you to clarify labels for publication, and collapse levels for high-level displays. The most general and powerful tool is fct_recode(). It allows you to recode, or change, the value of each level.

fct_recode() will leave levels that aren’t explicitly mentioned as is, and will warn you if you accidentally refer to a level that doesn’t exist. To combine groups, you can assign multiple old levels to the same new level.

fct_recode(
  gss_cat$partyid,
  "Republican, strong" = "Strong republican",
  "Republican, weak" = "Not str republican",
  "Independent, near rep" = "Ind,near rep",
  "Independent, near dem" = "Ind,near dem",
  "Democrat, weak" = "Not str democrat",
  "Democrat, strong" = "Strong democrat"
) %>%
  fct_count()

fct_recode(
  gss_cat$partyid,
  "Republican, strong" = "Strong republican",
  "Republican, weak" = "Not str republican",
  "Independent, near rep" = "Ind,near rep",
  "Independent, near dem" = "Ind,near dem",
  "Democrat, weak" = "Not str democrat",
  "Democrat, strong" = "Strong democrat",
  "Other" = "No answer",
  "Other" = "Don't know",
  "Other" = "Other party"
) %>%
  fct_count()

If you want to collapse a lot of levels, fct_collapse() is a useful variant of fct_recode(). For each new variable, you can provide a vector of old levels

fct_collapse(
  gss_cat$partyid,
  other = c("No answer", "Don't know", "Other party"),
  rep = c("Strong republican", "Not str republican"),
  ind = c("Ind,near rep", "Independent", "Ind,near dem"),
  dem = c("Not str democrat", "Strong democrat")
) %>%
  fct_count()

Sometimes you just want to lump together all the small groups to make a plot or table simpler. That’s the job of fct_lump().
The default behavior is to progressively lump together the smallest groups, ensuring that the aggregate is still the smallest group.
We can use the n parameter to specify how many groups (excluding other) we want to keep.

fct_lump(gss_cat$partyid) %>%
  fct_count()

fct_lump(gss_cat$partyid, n = 5) %>%
  fct_count()

fct_lump(gss_cat$partyid, n = 3, other_level = "Extra") %>%
  fct_count()

Parse

# Date
d1 <- ymd("2022-01-01")
d2 <- ymd("2022-01-02")
class(d1)

# Datetime
t1 <- ymd_hms("2022-01-01 10:30:15")
t2 <- ymd_hms("2022-01-02 22:30:15")
class(t1)

Differences

# Time Differences
d2 - d1
difftime(d2, d1, units = "auto")
difftime(d2, d1, units = "hours")
difftime(d2, d1, units = "weeks")
d1 - d2

Extract Time Periods

second(t1)
minute(t1)
hour(t1)
day(t1)
week(t1)
month(t1)
month(t1, label = TRUE, abbr = TRUE)
month(t1, label = TRUE, abbr = FALSE)
quarter(t1)
semester(t1)
year(t1)

it_locale <- "it_IT.UTF-8"
en_locale <- "en_US.UTF-8"
Sys.getlocale()
Sys.setlocale("LC_ALL", locale = en_locale)
month(d1, label = TRUE, abbr = TRUE)
month(d1, label = TRUE, abbr = FALSE)
Sys.setlocale("LC_ALL", locale = it_locale)

Round

t2
round_date(t2, unit = "hour")
round_date(t2, unit = "month")

floor_date(t2, unit = "hour")
ceiling_date(t2, unit = "hour")

Column Headers

A common type of messy dataset is tabular data designed for presentation, where variables form both the rows and columns, and column headers are values, not variable names. While I would call this arrangement messy, in some cases it can be extremely useful. It provides efficient storage for completely crossed designs, and it can lead to extremely efficient computation if desired operations can be expressed as matrix operations.

relig_income
relig_income %>% 
  pivot_longer(-religion, names_to = "income", values_to = "frequency")

billboard
names(billboard)
billboard %>% 
  pivot_longer(
    wk1:wk76, 
    names_to = "week", 
    values_to = "rank", 
    values_drop_na = TRUE
  )

Multiple Variables

After pivoting columns, the key column is sometimes a combination of multiple underlying variable names. Column headers in this format are often separated by a non-alphanumeric character (e.g. ., -, _, :), or have a fixed width format, like in this dataset. separate() makes it easy to split a compound variables into individual variables. You can either pass it a regular expression to split on (the default is to split on non-alphanumeric columns), or a vector of character positions.

# re-create the tuberculosis dataset
# gender & age into column names
tb <- tibble(
  iso2 = c(rep("IT", 5), rep("EN", 5)),
  year = c(1991:1995, 1991:1995),
  m014 = sample.int(20, 10),
  m1525 = c(NA, sample.int(20, 9)),
  m2534 = sample.int(20, 10),
  f014 = sample.int(20, 10),
  f1525 = sample.int(20, 10),
  f2534 = sample.int(20, 10)
)
tb
tb %>% 
  pivot_longer(
    !c(iso2, year), 
    names_to = "name", 
    values_to = "n"
  ) %>% 
  separate(col = name, into = c("sex", "age"), sep = 1) %>% 
  fill(n, .direction = "down")

tb %>% 
  pivot_longer(
    !c(iso2, year), 
    names_to = c("sex", "age"), 
    names_pattern = "(.)(.+)",
    values_to = "n"
  ) %>% 
  drop_na()

Variables in Rows and Columns

The most complicated form of messy data occurs when variables are stored in both rows and columns.

Re-create the weather dataset.
It has variables in individual columns (id, year, month), spread across columns (day, d1-d31) and across rows (tmin, tmax) (minimum and maximum temperature).
This dataset is mostly tidy, but the element column is not a variable; it stores the names of variables.
Fixing this requires widening the data: pivot_wider() is inverse of pivot_longer(), pivoting element and value back out across multiple columns.

weather <- tibble(
  id = c(rep("X1", 16), rep("X2", 16)),
  year = rep(c(rep(2000, 4), rep(2005, 4), rep(2000, 4), rep(2005, 4)), 2),
  month = rep(c(1, 4, 7, 10), 8),
  element = rep(c(rep("tmin", 8), rep("tmax", 8)), 2),
  d1 = runif(32, min = 0, max = 30),
  d10 = runif(32, min = 0, max = 30),
  d20 = runif(32, min = 0, max = 30),
  d30 = runif(32, min = 0, max = 30)
) %>% 
  arrange(id, year, month)
weather

weather %>% 
  pivot_longer(
    d1:d30, 
    names_to = "day", 
    values_to = "value"
  ) %>% 
  pivot_wider(
    names_from = element,
    values_from = value
  )

Multiple Types

Datasets often involve values collected at multiple levels, on different types of observational units. During tidying, each type of observational unit should be stored in its own table. This is closely related to the idea of database normalization, where each fact is expressed in only one place. It’s important because otherwise inconsistencies can arise.

The billboard dataset actually contains observations on two types of observational units: the song and its rank in each week. This manifests itself through the duplication of facts about the song: artist is repeated many times. This dataset needs to be broken down into two pieces: a song dataset which stores artist and song name, and a ranking dataset which gives the rank of the song in each week.

Normalization is useful for tidying and eliminating inconsistencies. However, there are few data analysis tools that work directly with relational data, so analysis usually also requires denormalization or the merging the datasets back into one table.

billboard_long <- billboard %>% 
  pivot_longer(
    wk1:wk76, 
    names_to = "week", 
    values_to = "rank", 
    values_drop_na = TRUE
  ) %>% 
  mutate(
    week = as.integer(stringr::str_remove(week, "wk")),
    date = as.Date(date.entered) + 7 * (week - 1),
    date.entered = NULL
  )
billboard_long

song <- billboard_long %>% 
  distinct(artist, track) %>%
  mutate(song_id = row_number(), .before = everything())
song

rank <- billboard_long %>%
  left_join(song, by = c("artist", "track")) %>%
  select(song_id, date, week, rank)
rank

One Type in Multiple Tables

It’s also common to find data values about a single type of observational unit spread out over multiple tables or files. These tables and files are often split up by another variable, so that each represents a single year, person, or location. As long as the format for individual records is consistent, this is an easy problem to fix:

• Read the files into a list of tables
  
• For each table, add a new column that records the original file name (the file name is often the value of an important variable)
  
• Combine all tables into a single table.

Rows

# filter
starwars %>% filter(skin_color == "light", eye_color == "brown")

starwars %>% filter(skin_color == "light" & eye_color == "brown")

starwars %>% filter(skin_color == "light" | eye_color == "brown")

# slice
starwars %>% slice(5:10)

starwars %>% slice(-1)

starwars %>% slice_head(n = 3)

starwars %>% slice_tail(n = 3)

starwars %>% slice_sample(n = 5)

starwars %>% slice_sample(prop = .2, replace = TRUE)

starwars %>% slice_max(height, n = 3)

starwars %>% slice_min(height, n = 3)

# arrange
starwars %>% arrange(height, mass)

starwars %>% arrange(desc(height))

starwars %>% arrange(name)

Columns

# select
starwars %>% select(height, mass, hair_color, skin_color, eye_color, birth_year)

starwars %>% select(height:birth_year)

starwars %>% select(!(height:birth_year))

starwars %>% select(ends_with("color"))

starwars %>% select(starts_with("h"))

starwars %>% select(matches("^h"))

starwars %>% select(contains("_"))

vars <- c("name", "height")
starwars %>% select(all_of(vars), "mass")

starwars %>% select(height:birth_year, everything()) relocate with select

# relocate
starwars %>% relocate(sex:homeworld, .before = height)

starwars %>% relocate(sex:homeworld, .after = mass)

# rename
starwars %>% rename(home_world = homeworld, person = name)

# mutate
starwars %>% mutate(height_m = height / 100)

starwars %>%
  mutate(height_m = height / 100) %>%
  select(name, height_m, height, everything())

starwars %>% mutate(height_m = height / 100, .before = height) 

starwars %>%
  mutate(
    height_m = height / 100,
    BMI = mass / (height_m ^ 2) allows direct reference & multiple operations
  ) %>%
  select(name, BMI, everything())

starwars %>%
  transmute( mutate that keeps only modified columns
    height_m = height / 100,
    BMI = mass / (height_m^2)
  )

starwars %>% mutate(across(where(is.numeric), ~ .x / 100))

Groups

# summarise
starwars %>% summarise(mean(height, na.rm = TRUE))

starwars %>%
  summarise(
    mean_height = mean(height, na.rm = TRUE),
    mean_mass = mean(mass, na.rm = TRUE),
    sd_height = sd(height, na.rm = TRUE),
    sd_mass = sd(mass, na.rm = TRUE)
  )

starwars %>% 
  summarise(across(c(height, mass), c(mean, sd), na.rm = TRUE))

starwars %>% 
  summarise(across(c(height, mass), list(mu = mean, sigma = sd), na.rm = TRUE))

starwars %>% 
  summarise(across(c(height, mass), list(mu = mean, sigma = sd), na.rm = TRUE)) %>% 
  pivot_longer(cols = everything()) %>% 
  separate(name, into = c("type", "measure"), sep = "_") %>% 
  pivot_wider(id_cols = type, names_from = measure, values_from = value)

starwars %>% 
  summarise(across(where(is.character), ~ length(unique(.x))))

starwars %>% 
  summarise(across(where(is.numeric), ~ mean(.x, na.rm = TRUE)))

min_max <- list(
  min = ~ min(.x, na.rm = TRUE), 
  max = ~ max(.x, na.rm = TRUE)
)
starwars %>% summarise(across(where(is.numeric), min_max))

# group_by
starwars %>% group_by(species)
starwars %>% group_by(species) %>% tally(sort = TRUE)

starwars %>% group_by(sex, gender)
starwars %>% group_by(sex, gender) %>% tally(sort = TRUE)

starwars %>% group_by(species) %>% ungroup() %>% tally(sort = TRUE)

# group_by + summarise
starwars %>% 
  group_by(species) %>%
  summarise(n = n(), height = mean(height, na.rm = TRUE))

starwars %>% 
  group_by(sex, gender) %>%
  summarise(n = n(), height = mean(height, na.rm = TRUE))

group_by + slice
starwars %>% 
  relocate(species) %>% 
  group_by(species) %>%
  slice(1:2)

starwars %>% 
  relocate(sex, gender) %>% 
  group_by(sex, gender) %>%
  slice(1:2)

group_by + mutate
starwars %>% 
  select(name, homeworld, height, mass) %>% 
  group_by(homeworld) %>% 
  mutate(
    mean_mass = mean(mass, na.rm = TRUE),
    standard_mass = mass - mean(mass, na.rm = TRUE),
    rank = min_rank(height)
  ) %>% 
  ungroup()

# count
starwars %>%
  count(species)

starwars %>%
  group_by(species) %>% 
  summarise(n = n())

Joins

It’s rare that a data analysis involves only a single table of data. In practice, you’ll normally have many tables that contribute to an analysis, and you need flexible tools to combine them. In dplyr, there are three families of verbs that work with two tables at a time:

• Mutating joins, which add new variables to one table from matching rows in another
  
• Filtering joins, which filter observations from one table based on whether or not they match an observation in the other table
  
• Set operations, which combine the observations in the data sets as if they were set elements.

All two-table verbs work similarly. The first two arguments are x and y, and provide the tables to combine. The output is always a new table with the same type as x.

library(nycflights13)
flights
weather
planes
airports

flights_small <- flights %>% 
  select(year:day, hour, origin, dest, tailnum, carrier)
flights_small

flights_small %>% left_join(airlines)

flights_small %>% left_join(weather)

flights_small %>% left_join(planes, by = "tailnum")

flights_small %>% left_join(airports, c("dest" = "faa"))

Utilities

distinct
starwars
starwars %>% distinct()
starwars %>% distinct(sex)
starwars %>% distinct(sex, .keep_all = TRUE)

pull
starwars[["name"]]
starwars %>% pull(name)

bind_rows / bind_columns
luke <- starwars %>% 
  select(name, homeworld, species) %>% 
  filter(name == "Luke Skywalker")
luke
obi <- starwars %>% 
  select(name, homeworld, species) %>% 
  filter(name == "Obi-Wan Kenobi")
obi

bind_rows(luke, obi)
bind_cols(luke, obi)

rowwise

dplyr, and R in general, are particularly well suited to performing operations over columns, and performing operations over rows is much harder. In this vignette, you’ll learn dplyr’s approach centered around the row-wise data frame created by rowwise().

There are three common use cases that we discuss in this vignette:

• Row-wise aggregates (e.g. compute the mean of x, y, z).
  
• Calling a function multiple times with varying arguments.
  
• Working with list-columns.

These types of problems are often easily solved with a for loop, but it’s nice to have a solution that fits naturally into a pipeline.

starwars %>% 
  select(name, height, mass) %>% 
  mutate(average = mean(c(height, mass), na.rm = TRUE))

starwars %>% 
  select(name, height, mass) %>% 
  rowwise() %>% 
  mutate(average = mean(c(height, mass), na.rm = TRUE))

starwars %>% 
  select(name, height, mass) %>% 
  summarise(average = mean(c(height, mass), na.rm = TRUE))

starwars %>% 
  select(name, height, mass) %>% 
  rowwise() %>% 
  summarise(average = mean(c(height, mass), na.rm = TRUE))

starwars %>% 
  select(name, height, mass) %>% 
  rowwise(name) %>% 
  summarise(average = mean(c(height, mass), na.rm = TRUE))

Initialization

A ggplot2 chart is created with two main functions:

1. ggplot() initializes a ggplot object. It can be used to declare the input data frame for a graphic and to specify the set of plot aesthetics intended to be common throughout all subsequent layers unless specifically overridden
   
2. geom_…() represents the layers of the plot, that is the type of chart one wants to draw (that is points, lines, bars, etc.)

Moreover, in order to draw the plot, it is necessary to provide the data and to map the desired aesthetics.

# initializes the pane to the plot
ggplot()

# adds a desired layer
ggplot() +
  geom_point()

# provides the dataframe
ggplot(data = mpg)

# maps the variables
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy))

With ggplot2, you begin a plot with the function ggplot(). ggplot() creates a coordinate system that you can add layers to. The first argument of ggplot() is the dataset to use in the graph.

You complete your graph by adding one or more layers to ggplot(). The function geom_point() adds a layer of points to your plot, which creates a scatterplot. ggplot2 comes with many geom functions that each add a different type of layer to a plot. You’ll learn a whole bunch of them throughout this chapter.

Each geom function in ggplot2 takes a mapping argument. This defines how variables in your dataset are mapped to visual properties. The mapping argument is always paired with aes(), and the x and y arguments of aes() specify which variables to map to the x and y axes. ggplot2 looks for the mapped variables in the data argument, in this case, mpg.

# Graphing Template
ggplot(data = <DATA>) +
  <GEOM_FUNCTION>(mapping = aes(<MAPPINGS>))

Aesthetics Mappings

You can add a third variable, like class, to a two dimensional scatter plot by mapping it to an aesthetic. An aesthetic is a visual property of the objects in your plot. Aesthetics include things like the size, the shape, or the color of your points. You can display a point in different ways by changing the values of its aesthetic properties. Since we already use the word “value” to describe data, let’s use the word “level” to describe aesthetic properties. Here we change the levels of a point’s size, shape, and color to make the point small, triangular, or blue.

mpg$class %>% table()

# coloring points by class using color aesthetic
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy, col = class))

To map an aesthetic to a variable, associate the name of the aesthetic to the name of the variable inside aes(). ggplot2 will automatically assign a unique level of the aesthetic (here a unique color) to each unique value of the variable, a process known as scaling. ggplot2 will also add a legend that explains which levels correspond to which values.

# changing the size of points by class using size aesthetic
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy, size = class))

# changing the transparency of points by class using alpha aesthetic
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy, alpha = class))

# changing the shape of points by class using shape aesthetic
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy, shape = class))

For each aesthetic, you use aes() to associate the name of the aesthetic with a variable to display. The aes() function gathers together each of the aesthetic mappings used by a layer and passes them to the layer’s mapping argument. The syntax highlights a useful insight about x and y: the x and y locations of a point are themselves aesthetics, visual properties that you can map to variables to display information about the data.

Once you map an aesthetic, ggplot2 takes care of the rest. It selects a reasonable scale to use with the aesthetic, and it constructs a legend that explains the mapping between levels and values. For x and y aesthetics, ggplot2 does not create a legend, but it creates an axis line with tick marks and a label. The axis line acts as a legend; it explains the mapping between locations and values.

You can also set the aesthetic properties of your geom manually. Here, the color doesn’t convey information about a variable, but only changes the appearance of the plot. To set an aesthetic manually, set the aesthetic by name as an argument of your geom function; i.e. it goes outside of aes(). You’ll need to pick a level that makes sense for that aesthetic: the name of a color as a character string; the size of a point in mm; the shape of a point as a number.

# manual aesthetic
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy), col = "blue")

Geometries

Each plot uses a different visual object to represent the data. In ggplot2 syntax, we say that they use different geoms. A geom is the geometrical object that a plot uses to represent data. People often describe plots by the type of geom that the plot uses. For example, bar charts use bar geoms, line charts use line geoms, boxplots use boxplot geoms, and so on. Scatterplots break the trend; they use the point geom. As we see above, you can use different geoms to plot the same data. To change the geom in your plot, change the geom function that you add to ggplot().

# points
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy))

# smooth line
ggplot(data = mpg) +
  geom_smooth(mapping = aes(x = displ, y = hwy))

Every geom function in ggplot2 takes a mapping argument. However, not every aesthetic works with every geom. You could set the shape of a point, but you couldn’t set the “shape” of a line. On the other hand, you could set the linetype of a line. geom_smooth() will draw a different line, with a different linetype, for each unique value of the variable that you map to linetype.

# does not work properly
ggplot(data = mpg) +
  geom_smooth(mapping = aes(x = displ, y = hwy, shape = drv))

# changing line type by drv
ggplot(data = mpg) +
  geom_smooth(mapping = aes(x = displ, y = hwy, linetype = drv))

It is also possible to use more than one aesthetic into a single geom.

# changing line type and color by drv
ggplot(data = mpg) +
  geom_smooth(mapping = aes(x = displ, y = hwy, linetype = drv, col = drv))

Many geoms, like geom_smooth(), use a single geometric object to display multiple rows of data. For these geoms, you can set the group aesthetic to a categorical variable to draw multiple objects. ggplot2 will draw a separate object for each unique value of the grouping variable. In practice, ggplot2 will automatically group the data for these geoms whenever you map an aesthetic to a discrete variable (as in the linetype example). It is convenient to rely on this feature because the group aesthetic by itself does not add a legend or distinguishing features to the geoms.

ggplot(data = mpg) +
  geom_smooth(mapping = aes(x = displ, y = hwy))

grouping by drv
ggplot(data = mpg) +
  geom_smooth(mapping = aes(x = displ, y = hwy, group = drv))

ggplot(data = mpg) +
  geom_smooth(
    mapping = aes(x = displ, y = hwy, color = drv),
    show.legend = FALSE
  )

One may want to display multiple geoms in the same plot. To do this simply add multiple geom functions to ggplot().

# point + smooth
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy)) +
  geom_smooth(mapping = aes(x = displ, y = hwy))

This, however, introduces some duplication in our code. Imagine if you wanted to change the y-axis to display cty instead of hwy. You’d need to change the variable in two places, and you might forget to update one. You can avoid this type of repetition by passing a set of mappings to ggplot(). ggplot2 will treat these mappings as global mappings that apply to each geom in the graph.

# global mapping, point + smooth
ggplot(data = mpg, mapping = aes(x = displ, y = hwy)) +
  geom_point() +
  geom_smooth()

If you place mappings in a geom function, ggplot2 will treat them as local mappings for the layer. It will use these mappings to extend or overwrite the global mappings for that layer only. This makes it possible to display different aesthetics in different layers.

# global mapping x & y, local mapping col, point + smooth
ggplot(data = mpg, mapping = aes(x = displ, y = hwy)) +
  geom_point(mapping = aes(color = class)) +
  geom_smooth()

You can use the same idea to specify different data for each layer. Here, our smooth line displays just a subset of the mpg dataset, the subcompact cars. The local data argument in geom_smooth() overrides the global data argument in ggplot() for that layer only.

# global data into point + local data into smooth
ggplot(data = mpg, mapping = aes(x = displ, y = hwy)) +
  geom_point(mapping = aes(color = class)) +
  geom_smooth(data = filter(mpg, class == "subcompact"), se = FALSE)

Statistical Transformations

Consider a basic bar chart, as drawn with geom_bar().

# geom_bar with default stat_count
ggplot(data = diamonds) +
  geom_bar(mapping = aes(x = cut))

On the x-axis, the chart displays cut, a variable from diamonds. On the y-axis, it displays count, but count is not a variable in diamonds! Where does count come from? Many graphs, like scatterplots, plot the raw values of your dataset. Other graphs, like bar charts, calculate new values to plot:

• bar charts, histograms, and frequency polygons bin your data and then plot bin counts, the number of points that fall in each bin
  
• smoothers fit a model to your data and then plot predictions from the model
  
• boxplots compute a robust summary of the distribution and then display a specially formatted box

The algorithm used to calculate new values for a graph is called a stat, short for statistical transformation. The figure below describes how this process works with geom_bar().

You can learn which stat a geom uses by inspecting the default value for the stat argument. For example, ?geom_bar shows that the default value for stat is “count”, which means that geom_bar() uses stat_count().
You can generally use geoms and stats interchangeably. For example, you can recreate the previous plot using stat_count() instead of geom_bar(). This works because every geom has a default stat; and every stat has a default geom. This means that you can typically use geoms without worrying about the underlying statistical transformation.

# stat_count with default geom_bar
ggplot(data = diamonds) +
  stat_count(mapping = aes(x = cut))

There are three reasons you might need to use a stat explicitly:

1. You might want to override the default stat. In the code below, I change the stat of geom_bar() from count (the default) to identity. This lets me map the height of the bars to the raw values of a y variable
   
2. You might want to override the default mapping from transformed variables to aesthetics. For example, you might want to display a bar chart of proportion, rather than count
   
3. You might want to draw greater attention to the statistical transformation in your code. For example, you might use stat_summary(), which summarises the y values for each unique x value, to draw attention to the summary that you’re computing

# changing count to proportions
ggplot(data = diamonds) +
    geom_bar(mapping = aes(x = cut, y = stat(prop), group = 1))

# computing specific stats
ggplot(data = diamonds) +
  stat_summary(
    mapping = aes(x = cut, y = depth),
    fun.min = min,
    fun.max = max,
    fun = median
  )

Position Adjustments

There’s one more piece of magic associated with bar charts. You can colour a bar chart using either the colour aesthetic, or, more usefully, fill.

# color by cut
ggplot(data = diamonds) +
  geom_bar(mapping = aes(x = cut, colour = cut))

# fill by cut
ggplot(data = diamonds) +
  geom_bar(mapping = aes(x = cut, fill = cut))

Note what happens if you map the fill aesthetic to another variable, like clarity: the bars are automatically stacked. Each colored rectangle represents a combination of cut and clarity.

# automatic stacked bars with fill by another variable (clarity)
ggplot(data = diamonds) +
  geom_bar(mapping = aes(x = cut, fill = clarity))

The stacking is performed automatically by the position adjustment specified by the position argument. If you don’t want a stacked bar chart, you can use one of three other options: “identity”, “dodge” or “fill”:

• position = “identity” will place each object exactly where it falls in the context of the graph. This is not very useful for bars, because it overlaps them. To see that overlapping we either need to make the bars slightly transparent by setting alpha to a small value, or completely transparent by setting fill = NA
  
• position = “fill” works like stacking, but makes each set of stacked bars the same height. This makes it easier to compare proportions across groups
  
• position = “dodge” places overlapping objects directly beside one another. This makes it easier to compare individual values

# position identity makes overlapping bars
ggplot(data = diamonds, mapping = aes(x = cut, fill = clarity)) +
  geom_bar(alpha = 1/5, position = "identity")
ggplot(data = diamonds, mapping = aes(x = cut, colour = clarity)) +
  geom_bar(fill = NA, position = "identity")

# position fill creates proportional stacked bars
ggplot(data = diamonds) +
  geom_bar(mapping = aes(x = cut, fill = clarity), position = "fill")

# position dodge creates bars next to each other
ggplot(data = diamonds) +
  geom_bar(mapping = aes(x = cut, fill = clarity), position = "dodge")

There’s one other type of adjustment that’s not useful for bar charts, but it can be very useful for scatter plots. The values in scatter plots are usually rounded so the points appear on a grid and many points overlap each other. This problem is known as overplotting. This arrangement makes it hard to see where the mass of the data is. You can avoid this gridding by setting the position adjustment to “jitter”. position = “jitter” adds a small amount of random noise to each point. This spreads the points out because no two points are likely to receive the same amount of random noise. Adding randomness seems like a strange way to improve your plot, but while it makes your graph less accurate at small scales, it makes your graph more revealing at large scales. Because this is such a useful operation, ggplot2 comes with a shorthand for geom_point(position = “jitter”): geom_jitter().

# scatterplot with overplotted points
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy))

# scatterplot with noise using position jitter or geom_jitter
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy), position = "jitter")
ggplot(data = mpg) +
  geom_jitter(mapping = aes(x = displ, y = hwy))

Coordinate Systems

Coordinate systems are probably the most complicated part of ggplot2. The default coordinate system is the Cartesian coordinate system where the x and y positions act independently to determine the location of each point. There are a number of other coordinate systems that are occasionally helpful:

• coord_flip() switches the x and y axes. This is useful (for example), if you want horizontal boxplots. It’s also useful for long labels: it’s hard to get them to fit without overlapping on the x-axis
  
• coord_quickmap() sets the aspect ratio correctly for maps. This is very important if you’re plotting spatial data with ggplot2
  
• coord_polar() uses polar coordinates. Polar coordinates reveal an interesting connection between a bar chart and a Coxcomb chart

# boxplot
ggplot(data = mpg, mapping = aes(x = class, y = hwy)) +
  geom_boxplot()

# boxplot with flipped x and y axes
ggplot(data = mpg, mapping = aes(x = class, y = hwy)) +
  geom_boxplot() +
  coord_flip()

bar <- ggplot(data = diamonds) +
  geom_bar(
    mapping = aes(x = cut, fill = cut),
    show.legend = FALSE,
    width = 1
  ) +
  theme(aspect.ratio = 1) +
  labs(x = NULL, y = NULL)

# bar with flipped x and y axes
bar + coord_flip()

# bar with polar axes
bar + coord_polar()

Facets

One way to add additional variables is with aesthetics. Another way, particularly useful for categorical variables, is to split your plot into facets, subplots that each display one subset of the data. To facet your plot by a single variable, use facet_wrap(). The first argument of facet_wrap() should be a formula, which you create with ~ followed by a variable name (here “formula” is the name of a data R structure in R, not a synonym for “equation”). The variable that you pass to facet_wrap() should be discrete.

# scatterplot faceted by class
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy)) +
  facet_wrap(~ class, nrow = 2)

To facet your plot on the combination of two variables, add facet_grid() to your plot call. The first argument of facet_grid() is also a formula. This time the formula should contain two variable names separated by a ~.

# scatterplot faceted by drv (to rows) and cyl (to columns)
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy)) +
  facet_grid(drv ~ cyl)

If you prefer to not facet in the rows or columns dimension, use a . instead of a variable name, e.g. + facet_grid(. ~ cyl).

# scatterplot faceted by cyl (to rows)
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy)) +
  facet_grid(cyl ~ .)

# scatterplot faceted by cyl (to columns)
ggplot(data = mpg) +
  geom_point(mapping = aes(x = displ, y = hwy)) +
  facet_grid(. ~ cyl)

Other Useful Graphical Designs

# Scale
ggplot(data = mpg, mapping = aes(x = class, y = hwy)) +
  geom_jitter() +
  geom_violin(mapping = aes(fill = drv), alpha = .5) +
  geom_hline(
    yintercept = mpg %>% group_by(drv) %>% summarise(means = mean(hwy)) %>% pull(means),
    linetype = 2,
    col = c("red", "green", "blue")
  ) +
  scale_fill_manual(values = c("red", "green", "blue"))

# Labs
ggplot(data = mpg, mapping = aes(x = class, y = hwy)) +
  geom_jitter() +
  geom_violin(mapping = aes(fill = drv), alpha = .5) +
  geom_hline(
    yintercept = mpg %>% group_by(drv) %>% summarise(means = mean(hwy)) %>% pull(means),
    linetype = 2,
    col = c("red", "green", "blue")
  ) +
  scale_fill_manual(values = c("red", "green", "blue")) +
  labs(
    title = "Boxplots",
    subtitle = "colored by drv",
    caption = "Fig. 1",
    x = "Classes",
    y = "HMY",
    fill = "DRV Types" legend title
  )

# Themes
ggplot(data = mpg, mapping = aes(x = class, y = hwy)) +
  geom_jitter() +
  geom_violin(mapping = aes(fill = drv), alpha = .5) +
  geom_hline(
    yintercept = mpg %>% group_by(drv) %>% summarise(means = mean(hwy)) %>% pull(means),
    linetype = 2,
    col = c("red", "green", "blue")
  ) +
  scale_fill_manual(values = c("red", "green", "blue")) +
  labs(
    title = "Boxplots",
    subtitle = "colored by drv",
    caption = "Fig. 1",
    x = "Classes",
    y = "HMY",
    fill = "DRV Types" legend title
  ) +
  theme_dark() +
  theme_bw() +
  theme_minimal() +
  theme(legend.position = "none") +
  NULL

The Grammar of Graphics Template

ggplot(data = <DATA>) +
  <GEOM_FUNCTION>(
    mapping = aes(<MAPPINGS>),
    stat = <STAT>,
    position = <POSITION>
  ) +
  <COORDINATE_FUNCTION> +
  <FACET_FUNCTION>

Our new template takes seven parameters, the bracketed words that appear in the template. In practice, you rarely need to supply all seven parameters to make a graph because ggplot2 will provide useful defaults for everything except the data, the mappings, and the geom function.

The seven parameters in the template compose the grammar of graphics, a formal system for building plots. The grammar of graphics is based on the insight that you can uniquely describe any plot as a combination of a dataset, a geom, a set of mappings, a stat, a position adjustment, a coordinate system, and a faceting scheme.

Ggiraph

library(ggplot2)
library(rvg)
library(ggiraph)

data <- mtcars
data$carname <- row.names(data)

gg_point <- data %>%
  ggplot() +
  geom_point_interactive(
    aes(x = wt, y = qsec, color = disp, tooltip = carname, data_id = carname)
  ) +
  theme_minimal()

girafe(ggobj = gg_point)

Gganimate

library(ggplot2)
library(gganimate)

For example, suppose we wanted to create an animation similar to the Gapminder world animation, using Jenny Bryan’s gapminder package for the data.

library(gapminder)
gapminder

ggplot(gapminder, aes(gdpPercap, lifeExp, size = pop, colour = country)) +
  geom_point(alpha = 0.7, show.legend = FALSE) +
  scale_colour_manual(values = country_colors) +
  scale_size(range = c(2, 12)) +
  scale_x_log10() +
  facet_wrap(~continent) +
  Here comes the gganimate specific bits
  labs(title = 'Year: {frame_time}', x = 'GDP per capita', y = 'life expectancy') +
  transition_time(year) +
  ease_aes('linear')

Ggrepel

library(ggplot2)
library(ggrepel)

data <- mtcars[1:8, ]

data %>%
  ggplot() +
  geom_point(aes(wt, mpg), color = 'red') +
  geom_text(aes(wt, mpg, label = rownames(data))) +
  theme_classic(base_size = 16)

set.seed(42)
data %>%
  ggplot() +
  geom_point(aes(wt, mpg), color = 'red') +
  geom_text_repel(aes(wt, mpg, label = rownames(data))) +
  theme_classic(base_size = 16)

Patchwork

Patchwork allows to combine different ggplot2 objects.

library(patchwork)
mpg
p1 <- ggplot(data = mpg, mapping = aes(x = class, y = hwy)) +
  geom_boxplot() +
  coord_flip()
p2 <- ggplot(data = mpg, mapping = aes(x = displ, y = hwy)) +
  geom_point(mapping = aes(color = class)) +
  geom_smooth()

p1 | p2 # patchwork sintax


wrap_plots(p1, p2, guides = "collect") &
  guides(colour = guide_legend(nrow = 1)) &
  theme(legend.position = "top")

p3 <- ggplot(data = mpg, mapping = aes(x = class, y = hwy)) +
  geom_jitter() +
  geom_violin(mapping = aes(fill = drv), alpha = .5) +
  geom_hline(
    yintercept = mpg %>% group_by(drv) %>% summarise(means = mean(hwy)) %>% pull(means),
    linetype = 2,
    col = c("red", "green", "blue")
  ) +
  scale_fill_manual(values = c("red", "green", "blue")) +
  labs(
    title = "Boxplots",
    subtitle = "colored by drv",
    caption = "Fig. 1",
    x = "Classes",
    y = "HMY",
    fill = "DRV Types"
  )

(p1 | p2) / p3 # patchwork sintax

Ggside

library(ggside)

mpg %>%
  ggplot(aes(hwy, cty, color = class)) +
  geom_point(size = 2, alpha = .3) +
  geom_smooth(aes(color = NULL), se = TRUE) +
  geom_xsidedensity(
    aes(y = after_stat(density), fill = class),
    alpha = .5, size = 1, position = "stack"
  ) +
  geom_ysidedensity(
    aes(x = after_stat(density), fill = class),
    alpha = .5, size = 1, position = "stack"
  ) +
  theme_minimal()

mpg %>%
  ggplot(aes(cty, hwy, color = class)) +
  geom_point(size = 2, alpha = .3) +
  geom_smooth(aes(color = NULL), se = TRUE) +
  geom_xsideboxplot(alpha = .5, size = 1) +
  facet_grid(cols = vars(cyl), scales = "free_x") +
  theme_minimal()

Ggdist

library(ggdist)

mpg %>%
  filter(cyl %in% c(4, 6, 8)) %>%
  ggplot(aes(factor(cyl), hwy, fill = factor(cyl))) +
  stat_halfeye(adjust = .5, justification = -.2, .width = 0, point_colour = NA) +
  geom_boxplot(width = .12, outlier.color = NA, alpha = .5) +
  stat_dots(side = "left", justification = 1.1, binwidth = .25) +
  coord_flip() +
  theme_minimal()

Grafify

library(grafify)

# scatter bars
mpg %>% plot_scatterbar_sd(cyl, hwy)

# scatter boxplots
mpg %>% plot_scatterbox(cyl, hwy, jitter = 0.2, alpha = 0.5)

# scatter violin
mpg %>% plot_dotviolin(cyl, hwy, dotsize = 1, ColPal = "bright")

# before & after
mpg %>%
  group_by(model, year) %>%
  summarise(mean_hwy = mean(hwy)) %>%
  ungroup() %>%
  plot_befafter_colors(year, mean_hwy, model)