The human voice aligns with whole-body kinetics

Supporting information

Show code setting up the packages

#R-packages
library(papaja)   #for Rmarkdown template for APA(ish) manuscript
library(distill) #for html output
library(ggplot2)    #for plotting
library(gridExtra)  #for plotting in panels
library(knitr)  #for document generation
library(magick)  #for plots
library(cowplot) #for plots
library(plyr) #for revalue
library(tinytex) #for outputting pdfs
library(readr)  #for data reading
library(bookdown) #for crossreferencing
library(rstudioapi) #for setting file paths
library(kableExtra) #for producing nicer tables
library(dplyr) #for data wrangling
library(ggbeeswarm) #for beeswarm plots
library(modelsummary) #for handling model summary tables
library(lme4) #for statistical modeling
library(lmerTest) #for statistical modeling
library(emmeans) #for post-hoc analysis
library(broom.mixed) #for turning fitted models into tidy data frames
library(tidyr) # data wrangling functions

r_refs("references.bib")

Supporting methods and results

This is a fully computationally reproducible manuscript written in R Markdown. The full dataset is available at the Donders Repository (redacted for review).

This document contains further information on the supporting methods and results of “The human voice aligns with whole-body kinetics”. For clarity, it is indicated on the right, what sections in the paper the respective part refers to. Code chunks are hidden by default, but can be made visible by clicking “Show code”.This supporting information is a computationally reproducible Rmarkdown document accessible on the github https://github.com/WimPouw/kineticsvoice and presented in html at https://wimpouw.github.io/kineticsvoice/. The full dataset needed for input to recreate this Rmarkdown reports is downloadable from here: https://doi.org/10.34973/p9se-mq71. Please note that all the high-resolution figures can be found here: https://github.com/WimPouw/kineticsvoice/tree/main/Images.

This refers to the section “Supporting information” of the paper.

Show code loading and preparing the data

#Local data: This points to your folder with the contents of the Donders Repository https://doi.org/10.34973/p9se-mq71
localfolder <- "F:/VENI_data_local/exp100masked/" 

#lets set all the data folders
rawd      <- paste0(localfolder, 'Trials/')    #raw trial level data
procd     <- paste0(localfolder, 'Processed/triallevel/') #processed data folder
procdtot  <- paste0(localfolder, 'Processed/complete_datasets/') #processed data folder
daset     <- paste0(localfolder, 'Dataset/')   #dataset folder
meta      <- paste0(localfolder, 'Meta/')   #Meta and trial data

#load in the trialinfo data (containing info like condition order, trial number etc)
trialf    <- list.files(meta, pattern = 'triallist*')
triallist <- data.frame()
for(i in trialf)
{
  tr        <- read.csv(paste0(meta, i), sep = ';')
  tr$ppn    <- parse_number(i)
  triallist <- rbind.data.frame(triallist,tr)
}

#make corrections to trial list due to experimenter errors
  # p10 trial 21 run with a weight
triallist$weight_condition[triallist$ppn==10 & triallist$trial=="21"] <- 'weight' #triallist$trialindex[triallist$ppn==10 & triallist$trial=="21"]
  # p14 double check trial "51", "52", "53" were all weight condition
triallist$weight_condition[triallist$ppn==14 & triallist$trial%in% c("51", "52", "53")] <- 'weight' #triallist$trialindex[triallist$ppn==14 & triallist$trial%in% c("51", "52", "53")]


#load in the metainfo data (containing info like gender, handedness, etc)
mtf     <- list.files(meta, pattern = 'bodymeta*')
mtlist  <- data.frame()
for(i in mtf)
{
  tr      <- read.csv(paste0(meta, i), sep = ';')
  mtlist  <- rbind.data.frame(mtlist,tr)
}
mtlist$ppn <- parse_number(as.character(mtlist$ppn))

Supporting methods

This research has been approved by the Ethics Committee Social Sciences (ECSS) of the Radboud University (reference nr.: 22N.002642). This study has been pre-registered before data collection (see (preregistration)[https://osf.io/jhdq4]).

Show code adding participant and condition information

# add some info about the dataset
participants  <- c(unique(mtlist$ppn))
numpart       <- length(participants)

# number & percentage female & male participants
part_fem <- round(sum(mtlist$sex=='f'), 1)
perc_fem  <- round((sum(mtlist$sex=='f')/numpart)*100, 1)
part_male <- round(sum(mtlist$sex=='m'), 1)
perc_male <- round((sum(mtlist$sex=='m')/numpart)*100, 1)

# body info
m_age           <- round(mean(mtlist$age), 1)
sd_age          <- round(sd(mtlist$age), 1)
m_weight        <- round(mean(mtlist$weight), 1)
sd_weight       <- round(sd(mtlist$weight), 1)
m_height        <- round(mean(mtlist$height), 1)
sd_height       <- round(sd(mtlist$height), 1)
bmi             <- mtlist$weight/((mtlist$height/100)^2)
m_bmi           <- round(mean(bmi), 1)
sd_bmi          <- round(sd(bmi), 1)

# other body measurements
  # Function to calculate the mean of a character vector
  mean_of_char_vector <- function(vec) {
    mean(as.numeric(vec))}

# we also measure upper/under arm length (but will leave it at this)
skinfold  <- strsplit(mtlist$upperarmfold, '_')
# Applying the function to each element of the list
skinfs    <- sapply(skinfold, mean_of_char_vector)
m_skin    <- round(mean(skinfs), 1)
sd_skin   <- round(sd(skinfs), 1)


# read data
tr_wd <- read.csv(paste0(procdtot, 'fulldata.csv'))
tr_wd <- subset(tr_wd, trialindex > 8 & vocal_condition=='vocalize') #only real trials and only vocalization
numtrials = nrow(tr_wd)
perc_w = round((sum(tr_wd$weight_condition=='weight')/numtrials)*100, 1) #percentage weight
perc_pas = round((sum(tr_wd$movement_condition=='no movement')/numtrials)*100, 1) #percentage passive
perc_int = round((sum(tr_wd$movement_condition=='internal rotation')/numtrials)*100, 1) #percentage internal rotation
perc_ext = round((sum(tr_wd$movement_condition=='external rotation')/numtrials)*100, 1) #percentage external rotation
perc_flex = round((sum(tr_wd$movement_condition=='flexion')/numtrials)*100, 1) #percentage flexion
perc_extens = round((sum(tr_wd$movement_condition=='extension')/numtrials)*100, 1) #percentage extension

Experimental design

This study concerned a two-level wrist-weight manipulation (no weight vs. weight), a two-level within-subject vocalization condition (expire vs. vocalize), and a five-level within-subject movement condition (‘no movement’, ‘extension’, ‘flexion’, ‘external rotation’, ‘internal rotation’). With 4 trial repetitions over the experiment, we yield 80 (2 weights x 2 vocalizations x 5 movements x 4 repetitions) trials per participant. Trials were blocked by weight condition and vocalization condition (so that weights and task did not switch from trial to trial). Within blocks all movement conditions were randomized.

Participants

The current pre-registered confirmatory experiment was supported by a power analysis (see preregistration). As planned, we collected N = (17) participants, with the following demographics and biometrics: 7 female, 10 male, M (SD) age = 28.50 (6.50), M (SD) body weight = 72.10 kg (10.20), M (SD) body height = 175.10 cm (8.50), M (SD) BMI = 23.40 (2.20), M (SD) triceps skinfold = 19.10 mm (4.30).

Exclusions and deviations from pre-registration

We also performed the experiment with one other participant, but due to an issue with LabStreamingLayer (LSL), this dataset could not be synchronized and was lost. Furthermore, due to running over time, one participant had to terminate the study earlier about halfway through. Note further, that in our pre-registration we wanted to admit participants with a BMI lower than 25, but since participants were difficult to recruit we accepted three participants with slightly higher BMIs too (max BMI of the current dataset: 27.10). Participants were all able-bodied and did not have any constraints in performing the task. Finally, as stated in the pre-registration we will only, or primarily, report results on the vocalization trials (and not the expiration-only trials). This means that for this report in total, we have N = 17 participants, with 636 analyzable trials, with balanced conditions: weight = 50.20%, no movement = 19.70%, internal rotation = 20.00 %, external rotation = 20.30%, flexion = 20.10 %, extension = 20.00 %.

Measurements and equipment

Body measurements

To enable future analyses of possible factors modulating individual-specific body properties, we collect some basic information about body properties. Namely, weight, under-arm length, upper-arm length, triceps skinfold, and upper-arm circumference.

Experiment protocol

The experiment was coded in Python using functions from PsychoPy. The experiment was controlled via a Brainvision Button Box (Brain Products GmbH, Munich, Germany), which was also streaming its output to the data collection PC unit.

Wrist weights

To manipulate the mass set in motion during upper limb movement, participants wore a wrist weight in half of the trials. We used a TurnTuri sports wrist weight of 1 kg.

Video and kinematics

The participants are recorded via a video camera (Logitech StreamCam), sampling at 60 frames per second. We used MediaPipe (Lugaresi et al. 2019) to track the skeleton and facial movements, which is implemented in Masked-piper which we also use for masking the videos (Owoyele et al. 2022). The motion-tracked skeleton, specifically the wrist of the dominant hand, is used to estimate movement initiation, peak speed, and the end of the movement. The motion tracking is, however, only used for determining movement windows and provides some descriptive information about kinematics, as kinetics is more of central concern here.

Muscle activity (Surface ElectroMyography: sEMG)

We measured sEMG using a wired BrainAmp ExG system (Brain Products GmbH, Munich, Germany). Disposable surface electrodes (Kendall 24mm Arbo H124SG) were used, and for each of the four muscle targets we had 3 (active, reference, ground) electrodes (12 electrodes total). The sEMG system sampled at 2,500 Hz (for post-processing filters see below).
For an overview of the electrode attachments, see Fig. S1. We prepare the skin surface for EMG application with a scrub gel (NuPrep) followed by a cotton ball swipe with alcohol (Podior 70 %). Active and reference electrodes were attached with a 15mm distance center to center.

Figure S1: Overview sEMG target muscles. Active (a), reference (r), and ground (g) sEMG electrodes, for each muscle target.

This figure is reproduced in Figure 1, panel A, of the main report.

We attached electrodes for focal muscles that directly participate in the internal (pectoralis major) and external rotation (infraspinatus) of the humerus. Electrodes were applied for focal muscles ipsilaterally (relative to the dominant hand). We attached electrodes to the muscle belly of the clavicular head of the pectoralis major, with a ground electrode on the clavicle on the opposite side.
We also attached electrodes for postural muscles which will likely anticipate and react to postural perturbations due to upper limb movements. Since these muscles should act in the opposite direction of the postural perturbation of the dominant hand, we applied electrodes contralaterally to the dominant hand. We attach electrodes to the rectus abdominis, with a ground electrode on the iliac crest on the opposite side. We also attached electrodes to the erector spinae muscle group (specifically, the iliocostalis lumborum).

Ground reaction measurements

We used an inhouse-built 1m² balance board with vertical pressure sensors. The sensors were derived and remodified from four Wii-Balance board sensors. The sampling rate was 400 Hz. The system was time-locked within millisecond accuracy and has a spatial accuracy of several sub-millimeters. A national instruments card, USB-62221 performed the A/D conversion and was connected via USB to the PC.

Acoustics

To ensure proper acoustic intensity measurements we used a headset microphone; MicroMic C520 (AKG, Inc.) headset condenser cardioid microphone sampling at 16 kHz. The gain levels of the condenser power source were set by the hardware (and could not be changed).

Here are five audio examples from a male participant producing the sustained vocalization in different conditions without added weight.

Show code setting up the audio examples

library(htmltools)

audioexamples <- paste0(basefolder, "/AudioExamples/")

audioexample_nomovement       <- paste0(audioexamples, "sample1_nomovement.wav")
audioexample_internalrotation <- paste0(audioexamples, "sample2_internalrotation.wav")
audioexample_flexion          <- paste0(audioexamples, "sample3_flexion.wav")
audioexample_externalrotation <- paste0(audioexamples, "sample4_externalrotation.wav")
audioexample_extension        <- paste0(audioexamples, "sample5_extension.wav")

html_tag_audio <- function(file, type = c("wav")) {
  type <- match.arg(type)
  htmltools::tags$audio(
    controls = NA,
    htmltools::tags$source(
      src = file,
      type = glue::glue("audio/{type}", type = type)
    )
  )
}

The first example is from the no movement condition:

The second example is from the internal rotation condition:

The third example is from the external rotation condition:

The fourth example is from the extension condition: