Introduction

This Rmarkdown notebook contains a detailed documentation of confirmatory analyses from the pre-registration, followed by exploratory analyses. It contains the statistical code, model syntax, and complete reproducible output and figures. Note that we pre-registered a conservative alpha to reduce false positive errors: we deem any result statistically reliable if and only if p-values are lower than the restricted alpha = .05/3 = .0166.

Loading and cleaning the data

#load in the timing data
curfolder <- getwd()
D <- read.csv(paste0(dirname(curfolder), "/ProcessedData/ProcessedTimingData/DD.csv"))

#this is the confirmatory study, so we can replace 'pilot' with 'confirmatory' to avoid confusion
D$ppn <- str_replace(D$ppn, "Pilot", "Confirmatory")

#rename/code some variables
D$accent  <- ifelse(D$accent=="yes", "stress mark present", "stress mark absent")
D$accent  <- factor(D$accent, levels=c("stress mark absent", "stress mark present"))
D$correct <- factor(D$correct, levels=c("L2 correct", "L2 incorrect & L1 match", "L2 incorrect & L1 mismatch"))
  #L2 correct: e.g., correctly saying *profeSOR*, with stress on the final syllable
  #L2 correct & L1 match: e.g., incorrectly saying *proFEsor*, with stress on the native language competitor syllable -fe
  #L2 correct & L1 mismatch: producing stress on another syllable altogether, e.g., *PROfesor*
D$stress  <- as.character(D$stress)
D$stress <- ifelse(D$stress=="same", "stress-matching", "stress-mismatching")
D$stress <-  factor(D$stress, levels=c("stress-matching", "stress-mismatching"))
D$condition <-  factor(D$condition, levels=c("nogesture", "gesture"))
D$ppn <- parse_number(D$ppn)

#add an accuracy variable
D$accuracy <- ifelse(D$correct== "L2 correct", 1, 0)

#remove a manually coded column with a Dutch name
D$Handmatig.verschil.L1.L2 <- NULL


#create a variable that codes for the direction of (incorrect) stress placement
D$stressdiff_cat <- ifelse(D$stressed.syllable.L1>D$stressed.syllable.L2, 'L1 follows', ifelse(D$stressed.syllable.L1<D$stressed.syllable.L2, 'L1 precedes', 'L1_L2 match'))

  #Note that somehow the items pragMAtica and ensaLAda were mislabeled.
  table(D$stressdiff_cat,D$stress)
##              
##               stress-matching stress-mismatching
##   L1 follows                0               1404
##   L1 precedes              52               1040
##   L1_L2 match            2444                 52
  # pragMAtica was mislabeled as stress-mismatching, while it is stress-matching (NL: pragMAtisch)
  unique(D[D$stressdiff_cat=='L1_L2 match' & D$stress!="stress-matching",]$target)
## [1] "pragm\xe1tica"
  # fix pragMAtica:
  D$stress[D$stressdiff_cat=='L1_L2 match'] <- 'stress-matching'
  
  # ensaLAda was mislabeled as 'L1 precedes', while it is stress-matching (NL: saLAde)
  unique(D[D$stressdiff_cat!='L1_L2 match' & D$stress=="stress-matching",]$target)
## [1] "ensalada"
  # fix ensaLAda:
  D$stressdiff_cat[D$stress=='stress-matching'] <- 'L1_L2 match'
  
  # And now it's correct:
  table(D$stressdiff_cat,D$stress)
##              
##               stress-matching stress-mismatching
##   L1 follows                0               1404
##   L1 precedes               0               1040
##   L1_L2 match            2548                  0

Exclusions

Two authors (LvM and MH) manually assessed for each trial whether the correct word was produced, or not (e.g., saying a different word, omitting sounds or syllables, disfluencies). In addition, sometimes the forced aligner EasyAlign failed to give output or produced output with unexpected syllable counts. All these trials were excluded (6.74% of total dataset excluded).

#See the Exclusion_info folder for all details.
exclusions <- read.csv(paste0(dirname(curfolder), '/Exclusion_info/files_met_errors_v4.csv'))
exclusions$gesturecondition <- str_remove(exclusions$gesturecondition, '.wav')
  #before exclusions
  before <- nrow(D)
  #apply exclusion based on hand-check
  D <- D[!(paste(D$ppn, D$trial, D$condition) %in% paste(exclusions$ppnr, exclusions$itemID, exclusions$gesturecondition)),]
  after1 <- nrow(D)
  changebefore_1 <- (1-(after1/before))*100
  # now: 3.95% data excluded
  
  #then the final exclusions are based on errors with easyalign,
  #either generating no textgrid at all, or detecting an unexpected number of syllables
  count(D$error) #types of errrors
##                                     x freq
## 1 no textgrid info (easy align error)  133
## 2                 only one syllable 1    1
## 3                                <NA> 4661
  D <- D[is.na(D$error),]
  after2 <- nrow(D)
  changeafter_after2 <- (1-(after2/after1))*100
  changebefore_after2 <- changebefore_1+changeafter_after2 #total amount of exclusions expressed in percentage
  # total excluded: 6.74% of data
  
  #Finally, there was one trial of the stress-matching item *ensaLAda* that was
  #incorrectly labeled as "L2 incorrect & L1 match" (which is impossible for stress-matching items).
  # This happened on a trial where the target was ensaLAda (in Dutch: saLAde)
  # and the participant pronounced enSAlada.
  # Therefore, this trial is recoded to: L2 incorrect & L1 mismatch
  D$correct <- as.character(D$correct)
  trials_with_wronglabel <- D[D$stress=="stress-matching" & D$correct=="L2 incorrect & L1 match",]
  len_trials_with_wronglabel <- length(trials_with_wronglabel)
  #recode the trial with wrong label
  D$correct[D$stress=="stress-matching" & D$correct=="L2 incorrect & L1 match"] <- "L2 incorrect & L1 mismatch"
  D$correct <- as.factor(D$correct)

Confirmatory analyses

We pre-registered five confirmatory analyses. The first two concerned acoustic analyses comparing gesture vs. no-gesture conditions. The final three tested the gesture-speech synchrony in time.

  • 1A: Does gesturing help learners to more accurately place stress on the correct syllable in the foreign words (relative to no-gesture)?
  • 1B: Does gesturing boost the acoustic markers of stress (relative to no-gesture)?
  • 2A: Do learners produce more variable (i.e., ‘sloppy’) gesture-speech synchrony is stress-mismatching vs. stress-matching trials due to competition from their native language?
  • 2B: Focusing on stress-mismatching trials: when learners get the stress placement correct (i.e., acoustically), does the gestural timing show evidence of temporal attraction towards the native language competitor syllable?
  • 2C: Focusing on stress-mismatching trials: when learners get the stress placement incorrect (i.e., incorrectly stressing the native language competitor syllable), does the gestural timing show evidence of temporal attraction towards by the foreign language target syllable?

ConfRQ.1A: stress placement accuracy

Here we provide a descriptive overview of participants’ stress placement accuracy, separately for stress-(mis)matching items.

(#tab:table0102)
Table 1. Percentage of accurate stress placement trials in stress-matching items.
accuracy category percentage correct
L2 correct 60.32
L2 incorrect & L1 match 0.00
L2 incorrect & L1 mismatch 39.68
Table 2. Percentage of accurate stress placement trials in stress-mismatching items.
accuracy category percentage correct
L2 correct 53.09
L2 incorrect & L1 match 26.97
L2 incorrect & L1 mismatch 19.94

Figure 1 plots the percentage of trials with correct stress placement separately for the gesture vs. no-gesture condition, for stress-matching and stress-mismatching items, and for items with or without an orthographic stress mark. Each line represents one participant, with the boxplots showing the group behavior.

To test the effect of gesturing on stress placement accuracy, we first build a base Generalized Linear Mixed Model predicting accuracy (binomial dependent variable), with only random intercepts by Participants and Items (model0). This base model is then extended with the fixed effect condition (gesture vs. no-gesture), stress (stress-matching vs. stress-mismatching) and even the 3-way interaction condition-by-stress-by-accent. However, none of these more complex models showed a significantly better fit to the data. Therefore, we conclude that gesturing did not influence participants’ accuracy in getting the stress placement right.

D$ppn <- as.factor(D$ppn)
#basemodel predicting the overall accuracy
model0 <- glmer(accuracy ~  1 + (1 | ppn)+(1|target), data = D, family = binomial(link="logit"))

#alternative model with gesture versus nogesture as predictors
model1 <- glmer(accuracy ~  condition  + (1 | ppn)+(1|target),data = D, family = binomial(link="logit"))

#alternative model with stressmatch versus stressmismatch as predictors
model2 <- glmer(accuracy ~  stress  + (1 | ppn)+(1|target),data = D, family = binomial(link="logit"))

#alternative model with, stress, accentedness, and gesture versus no gesture as predictors
model3 <- glmer(accuracy ~  condition*accent*stress + (1|ppn) + (1|target), data = D, family = binomial(link="logit"))

#compare models
anovcomp01 <- anova(model0, model1, model2, model3) #test difference basemodel versus alternative models

#save model 1 (only condition) results 
sum1 <- summary(model1) 

#save model 2 (only stress match vs mismatch) results 
sum2 <- summary(model2)

#save model 3 (3way interaction gesture, stress, accent) results 
sum3 <- summary(model3)

posthoc3 <- lsmeans(model3, list(pairwise ~ condition|accent|stress),  adjust="bonferroni")
Click here for model 1 R output(model coefficients, effect sizes) 
```r
sum1
```

```
## Generalized linear mixed model fit by maximum likelihood (Laplace
##   Approximation) [glmerMod]
##  Family: binomial  ( logit )
## Formula: accuracy ~ condition + (1 | ppn) + (1 | target)
##    Data: D
## 
##      AIC      BIC   logLik deviance df.resid 
##   5316.3   5342.1  -2654.1   5308.3     4657 
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -5.5116 -0.7215  0.3024  0.6866  4.0079 
## 
## Random effects:
##  Groups Name        Variance Std.Dev.
##  target (Intercept) 1.6858   1.2984  
##  ppn    (Intercept) 0.3357   0.5794  
## Number of obs: 4661, groups:  target, 94; ppn, 26
## 
## Fixed effects:
##                  Estimate Std. Error z value Pr(>|z|)  
## (Intercept)       0.36470    0.18278   1.995    0.046 *
## conditiongesture  0.03399    0.06867   0.495    0.621  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr)
## conditngstr -0.188
```
Click here for model 2 R output(model coefficients, effect sizes) 
```r
sum2
```

```
## Generalized linear mixed model fit by maximum likelihood (Laplace
##   Approximation) [glmerMod]
##  Family: binomial  ( logit )
## Formula: accuracy ~ stress + (1 | ppn) + (1 | target)
##    Data: D
## 
##      AIC      BIC   logLik deviance df.resid 
##   5313.9   5339.7  -2653.0   5305.9     4657 
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -5.4322 -0.7217  0.3013  0.6871  3.8937 
## 
## Random effects:
##  Groups Name        Variance Std.Dev.
##  target (Intercept) 1.6397   1.2805  
##  ppn    (Intercept) 0.3356   0.5793  
## Number of obs: 4661, groups:  target, 94; ppn, 26
## 
## Fixed effects:
##                          Estimate Std. Error z value Pr(>|z|)   
## (Intercept)                0.6039     0.2254   2.679  0.00738 **
## stressstress-mismatching  -0.4430     0.2743  -1.615  0.10632   
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr)
## strssstrss- -0.613
```
Click here for more complex model 3 R output (model coefficients, effect sizes, and posthoc) 
```r
sum3
```

```
## Generalized linear mixed model fit by maximum likelihood (Laplace
##   Approximation) [glmerMod]
##  Family: binomial  ( logit )
## Formula: accuracy ~ condition * accent * stress + (1 | ppn) + (1 | target)
##    Data: D
## 
##      AIC      BIC   logLik deviance df.resid 
##   5321.1   5385.6  -2650.6   5301.1     4651 
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -5.4983 -0.7237  0.3013  0.6879  3.9502 
## 
## Random effects:
##  Groups Name        Variance Std.Dev.
##  target (Intercept) 1.5649   1.2510  
##  ppn    (Intercept) 0.3356   0.5793  
## Number of obs: 4661, groups:  target, 94; ppn, 26
## 
## Fixed effects:
##                                                                      Estimate
## (Intercept)                                                          0.529974
## conditiongesture                                                     0.112007
## accentstress mark present                                            0.118492
## stressstress-mismatching                                            -0.005799
## conditiongesture:accentstress mark present                          -0.168267
## conditiongesture:stressstress-mismatching                           -0.071138
## accentstress mark present:stressstress-mismatching                  -0.866619
## conditiongesture:accentstress mark present:stressstress-mismatching  0.161588
##                                                                     Std. Error
## (Intercept)                                                           0.293028
## conditiongesture                                                      0.137399
## accentstress mark present                                             0.393763
## stressstress-mismatching                                              0.388412
## conditiongesture:accentstress mark present                            0.197038
## conditiongesture:stressstress-mismatching                             0.194978
## accentstress mark present:stressstress-mismatching                    0.554455
## conditiongesture:accentstress mark present:stressstress-mismatching   0.274916
##                                                                     z value
## (Intercept)                                                           1.809
## conditiongesture                                                      0.815
## accentstress mark present                                             0.301
## stressstress-mismatching                                             -0.015
## conditiongesture:accentstress mark present                           -0.854
## conditiongesture:stressstress-mismatching                            -0.365
## accentstress mark present:stressstress-mismatching                   -1.563
## conditiongesture:accentstress mark present:stressstress-mismatching   0.588
##                                                                     Pr(>|z|)  
## (Intercept)                                                           0.0705 .
## conditiongesture                                                      0.4150  
## accentstress mark present                                             0.7635  
## stressstress-mismatching                                              0.9881  
## conditiongesture:accentstress mark present                            0.3931  
## conditiongesture:stressstress-mismatching                             0.7152  
## accentstress mark present:stressstress-mismatching                    0.1181  
## conditiongesture:accentstress mark present:stressstress-mismatching   0.5567  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) cndtng accnmp strss- cnd:mp cndt:- ampr:-
## conditngstr -0.232                                          
## accntstrsmp -0.632  0.172                                   
## strssstrss- -0.641  0.175  0.476                            
## cndtngst:mp  0.161 -0.697 -0.250 -0.122                     
## cndtngstr:-  0.163 -0.705 -0.121 -0.249  0.491              
## amprsnt:st-  0.448 -0.123 -0.710 -0.700  0.178  0.175       
## cndtng:mp:- -0.116  0.500  0.179  0.177 -0.717 -0.709 -0.248
```

```r
   # Dmod2
posthoc3
```

```
## $`lsmeans of condition | accent, stress`
## accent = stress mark absent, stress = stress-matching:
##  condition lsmean    SE  df asymp.LCL asymp.UCL
##  nogesture  0.530 0.293 Inf   -0.0444     1.104
##  gesture    0.642 0.293 Inf    0.0669     1.217
## 
## accent = stress mark present, stress = stress-matching:
##  condition lsmean    SE  df asymp.LCL asymp.UCL
##  nogesture  0.648 0.309 Inf    0.0438     1.253
##  gesture    0.592 0.308 Inf   -0.0116     1.196
## 
## accent = stress mark absent, stress = stress-mismatching:
##  condition lsmean    SE  df asymp.LCL asymp.UCL
##  nogesture  0.524 0.302 Inf   -0.0668     1.115
##  gesture    0.565 0.302 Inf   -0.0260     1.156
## 
## accent = stress mark present, stress = stress-mismatching:
##  condition lsmean    SE  df asymp.LCL asymp.UCL
##  nogesture -0.224 0.296 Inf   -0.8032     0.355
##  gesture   -0.190 0.296 Inf   -0.7692     0.390
## 
## Results are given on the logit (not the response) scale. 
## Confidence level used: 0.95 
## 
## $`pairwise differences of condition | accent, stress`
## accent = stress mark absent, stress = stress-matching:
##  3                   estimate    SE  df z.ratio p.value
##  nogesture - gesture  -0.1120 0.137 Inf  -0.815  0.4150
## 
## accent = stress mark present, stress = stress-matching:
##  3                   estimate    SE  df z.ratio p.value
##  nogesture - gesture   0.0563 0.141 Inf   0.398  0.6904
## 
## accent = stress mark absent, stress = stress-mismatching:
##  3                   estimate    SE  df z.ratio p.value
##  nogesture - gesture  -0.0409 0.138 Inf  -0.295  0.7677
## 
## accent = stress mark present, stress = stress-mismatching:
##  3                   estimate    SE  df z.ratio p.value
##  nogesture - gesture  -0.0342 0.133 Inf  -0.258  0.7968
## 
## Results are given on the log odds ratio (not the response) scale.
```

ConfRQ.1B: gesturing boosts peak intensity

Figure 2 plots the three acoustic markers of stress (peak F0, peak envelope, vocal duration) for each stressed syllable (regardless of correct or incorrect placement), split by gesture condition. The last panel gives the acoustic stress score, which is a composite measure that was used for stress detection.

Figure 2. Effect of gesture vs. no gesture on acoustic markers of stress (also only boxplots for visibility)

We performed a mixed linear regression with normalized acoustic markers as dependent variable, and acoustic marker type (peak F0, peak envelope, and duration) x condition (gesture vs. no-gesture) as independent variables. This analysis includes both correct and incorrect trials. Note that peak envelope was mapped onto the intercept and therefore the simple effect of Condition (positive estimates in model01,model02) reveals that gesturing boosts the peak envelope in stressed syllables. The interactions between Condition and Marker (negative estimates) show that this effect only held for peak envelope, not for f0 or duration.

No interactions between Condition and Accuracy were found, suggesting that this intensity-boosting effect of gesture held for similarly across correct and incorrect trials. However, we did find an unexpected simple effect of Accuracy, suggesting that, when speakers produced stress on the correct foreign language target syllable, their peak intensity was higher than when stress placement was incorrect. This may indicate an ‘inconfidence effect’ on trials where stress was placed incorrectly.

Dlong <- gather(D, "marker", "acoust_out", 15:17)

#At present, accuracy is 0 or 1. In order to map the grand mean onto intercept,
#accuracy_devcod maps 0 onto -0.5 and 1 onto +0.5.
#This way, simple effects of other predictors hold for the entire dataset,
#irrespective of accuracy. For instance, the simple effect of gesture
#tests whether gesturing boosts peak amplitude across both accurate &
#inaccurate trials.
Dlong$accuracy_devcod <- Dlong$accuracy - 0.5

#alternative model with gesture versus no gesture as predictor
model0 <- lme(acoust_out~1, data = Dlong, random =  list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)
#model1 <- lme(acoust_out~marker*condition, data = Dlong, random =  list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)
model1 <- lme(acoust_out~marker*condition, data = Dlong, random =  list(~1|ppn), method = "ML", na.action = na.exclude)
model2 <- lme(acoust_out~marker*condition*accuracy_devcod, data = Dlong, random =  list(~1|ppn), method = "ML", na.action = na.exclude)
anova(model0, model1, model2) #test difference basemodel versus alternative models
##        Model df      AIC      BIC    logLik   Test  L.Ratio p-value
## model0     1  4 29437.59 29467.78 -14714.80                        
## model1     2  8 24118.21 24178.57 -12051.10 1 vs 2 5327.388  <.0001
## model2     3 14 23809.49 23915.13 -11890.75 2 vs 3  320.714  <.0001
#summary model 1 post hoc
anovamod0mod1 <- anova(model0, model1)
sum1 <- summary(model1)
posthocsum1 <- lsmeans(model1, list(pairwise ~ condition|marker),  adjust="bonferroni")
sum2 <- summary(model2)
Click here for model 1 R output 
```r
sum1
```

```
## Linear mixed-effects model fit by maximum likelihood
##   Data: Dlong 
##        AIC      BIC   logLik
##   24118.21 24178.57 -12051.1
## 
## Random effects:
##  Formula: ~1 | ppn
##         (Intercept)  Residual
## StdDev:   0.1965819 0.5706514
## 
## Fixed effects:  acoust_out ~ marker * condition 
##                                     Value  Std.Error    DF   t-value p-value
## (Intercept)                     1.8000979 0.04033922 13952  44.62402  0.0000
## markerpeakF0z                  -0.5294379 0.01674767 13952 -31.61262  0.0000
## markersDURz                    -0.9166110 0.01674767 13952 -54.73065  0.0000
## conditiongesture                0.0603662 0.01672225 13952   3.60993  0.0003
## markerpeakF0z:conditiongesture -0.0553044 0.02364677 13952  -2.33877  0.0194
## markersDURz:conditiongesture   -0.0588253 0.02364677 13952  -2.48767  0.0129
##  Correlation: 
##                                (Intr) mrkrF0 mrkDUR cndtng mrkF0:
## markerpeakF0z                  -0.208                            
## markersDURz                    -0.208  0.500                     
## conditiongesture               -0.208  0.501  0.501              
## markerpeakF0z:conditiongesture  0.147 -0.708 -0.354 -0.707       
## markersDURz:conditiongesture    0.147 -0.354 -0.708 -0.707  0.500
## 
## Standardized Within-Group Residuals:
##         Min          Q1         Med          Q3         Max 
## -4.85998683 -0.55253726  0.01128636  0.59271935  8.60616108 
## 
## Number of Observations: 13983
## Number of Groups: 26
```
Click here for model 2 R output 
```r
sum2
```

```
## Linear mixed-effects model fit by maximum likelihood
##   Data: Dlong 
##        AIC      BIC    logLik
##   23809.49 23915.13 -11890.75
## 
## Random effects:
##  Formula: ~1 | ppn
##         (Intercept)  Residual
## StdDev:   0.1913563 0.5641606
## 
## Fixed effects:  acoust_out ~ marker * condition * accuracy_devcod 
##                                                     Value  Std.Error    DF
## (Intercept)                                     1.7823027 0.03935873 13946
## markerpeakF0z                                  -0.5136284 0.01670153 13946
## markersDURz                                    -0.8958092 0.01670153 13946
## conditiongesture                                0.0564300 0.01668716 13946
## accuracy_devcod                                 0.2730674 0.02370745 13946
## markerpeakF0z:conditiongesture                 -0.0505263 0.02359751 13946
## markersDURz:conditiongesture                   -0.0517153 0.02359751 13946
## markerpeakF0z:accuracy_devcod                  -0.2440233 0.03340306 13946
## markersDURz:accuracy_devcod                    -0.3210807 0.03340306 13946
## conditiongesture:accuracy_devcod                0.0372685 0.03339189 13946
## markerpeakF0z:conditiongesture:accuracy_devcod -0.0512759 0.04719502 13946
## markersDURz:conditiongesture:accuracy_devcod   -0.0792746 0.04719502 13946
##                                                  t-value p-value
## (Intercept)                                     45.28355  0.0000
## markerpeakF0z                                  -30.75337  0.0000
## markersDURz                                    -53.63635  0.0000
## conditiongesture                                 3.38164  0.0007
## accuracy_devcod                                 11.51821  0.0000
## markerpeakF0z:conditiongesture                  -2.14117  0.0323
## markersDURz:conditiongesture                    -2.19156  0.0284
## markerpeakF0z:accuracy_devcod                   -7.30542  0.0000
## markersDURz:accuracy_devcod                     -9.61231  0.0000
## conditiongesture:accuracy_devcod                 1.11609  0.2644
## markerpeakF0z:conditiongesture:accuracy_devcod  -1.08647  0.2773
## markersDURz:conditiongesture:accuracy_devcod    -1.67972  0.0930
##  Correlation: 
##                                                (Intr) mrkrF0 mrkDUR cndtng
## markerpeakF0z                                  -0.212                     
## markersDURz                                    -0.212  0.500              
## conditiongesture                               -0.212  0.500  0.500       
## accuracy_devcod                                -0.039  0.091  0.091  0.091
## markerpeakF0z:conditiongesture                  0.150 -0.708 -0.354 -0.707
## markersDURz:conditiongesture                    0.150 -0.354 -0.708 -0.707
## markerpeakF0z:accuracy_devcod                   0.027 -0.130 -0.065 -0.065
## markersDURz:accuracy_devcod                     0.027 -0.065 -0.130 -0.065
## conditiongesture:accuracy_devcod                0.027 -0.065 -0.065 -0.134
## markerpeakF0z:conditiongesture:accuracy_devcod -0.019  0.092  0.046  0.095
## markersDURz:conditiongesture:accuracy_devcod   -0.019  0.046  0.092  0.095
##                                                accrc_ mrkF0: mrDUR: mrF0:_
## markerpeakF0z                                                             
## markersDURz                                                               
## conditiongesture                                                          
## accuracy_devcod                                                           
## markerpeakF0z:conditiongesture                 -0.065                     
## markersDURz:conditiongesture                   -0.065  0.500              
## markerpeakF0z:accuracy_devcod                  -0.704  0.092  0.046       
## markersDURz:accuracy_devcod                    -0.704  0.046  0.092  0.500
## conditiongesture:accuracy_devcod               -0.704  0.095  0.095  0.500
## markerpeakF0z:conditiongesture:accuracy_devcod  0.499 -0.134 -0.067 -0.708
## markersDURz:conditiongesture:accuracy_devcod    0.499 -0.067 -0.134 -0.354
##                                                mDUR:_ cndt:_ mF0::_
## markerpeakF0z                                                      
## markersDURz                                                        
## conditiongesture                                                   
## accuracy_devcod                                                    
## markerpeakF0z:conditiongesture                                     
## markersDURz:conditiongesture                                       
## markerpeakF0z:accuracy_devcod                                      
## markersDURz:accuracy_devcod                                        
## conditiongesture:accuracy_devcod                0.500              
## markerpeakF0z:conditiongesture:accuracy_devcod -0.354 -0.707       
## markersDURz:conditiongesture:accuracy_devcod   -0.708 -0.707  0.500
## 
## Standardized Within-Group Residuals:
##         Min          Q1         Med          Q3         Max 
## -4.98717546 -0.55670518  0.01282484  0.58405813  8.70241992 
## 
## Number of Observations: 13983
## Number of Groups: 26
```
Click here for posthoc model 1 output 
```r
posthocsum1
```

```
## $`lsmeans of condition | marker`
## marker = peakAMPz:
##  condition lsmean     SE df lower.CL upper.CL
##  nogesture  1.800 0.0403 25    1.717    1.883
##  gesture    1.860 0.0403 25    1.777    1.944
## 
## marker = peakF0z:
##  condition lsmean     SE df lower.CL upper.CL
##  nogesture  1.271 0.0403 25    1.188    1.354
##  gesture    1.276 0.0403 25    1.193    1.359
## 
## marker = sDURz:
##  condition lsmean     SE df lower.CL upper.CL
##  nogesture  0.883 0.0403 25    0.800    0.967
##  gesture    0.885 0.0403 25    0.802    0.968
## 
## Degrees-of-freedom method: containment 
## Confidence level used: 0.95 
## 
## $`pairwise differences of condition | marker`
## marker = peakAMPz:
##  2                   estimate     SE    df t.ratio p.value
##  nogesture - gesture -0.06037 0.0167 13952  -3.610  0.0003
## 
## marker = peakF0z:
##  2                   estimate     SE    df t.ratio p.value
##  nogesture - gesture -0.00506 0.0167 13952  -0.303  0.7621
## 
## marker = sDURz:
##  2                   estimate     SE    df t.ratio p.value
##  nogesture - gesture -0.00154 0.0167 13952  -0.092  0.9266
## 
## Degrees-of-freedom method: containment
```

ConfRQ.2A: gesture-speech (a)synchrony

RQ2A asked whether learners show more variable gesture-speech synchrony in stress-mismatching items compared to stress-matching items due to competition from their native language. Figure 3 visualizes the gesture-speech synchrony in stress-matching (yellow) and stress-mismatching (red) items, separately for items with and items without an orthographic stress mark. Time point 0 is the time of the acoustic intensity peak in the stressed syllable. Therefore, trials with a negative gesture-speech synchrony reflect trials where the gesture arrived earlier, and positive gesture-speech synchrony reflects trials where the gesture arrived later than the acoustic intensity peak.

Figure 3. Gesture-speech (a)synchrony depending on stress-(mis)match and stress mark

## Warning: Using `size` aesthetic for lines was deprecated in ggplot2 3.4.0.
## ℹ Please use `linewidth` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.

We calculated the absolute asynchrony to be able to quantify the width of these density distributions. Using a similar linear mixed modeling approach as before, we first constructed a base model (model0). This base model is then extended with fixed effects of stress and accent (model1), and their interaction (model2). However, neither of these two more complex models showed a significantly better fit to the data. This indicates that stress-(mis)match did not influence the absolute asynchrony. In fact, this is not very surprising because this analysis pools across items where the native language competitor syllable precedes the foreign language target syllable (as in: SP profeSOR vs. NL proFESsor) and items where the competitor syllable follows the target syllable (as in: SP MAquina vs. NL maCHIne). Therefore, RQ 2A and RQ 2B present a more fine-grained insight into the gesture-speech synchrony in stress-mismatching items.

subD$abs_asynchrony <- abs(subD$asynchrony)
#alternative model with gesture versus no gesture as predictor
model0 <- lme(abs_asynchrony~1, data = subD, random =  list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)
model1 <- lme(abs_asynchrony~stress+accent, data = subD, random =  list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)
model2 <- lme(abs_asynchrony~stress*accent, data = subD, random =  list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)

anovmod01 <- anova(model0, model1) #test difference basemodel versus model 1
anovmod02 <- anova(model0, model2) #test difference basemodel versus model 1

#if interaction effects are reliable we will follow up with post-hocs
sum1 <- summary(model1)
sum2 <- summary(model2)
posthoc2 <- lsmeans(model2, list(pairwise ~ stress|accent),  adjust="bonferroni")
#Dmod1 <- lme.dscore(model1, subD, type="nlme")
Click here for posthoc model 1 and 2 output 
```r
sum1
```

```
## Linear mixed-effects model fit by maximum likelihood
##   Data: subD 
##       AIC      BIC    logLik
##   29021.3 29055.85 -14504.65
## 
## Random effects:
##  Formula: ~1 | ppn
##         (Intercept)
## StdDev:    59.03449
## 
##  Formula: ~1 | target %in% ppn
##         (Intercept) Residual
## StdDev:    109.6153 42.61278
## 
## Fixed effects:  abs_asynchrony ~ stress + accent 
##                               Value Std.Error   DF   t-value p-value
## (Intercept)               152.79651 12.296537 2310 12.425979  0.0000
## stressstress-mismatching    6.29972  4.872941 2310  1.292797  0.1962
## accentstress mark present   4.03772  4.874024 2310  0.828416  0.4075
##  Correlation: 
##                           (Intr) strss-
## stressstress-mismatching  -0.189       
## accentstress mark present -0.185 -0.041
## 
## Standardized Within-Group Residuals:
##         Min          Q1         Med          Q3         Max 
## -1.01626856 -0.25265232 -0.06903372  0.20312430  1.61939178 
## 
## Number of Observations: 2338
## Number of Groups: 
##             ppn target %in% ppn 
##              26            2338
```

```r
sum2
```

```
## Linear mixed-effects model fit by maximum likelihood
##   Data: subD 
##        AIC      BIC    logLik
##   29023.03 29063.33 -14504.51
## 
## Random effects:
##  Formula: ~1 | ppn
##         (Intercept)
## StdDev:    59.03349
## 
##  Formula: ~1 | target %in% ppn
##         (Intercept) Residual
## StdDev:    109.6087 42.61032
## 
## Fixed effects:  abs_asynchrony ~ stress * accent 
##                                                        Value Std.Error   DF
## (Intercept)                                        151.60194 12.506581 2309
## stressstress-mismatching                             8.80474  6.812227 2309
## accentstress mark present                            6.59294  6.879981 2309
## stressstress-mismatching:accentstress mark present  -5.13129  9.749556 2309
##                                                      t-value p-value
## (Intercept)                                        12.121774  0.0000
## stressstress-mismatching                            1.292490  0.1963
## accentstress mark present                           0.958278  0.3380
## stressstress-mismatching:accentstress mark present -0.526310  0.5987
##  Correlation: 
##                                                    (Intr) strss- accnmp
## stressstress-mismatching                           -0.260              
## accentstress mark present                          -0.257  0.472       
## stressstress-mismatching:accentstress mark present  0.181 -0.699 -0.706
## 
## Standardized Within-Group Residuals:
##         Min          Q1         Med          Q3         Max 
## -1.02058163 -0.25288836 -0.06904729  0.20150862  1.62345644 
## 
## Number of Observations: 2338
## Number of Groups: 
##             ppn target %in% ppn 
##              26            2338
```

```r
posthoc2
```

```
## $`lsmeans of stress | accent`
## accent = stress mark absent:
##  stress             lsmean   SE df lower.CL upper.CL
##  stress-matching       152 12.5 25      126      177
##  stress-mismatching    160 12.6 25      134      186
## 
## accent = stress mark present:
##  stress             lsmean   SE df lower.CL upper.CL
##  stress-matching       158 12.6 25      132      184
##  stress-mismatching    162 12.6 25      136      188
## 
## Degrees-of-freedom method: containment 
## Confidence level used: 0.95 
## 
## $`pairwise differences of stress | accent`
## accent = stress mark absent:
##  2                                        estimate   SE   df t.ratio p.value
##  (stress-matching) - (stress-mismatching)    -8.80 6.81 2309  -1.292  0.1963
## 
## accent = stress mark present:
##  2                                        estimate   SE   df t.ratio p.value
##  (stress-matching) - (stress-mismatching)    -3.67 6.98 2309  -0.527  0.5985
## 
## Degrees-of-freedom method: containment
```

ConfRQ.2B: correct productions

Directional gesture-speech (a)synchrony

RQ2A asks: when participants correctly produce acoustic stress on the appropriate foreign language target syllable (i.e., correctly saying profeSOR), do the hands show temporal attraction towards the native language competitor syllable? In an item like profeSOR, this would show up by the maximum extension of the hand falling earlier than the peak intensity of the stressed syllable. However, in an item like MÁquina (NL: maCHIne), this would mean the hand would follow the stressed syllable.

In order to assess this native language attraction across both types of items, we calculated the directional gesture-speech (a)synchrony, considering correct trials only. This measure was the original gesture-speech (a)synchrony (time in ms between maximum extension and stress peak intensity), but the sign depended on the position of the native language competitor syllable being earlier or later than the foreign language target syllable. Negative sign meant ‘attracted towards the native language competitor syllable’ and positive sign meant ‘attracted away from the native language competitor syllable’.

For instance, if the original gesture-speech (a)synchrony on an item like profeSOR was +20 ms (i.e., hand falling late relative to the stressed syllable), the directional gesture-speech (a)synchrony was also +20 ms because the Dutch competitor word has stress on the earlier second syllable (proFESsor). However, if the original gesture-speech (a)synchrony on an item like MÁquina was +40 ms (i.e., hand falling late relative to the stressed syllable), the directional gesture-speech (a)synchrony was -40 ms because the Dutch competitor word has stress on the second syllable (maCHIne). With this coding, we predicted more negative directional gesture-speech (a)synchrony, biased by the native language, in stress-mismatching compared to stress-matching words.

Figure 4 plots the directional gesture-speech (a)synchrony in stress-mismatching and stress-matching items.

Figure 5 shows the directional gesture-speech (a)synchrony in stress-mismatching and stress-matching items for each individual participant. Participants with more negative directional gesture-speech (a)synchrony in stress-mismatching (upper panels) vs. stress-matching trials (lower panels) would show greater temporal native language competition (e.g., 2, 13, 14, 23)

#overall asynchrony in L1 direction (left=L1) by participants
bypps <- ggplot(Dsub, aes(x = stress, y= asynchrony_L2L1))+
  geom_hline(yintercept = 0, color = "red", size = 0.5)+geom_violin(fill=NA) +
  geom_quasirandom(color = "black", size = 0.7, alpha = 0.5)+
  geom_boxplot(alpha = 0)+theme_bw() + coord_flip() +
  facet_wrap(~ppn)
ggplotly(bypps)

Using a similar linear mixed modeling approach as before, we built linear mixed models testing the directional gesture-speech (a)synchrony (variable name: asynchrony_L2L1). The base model (model0) included random intercepts for Participants and Items only. The model1 extended model0 by including the fixed effect Stress, comparing stress-mismatching to stress-matching (mapped onto the intercept). Model comparison based on a log-likelihood ratio test indicated a significant effect of Stress, with a negative estimate. This indicates that participants’ hands demonstrated native language competition, even when their speech was acoustically correct.

Dsub <- subset(D, (condition == "gesture") & (correct == "L2 correct"))

model0 <- lme(asynchrony_L2L1~1, data = Dsub, random = list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)

model1 <- lme(asynchrony_L2L1~stress, data = Dsub, random =  list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)
print(anova(model0, model1)) #test difference basemodel versus model 1
##        Model df      AIC      BIC    logLik   Test L.Ratio p-value
## model0     1  4 17081.83 17102.61 -8536.917                       
## model1     2  5 17068.32 17094.29 -8529.160 1 vs 2 15.5142   1e-04
summary(model1) 
## Linear mixed-effects model fit by maximum likelihood
##   Data: Dsub 
##        AIC      BIC   logLik
##   17068.32 17094.29 -8529.16
## 
## Random effects:
##  Formula: ~1 | ppn
##         (Intercept)
## StdDev:      55.492
## 
##  Formula: ~1 | target %in% ppn
##         (Intercept) Residual
## StdDev:    133.5245 51.40608
## 
## Fixed effects:  asynchrony_L2L1 ~ stress 
##                              Value Std.Error   DF   t-value p-value
## (Intercept)              -47.97998 12.184761 1305 -3.937703   1e-04
## stressstress-mismatching -31.30987  7.931468 1305 -3.947550   1e-04
##  Correlation: 
##                          (Intr)
## stressstress-mismatching -0.305
## 
## Standardized Within-Group Residuals:
##         Min          Q1         Med          Q3         Max 
## -1.43294624 -0.20251932  0.01261942  0.19539856  1.62853786 
## 
## Number of Observations: 1332
## Number of Groups: 
##             ppn target %in% ppn 
##              26            1332

To further explore this evidence for native language attraction, we tested whether the attraction of the gesture is gradient, being dependent on the temporal distance between the native language competitor syllable and the foreign language target syllable. For instance, in an item like profeSOR (NL: proFESsor), the distance between the two stress targets is only 1 syllable. However, in an item like uniFORme (NL: Uniform), the distance is 2 syllables. Is the timing of the gesture sensitive to this distance?

Using a similar linear mixed modeling approach as before, we built a linear mixed model testing the gesture-speech (a)synchrony (variable name: asynchrony_L2L1). The base model (model0) included random intercepts for Participants and Items only. The model1 extended model0 by including the fixed effect Distance, quantifying the distance between the native language competitor syllable and the foreign language target syllable. The analysis was restricted to stress-mismatching items only, because this distance is always zero for stress-matching items.

Model comparison based on a log-likelihood ratio test indicated a significant effect of Distance, with a negative estimate. This indicates that, in stress-mismatching items, participants’ hands demonstrated greater native language competition with greater distance between the native vs. foreign language stress targets.

Dsub_mismatching <- subset(D, (condition == "gesture") & (correct == "L2 correct") & (stressdiff_cat != "L1_L2 match"))
Dsub_mismatching$distance <- abs(Dsub_mismatching$stressed.syllable.L1 - Dsub_mismatching$stressed.syllable.L2)

model0 <- lme(asynchrony_L2L1~1, data = Dsub_mismatching, random = list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)

model1 <- lme(asynchrony_L2L1~distance, data = Dsub_mismatching, random =  list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)
print(anova(model0, model1)) #test difference basemodel versus model 1
##        Model df      AIC      BIC    logLik   Test  L.Ratio p-value
## model0     1  4 7995.435 8013.154 -3993.718                        
## model1     2  5 7971.295 7993.443 -3980.647 1 vs 2 26.14062  <.0001
summary(model1) 
## Linear mixed-effects model fit by maximum likelihood
##   Data: Dsub_mismatching 
##        AIC      BIC    logLik
##   7971.295 7993.443 -3980.647
## 
## Random effects:
##  Formula: ~1 | ppn
##         (Intercept)
## StdDev:    34.27775
## 
##  Formula: ~1 | target %in% ppn
##         (Intercept) Residual
## StdDev:      136.64 51.62177
## 
## Fixed effects:  asynchrony_L2L1 ~ distance 
##                 Value Std.Error  DF   t-value p-value
## (Intercept)  14.63929  20.00111 593  0.731924  0.4645
## distance    -77.05851  14.91624 593 -5.166081  0.0000
##  Correlation: 
##          (Intr)
## distance -0.893
## 
## Standardized Within-Group Residuals:
##         Min          Q1         Med          Q3         Max 
## -1.16449471 -0.17981502  0.02941976  0.21664917  1.39054935 
## 
## Number of Observations: 620
## Number of Groups: 
##             ppn target %in% ppn 
##              26             620

Raw gesture-speech (a)synchrony.

In order to see the native language attraction more clearly, we also calculated the original gesture-speech (a)synchrony (i.e., not directional) separately for items where the native language competitor syllable preceded than the foreign language target syllable (as in: profeSOR) and items where the native language competitor syllable followed the foreign language target syllable (as in: MÁquina). Figure 6 plots the original gesture-speech (a)synchrony (negative = hand is earlier than acoustic stress; positive = hand is later than acoustic stress) for three conditions: stress-mismatching with early native language competitor, stress-mismatching with late native language competitor, and stress-matching trials for comparison. These data are also illustrated in Figure 1 in the manuscript.

The bottom panel shows the stress-matching trials (e.g., SP bailaRIna - NL balleRIna; panel is identical to bottom panel of Figure 4). These serve as control since we don’t expect any native language competition in these items. Indeed, this panel seems to show close gesture-speech synchrony, with gesture generally slightly leading the voice by ca. -40 ms.

The top panel seems to demonstrate more negative gesture-speech (a)synchrony values, while the middle seems to have more positive gesture-speech (a)synchrony – relative to the stress-matching data. This seems to support our hypothesis that, if the native language competitor syllable precedes the correct foreign language target syllable (top panel), gestures are temporally attracted to fall relatively early. Conversely, if the native language competitor syllable follows the correct foreign language target syllable (middle panel), gestures would be temporally attracted to fall relatively late.

These patterns were statistically assessed by means of Linear Mixed Models. The dependent variable was the raw gesture-speech (a)synchrony, with negative values indicating early gesture relative to the acoustic stress and positive values indicating late gesture relative to the acoustic stress (i.e., stressed peak intensity at 0). We first constructed a base model (model0), with only random intercepts for Participants and Items. This model was then compared to model1 that included the predictor stressdiff_cat_iMatch, with the levels: stress-matching (mapped onto the intercept), stress-mismatching – L1 follows (middle panel in Figure 6), stress-mismatching – L1 precedes (top panel in Figure 6). This model demonstrated a significantly better fit to the data, showing significant effects in the expected direction for both contrasts (stress-matching vs. L1 follows; stress-matching vs. L1 precedes). Therefore, we find evidence for a ‘kinematic accent’: even when the voice is accurately stressing the correct foreign language syllable, the hands are temporally attracted towards the native language competitor syllable.

model0 <- lme(asynchrony~1, data = Dsub, random =  list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)

# make sure the 'L1_L2 match' is mapped onto the intercept
Dsub$stressdiff_cat_iMatch <- relevel(as.factor(Dsub$stressdiff_cat),"L1_L2 match")

model3 <- lme(asynchrony~stressdiff_cat_iMatch, data = Dsub, random =  list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)
print(anova(model0,model3))
##        Model df      AIC      BIC    logLik   Test  L.Ratio p-value
## model0     1  4 16910.25 16931.03 -8451.124                        
## model3     2  6 16742.13 16773.30 -8365.067 1 vs 2 172.1128  <.0001
print(summary(model3))
## Linear mixed-effects model fit by maximum likelihood
##   Data: Dsub 
##        AIC     BIC    logLik
##   16742.13 16773.3 -8365.067
## 
## Random effects:
##  Formula: ~1 | ppn
##         (Intercept)
## StdDev:    77.41498
## 
##  Formula: ~1 | target %in% ppn
##         (Intercept) Residual
## StdDev:    116.6567 46.12543
## 
## Fixed effects:  asynchrony ~ stressdiff_cat_iMatch 
##                                      Value Std.Error   DF    t-value p-value
## (Intercept)                      -47.69040 15.939709 1304  -2.991924  0.0028
## stressdiff_cat_iMatchL1 follows   45.44514  8.857125 1304   5.130914  0.0000
## stressdiff_cat_iMatchL1 precedes -88.36477  8.454917 1304 -10.451288  0.0000
##  Correlation: 
##                                  (Intr) s__MLf
## stressdiff_cat_iMatchL1 follows  -0.156       
## stressdiff_cat_iMatchL1 precedes -0.173  0.296
## 
## Standardized Within-Group Residuals:
##          Min           Q1          Med           Q3          Max 
## -1.748316214 -0.210412112  0.009264972  0.197591041  1.584561747 
## 
## Number of Observations: 1332
## Number of Groups: 
##             ppn target %in% ppn 
##              26            1332

We further explored whether this evidence for a ‘kinematic accent’ could be accounted for by a potential confound with stress position. That is, some items where the native language competitor syllable followed the foreign language target syllable, had word-initial stress (e.g., MÁquina vs. NL: maCHIne). If a word has initial stress, it is very unlikely that participants would time their gesture before the stressed syllable, simply because there is no speech to align the gesture to. Hence, this might create an overall ‘late gesture’ bias in these items without the need for any native language competition. Note, however, that out of the total set of 32 “L1-follows items”, only 5 had initial stress.

Conversely, some items where the native language competitor syllable preceded the foreign language target syllable, had word-final stress (e.g., profeSOR vs. NL: proFESsor). Likewise, if a word has final stress, it is very unlikely that participants would time their gesture after the stressed syllable, simply because there is no speech to align the gesture to. Hence, this might create an overall ‘early gesture’ bias in these items without the need for any native language competition. Still, note that this concerned 15 items out of the total of 32 “L1-precedes items”.

To assess to what extent stress position influenced the overall evidence for native language attraction, we reran the models above but this time excluding the 5 + 15 items of concern, leaving 44 stress-mismatching items for analysis.

This analysis qualitatively replicated the results presented above, showing significant effects in the expected direction for both contrasts (stress-matching vs. L1 follows; stress-matching vs. L1 precedes). Therefore, even when we exclude items with initial stress or final stress, we still find evidence for a ‘kinematic accent’, independent from any possible influence of stress position.

length(unique(Dsub[Dsub$stressdiff_cat == "L1 follows" & Dsub$stressed.syllable.L2==1,]$target))
## [1] 5
unique(Dsub[Dsub$stressdiff_cat == "L1 follows" & Dsub$stressed.syllable.L2==1,]$target)
## [1] "f\xf3rmula" "gr\xe1fico" "clima"      "m\xe1quina" "s\xedmbolo"
length(unique(Dsub[Dsub$stressdiff_cat == "L1 precedes" & Dsub$stressed.syllable.L2==Dsub$Nsyllables_correct,]$target))
## [1] 15
unique(Dsub[Dsub$stressdiff_cat == "L1 precedes" & Dsub$stressed.syllable.L2==Dsub$Nsyllables_correct,]$target)
##  [1] "profesor"   "internet"   "doctor"     "color"      "factor"    
##  [6] "festival"   "ventilador" "mamut"      "ilustrador" "carnaval"  
## [11] "voleibol"   "sof\xe1"    "error"      "radiador"   "alcohol"
Dsub_excl <- Dsub
Dsub_excl$excludeditems <- 0
Dsub_excl[Dsub_excl$stressdiff_cat == "L1 follows" & Dsub_excl$stressed.syllable.L2==1,]$excludeditems <- 1
Dsub_excl[Dsub_excl$stressdiff_cat == "L1 precedes" & Dsub_excl$stressed.syllable.L2==Dsub_excl$Nsyllables_correct,]$excludeditems <- 1
Dsub_excl <- Dsub_excl[Dsub_excl$excludeditems!=1,]

model0_excl <- lme(asynchrony~1, data = Dsub_excl, random =  list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)

# make sure the 'L1_L2 match' is mapped onto the intercept
Dsub_excl$stressdiff_cat_iMatch <- relevel(as.factor(Dsub_excl$stressdiff_cat),"L1_L2 match")

model3_excl <- lme(asynchrony~stressdiff_cat_iMatch, data = Dsub_excl, random =  list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)
print(anova(model0_excl,model3_excl))
##             Model df      AIC      BIC    logLik   Test  L.Ratio p-value
## model0_excl     1  4 13005.60 13025.36 -6498.802                        
## model3_excl     2  6 12968.98 12998.61 -6478.490 1 vs 2 40.62303  <.0001
print(summary(model3_excl))
## Linear mixed-effects model fit by maximum likelihood
##   Data: Dsub_excl 
##        AIC      BIC   logLik
##   12968.98 12998.61 -6478.49
## 
## Random effects:
##  Formula: ~1 | ppn
##         (Intercept)
## StdDev:    74.56746
## 
##  Formula: ~1 | target %in% ppn
##         (Intercept) Residual
## StdDev:    116.5572 46.09826
## 
## Fixed effects:  asynchrony ~ stressdiff_cat_iMatch 
##                                      Value Std.Error   DF   t-value p-value
## (Intercept)                      -47.67683 15.421700 1003 -3.091542   2e-03
## stressdiff_cat_iMatchL1 follows   32.56651  9.527331 1003  3.418219   7e-04
## stressdiff_cat_iMatchL1 precedes -70.77617 14.693252 1003 -4.816917   0e+00
##  Correlation: 
##                                  (Intr) s__MLf
## stressdiff_cat_iMatchL1 follows  -0.150       
## stressdiff_cat_iMatchL1 precedes -0.100  0.163
## 
## Standardized Within-Group Residuals:
##          Min           Q1          Med           Q3          Max 
## -1.736839854 -0.197548258  0.008019638  0.184242245  1.589379284 
## 
## Number of Observations: 1031
## Number of Groups: 
##             ppn target %in% ppn 
##              26            1031

ConfRQ.2C: incorrect productions

Raw gesture-speech (a)synchrony.

We should be careful to immediately interpret the results from RQ2B as indicating that ‘accurate gestural timing is acquired after appropriate acoustic stress in foreign language acquisition’. This is because we have not yet investigated RQ2C: what is the temporal attraction of gesture in trials where the voice placed stress on the incorrect native language competitor syllable?

Figure 7 has three panels, just like Figure 6. The bottom panel is identical to the bottom panel in Figure 6 for comparision. However, the middle and top panel show new data. These panels show trials where the acoustic stress was incorrectly placed on the native language competitor syllable (timepoint zero). In items where the native language competitor syllable precedes the foreign language target syllable (top panel; e.g., showing trials where participants incorrectly said proFEsor), we seem to find that the gesture is timed relatively late, being temporally attracted towards the later foreign language target syllable. Conversely, in items where the native language competitor syllable follows the foreign language target syllable (middle panel; e.g., showing trials where participants incorrectly said máQUIna), we seem to find that the gesture is timed relatively early, likewise being temporally attracted towards the earlier foreign language target syllable.

This converse temporal attraction was tested statistically by means of similar linear mixed models. Once again, we constructed a base model with only random intercepts for Participants and Items (model0). This was then extended in model1 with the same predictor as before, comparing stress-matching to L1-follows and stress-matching to L1-precedes. We found significant effects in the expected directions for both contrasts. This indicates that when the speakers were incorrectly stressing the native language competitor syllable (e.g., proFEsor), their gestures showed temporal attraction towards the correct foreign language target syllable (e.g., falling late).

# only trials with gesture where stressed is incorrectly produced on L1 target
Dsub_l2attr <- rbind(DsubA,DsubB,Dsub3)
Dsub_l2attr$stressdiff_cat_iMatch <- relevel(as.factor(Dsub_l2attr$stressdiff_cat),"L1_L2 match")

#basemodel predicting the overall asynchrony
model0 <- lme(asynchrony~1, data = Dsub_l2attr, random = list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)

model1 <- lme(asynchrony~stressdiff_cat_iMatch, data = Dsub_l2attr, random =  list(~1|ppn, ~1|target), method = "ML", na.action = na.exclude)
print(anova(model0, model1)) #test difference basemodel versus model 1
##        Model df      AIC      BIC    logLik   Test  L.Ratio p-value
## model0     1  4 13209.15 13228.88 -6600.577                        
## model1     2  6 13062.95 13092.54 -6525.476 1 vs 2 150.2015  <.0001
print(summary(model1)) 
## Linear mixed-effects model fit by maximum likelihood
##   Data: Dsub_l2attr 
##        AIC      BIC    logLik
##   13062.95 13092.54 -6525.476
## 
## Random effects:
##  Formula: ~1 | ppn
##         (Intercept)
## StdDev:    80.51458
## 
##  Formula: ~1 | target %in% ppn
##         (Intercept) Residual
## StdDev:    128.2123 50.64848
## 
## Fixed effects:  asynchrony ~ stressdiff_cat_iMatch 
##                                       Value Std.Error  DF    t-value p-value
## (Intercept)                       -47.80780  16.67991 995  -2.866191  0.0042
## stressdiff_cat_iMatchL1 follows  -119.02719  10.85972 995 -10.960431  0.0000
## stressdiff_cat_iMatchL1 precedes   71.58506  15.40126 995   4.647999  0.0000
##  Correlation: 
##                                  (Intr) s__MLf
## stressdiff_cat_iMatchL1 follows  -0.157       
## stressdiff_cat_iMatchL1 precedes -0.103  0.156
## 
## Standardized Within-Group Residuals:
##          Min           Q1          Med           Q3          Max 
## -1.620263024 -0.208206646  0.008958533  0.195505935  1.546757562 
## 
## Number of Observations: 1023
## Number of Groups: 
##             ppn target %in% ppn 
##              26            1023

Exploratory analyses

ExplRQ.1: language proficiency

As an exploratory analysis, we asked whether the ‘kinematic accent’ would be modulated by the speakers’ foreign language proficiency. We argued that perhaps more proficient learners would show less of such a kinematic accent, while less proficient learners would be more likely to show such an accent. We quantified the kinematic accent by subtracting the directional gesture-speech (a)synchrony in stress-matching trials from that in stress-mismatching trials (variable name: L1attr). As a result, a more negative effect size would show greater native language attraction (kinematic accent). Consequently, we would predict a positive correlation between the native language attraction and foreign language proficiency: the higher the proficiency, the less negative/more positive the native language attraction value.

We collected self-reported proficiency data according to the Common European Framework of Reference (CEFR), with six levels (A1 < A2 < B1 < B2 < C1 < C2). These were converted into a numeric variable from 1-6 (variable name: proflevel).

bypps.async.L2L1 <- as.data.frame(tapply(Dsub$asynchrony_L2L1, list(Dsub$ppn,Dsub$stress), mean))
bypps.async.L2L1$ppn <- 1:26
bypps.async.L2L1$L1attr <- bypps.async.L2L1$`stress-mismatching` - bypps.async.L2L1$`stress-matching`


proficiency <- read.csv(paste0(dirname(curfolder), "/ProcessedData/ProficiencyData/proficiency_numeric.csv"),sep=";")
proficiency$ppn <- proficiency$Participant.nummer
proficiency$proflevel <- proficiency$L2.Spaans
proficiency <- proficiency[,c("ppn","proflevel")]
proficiency[is.na(proficiency$proflevel),]$proflevel <- 1

# greater negative values (-100) indicates more L1 attraction, less negative values or even positive values indicate less L1 attraction
indivdiffs <- merge(bypps.async.L2L1,proficiency,by="ppn")
print(indivdiffs[order(indivdiffs$L1attr),])
##    ppn stress-matching stress-mismatching      L1attr proflevel
## 16  16        73.31579          -77.28000 -150.595789         2
## 23  23        12.80000         -109.39130 -122.191304         4
## 20  20       -35.93103         -142.80000 -106.868966         2
## 13  13        11.42857          -87.69231  -99.120879         1
## 14  14        40.23529          -57.78947  -98.024768         1
## 15  15        18.96552          -56.00000  -74.965517         2
## 3    3        -0.80000          -71.92308  -71.123077         1
## 11  11        37.61290          -23.03704  -60.649940         1
## 18  18        25.66667          -33.40000  -59.066667         2
## 7    7      -215.76471         -269.76471  -54.000000         1
## 10  10       -96.63158         -150.40000  -53.768421         1
## 9    9        11.84615          -38.00000  -49.846154         4
## 2    2         3.04000          -42.80000  -45.840000         1
## 8    8       -74.57143         -102.13333  -27.561905         1
## 4    4       -61.72727          -85.53846  -23.811189         1
## 1    1       -26.35294          -49.40000  -23.047059         1
## 21  21       -70.17391          -91.90000  -21.726087         3
## 5    5       -48.42424          -68.38462  -19.960373         1
## 25  25      -115.83333         -132.18182  -16.348485         3
## 26  26       -50.28571          -58.40000   -8.114286         2
## 24  24       -20.24000          -22.42857   -2.188571         2
## 19  19       -70.28571          -63.63636    6.649351         2
## 17  17       -35.54286          -19.33333   16.209524         1
## 12  12      -169.10000          -81.82609   87.273913         1
## 22  22      -181.80000          -50.58333  131.216667         1
## 6    6      -231.24324          -88.00000  143.243243         2

We calculated the Pearson’s correlation between native language attraction (L1attr) and foreign language proficiency (proflevel). However, no significant correlation was observed. This is likely due to the lack of sufficient variability in our sample, since most participants reported to be beginning learners of Spanish (A1).

cor.test(indivdiffs$L1attr,indivdiffs$proflevel)
## 
##  Pearson's product-moment correlation
## 
## data:  indivdiffs$L1attr and indivdiffs$proflevel
## t = -0.88799, df = 24, p-value = 0.3834
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  -0.5291497  0.2245076
## sample estimates:
##        cor 
## -0.1783546
## `geom_smooth()` using formula = 'y ~ x'