In this script, we will work with the audio files. We will extract the following features:
intensity
f0
spectral centroid / spectral center of gravity
Code to load packages and prepare the environment
# packagesimport osimport globimport numpy as npimport pandas as pdimport matplotlib.pyplot as pltimport scipyfrom scipy.signal import butter, filtfilt, sosfiltimport librosaimport parselmouthimport IPython.display as ipdimport seaborn as snsimport pickleimport randomcurfolder = os.getcwd()datasetfolder = os.path.join(curfolder, '..', '..')# BalanceCorpusaudiofolder = os.path.join(datasetfolder, 'audios', '*') # last level for participant folderwavtotrack = glob.glob(os.path.join(audiofolder, "*.wav"))balancemeta = pd.read_csv(os.path.join(datasetfolder, 'metadata.csv'))balancedemo = pd.read_csv(os.path.join(datasetfolder, 'demographics.csv'))print('Number of audio files found: ', len(wavtotrack))# Create folder to save processed acoustic time seriesACfolder_processed = os.path.join(curfolder, '..', '..', 'TS_acoustics')ifnot os.path.exists(ACfolder_processed): os.makedirs(ACfolder_processed)
Number of audio files found: 80
Here is an audio example
And here it is visualized as a waveform
#TODO: add spectogram
Custom functions
def chunk_and_smooth(df, var, window=25, order=3):""" Processes a DataFrame by identifying contiguous non-NaN segments (chunks) in a specified column, then applies Savitzky-Golay smoothing to each sufficiently long chunk. Parameters: ----------- df : pandas.DataFrame Input DataFrame containing the data to be processed var : str Name of the column to process window : int, optional Window length for Savitzky-Golay filter (default: 25) order : int, optional Polynomial order for Savitzky-Golay filter (default: 3) Returns: -------- pandas.DataFrame Processed DataFrame with smoothed values in the specified column Notes: ------ - Chunks are defined as contiguous non-NaN values in the specified column - Only chunks with length >= 5 are smoothed - The original DataFrame is not modified; a new DataFrame is returned """ df['chunk'] =None chunk =0for index, row in df.iterrows():if np.isnan(row[var]):continueelse: df.loc[index, 'chunk'] = chunk# if the next value is NaN or this is the last row, increase the chunkif index ==len(df)-1:continueelif np.isnan(df.loc[index+1, var]): chunk +=1# now we can smooth the spectralCent values in each chunk chunks = df['chunk'].unique()# skip if chunks are empty (that means that there is no var trace)iflen(chunks) >1:# ignore the first chunk (None) chunks = chunks[1:]for chunk in chunks:# get the rows of the chunk chunkrows = df[df['chunk'] == chunk].copy()# dont smooth chunks shorter than 5iflen(chunkrows) <5:continueelse:# smooth var with savgol filter chunkrows[var] = scipy.signal.savgol_filter(chunkrows[var], window, order) # put it back to the df df.loc[df['chunk'] == chunk, var] = chunkrows[var]# get rid of the chunk column df = df.drop('chunk', axis=1)return df
Extracting intensity (vocalic energy)
To extract the amplitude envelope of the acoustic signal, we follow a method by @tilsen_arvaniti13, adapted by @pouw24. We use a bandpass and 2nd order 10Hz low-pass zero-phase Butterworth filter.
Code with functions to extract the amplitude envelope
def butter_bandpass(lowcut, highcut, fs, order=2):"""Design a Butterworth bandpass filter. Args: lowcut: Lower frequency bound of the bandpass filter (Hz) highcut: Upper frequency bound of the bandpass filter (Hz) fs: Sampling frequency (Hz) order: Order of the filter (default: 2) Returns: b: Numerator coefficients of the filter a: Denominator coefficients of the filter """ nyquist =0.5* fs low = lowcut / nyquist high = highcut / nyquist b, a = butter(order, [low, high], btype='band')return b, adef butter_bandpass_filtfilt(data, lowcut, highcut, fs, order=2):"""Apply a Butterworth bandpass filter to the input data using filtfilt. Args: data: Input signal to be filtered lowcut: Lower frequency bound of the bandpass filter (Hz) highcut: Upper frequency bound of the bandpass filter (Hz) fs: Sampling frequency (Hz) order: Order of the filter (default: 2) Returns: y: Filtered signal """ b, a = butter_bandpass(lowcut, highcut, fs, order=order) y = filtfilt(b, a, data)return y# Define the lowpass filterdef butter_lowpass(cutoff, fs, order=2):"""Design a Butterworth lowpass filter. Args: cutoff: Cutoff frequency of the lowpass filter (Hz) fs: Sampling frequency (Hz) order: Order of the filter (default: 2) Returns: b: Numerator coefficients of the filter a: Denominator coefficients of the filter """ nyquist =0.5* fs normal_cutoff = cutoff / nyquist b, a = butter(order, normal_cutoff, btype='low')return b, adef butter_lowpass_filtfilt(data, cutoff, fs, order=2):"""Apply a Butterworth lowpass filter to the input data using filtfilt. Args: data: Input signal to be filtered cutoff: Cutoff frequency of the lowpass filter (Hz) fs: Sampling frequency (Hz) order: Order of the filter (default: 2) Returns: y: Filtered signal """ b, a = butter_lowpass(cutoff, fs, order=order) y = filtfilt(b, a, data)return y# Function to extract amplitude envelopedef amp_envelope(audiofilename):"""Extract the amplitude envelope from an audio file. Args: audiofilename: Path to the audio file Returns: data: The amplitude envelope of the filtered signal sr: Sampling rate of the audio file """# load audio with librosa audio, sr = librosa.load(audiofilename, sr=None, mono=True)# Bandpass filter 400-4000Hz data = butter_bandpass_filtfilt(audio, 400, 4000, sr, order=2)# Lowpass filter 10Hz data = butter_lowpass_filtfilt(np.abs(data), 10, sr, order=2)return data, sr
Here is an example how the vocalic energy is extracted
Now we loop over all the audio files and extract the vocalic energy.
Note that we updated the pipeline such that we first extract envelope for all trials and sessions and only later normalize within participant and extract envelope change. Before, we normalized within a trial which was an unintended mistake.
env_error = []desired_sr =0.5# Loop over wav filesfor audiofile in wavtotrack: basename = os.path.basename(audiofile)print('working on '+ basename)# apply the function ampv, sr = amp_envelope(audiofile)# Extract and plot the original signal rawaudio, sr = librosa.load(audiofile, sr=None)# create a time vector time_env = np.arange(0, len(rawaudio)/sr, 1/sr)# Ensure the lengths match by padding ampv if necessary (Note that is a quick fix)iflen(ampv) <len(time_env): ampv = np.pad(ampv, (0, len(time_env) -len(ampv)), mode='constant')eliflen(ampv) >len(time_env): ampv = ampv[:len(time_env)]# the same for rawaudioiflen(rawaudio) <len(time_env): rawaudio = np.pad(rawaudio, (0, len(time_env) -len(rawaudio)), mode='constant')eliflen(rawaudio) >len(time_env): rawaudio = rawaudio[:len(time_env)]# save the audio and envelopetry: audio = pd.DataFrame({'time': time_env, 'audio': rawaudio, 'envelope': ampv, 'filename': basename})# convert time to ms audio['time'] = audio['time'] *1000# downsample to 500 Hz time_new = np.arange(0, max(audio['time']), 1/desired_sr) env_down = pd.DataFrame({'time': time_new}) colstoint = audio.columns colstoint = [x for x in colstoint if'time'notin x] colstoint = [x for x in colstoint if'filename'notin x]for col in colstoint: env_down[col] = np.interp(env_down['time'], audio['time'], audio[col]) env_down['filename'] = basename# write as csv env_down.to_csv(os.path.join(ACfolder_processed, 'env_'+ basename +'_nenorm.csv'), index=False)exceptValueError:print('ValueError: '+ basename) env_error.append(basename)continue#break
working on 103_203_12_1_20250113_152455_doughnut_board.wav
working on 103_203_13_1_20250113_152513_spinach_board.wav
working on 103_203_14_1_20250113_152536_balloon_board.wav
working on 103_203_15_1_20250113_152557_bacon_board.wav
working on 103_203_16_1_20250113_152613_chlorine_board.wav
working on 103_203_17_1_20250113_152643_leather_board.wav
working on 103_203_18_1_20250113_152657_pillow_board.wav
working on 103_203_19_1_20250113_152712_traffic_board.wav
working on 103_203_20_1_20250113_152735_bitter_board.wav
working on 103_203_21_1_20250113_152824_tiger_board.wav
working on 103_203_23_1_20250113_152946_trumpet_ground.wav
working on 103_203_24_1_20250113_152957_pasta_ground.wav
working on 103_203_25_1_20250113_153028_hamster_ground.wav
working on 103_203_26_1_20250113_153140_siren_ground.wav
working on 103_203_27_1_20250113_153217_jungle_ground.wav
working on 103_203_28_1_20250113_153235_birthday_ground.wav
working on 103_203_29_1_20250113_153255_cherry_ground.wav
working on 103_203_30_1_20250113_153310_exam_ground.wav
working on 103_203_31_1_20250113_153333_flower_ground.wav
working on 103_203_32_1_20250113_153341_coffee_ground.wav
working on 103_203_34_1_20250113_153427_picture_ground.wav
working on 103_203_35_1_20250113_153443_yellow_ground.wav
working on 103_203_36_1_20250113_153454_flannel_ground.wav
working on 103_203_37_1_20250113_153517_movie_ground.wav
working on 103_203_38_1_20250113_153528_puppy_ground.wav
working on 103_203_39_1_20250113_153540_muscle_ground.wav
working on 103_203_40_1_20250113_153555_beauty_ground.wav
working on 103_203_41_1_20250113_153604_sugar_ground.wav
working on 103_203_42_1_20250113_153614_sushi_ground.wav
working on 103_203_43_1_20250113_153626_gravy_ground.wav
working on 103_203_45_1_20250113_153745_walnut_board.wav
working on 103_203_46_1_20250113_153759_blanket_board.wav
working on 103_203_47_1_20250113_153811_soda_board.wav
working on 103_203_48_1_20250113_153828_hunger_board.wav
working on 103_203_49_1_20250113_153839_christmas_board.wav
working on 103_203_50_1_20250113_153852_butter_board.wav
working on 103_203_51_1_20250113_153926_cologne_board.wav
working on 103_203_52_1_20250113_154037_zipper_board.wav
working on 103_203_53_1_20250113_154055_river_board.wav
working on 103_203_54_1_20250113_154107_music_board.wav
working on 108_208_12_1_20250114_162456_flower_ground.wav
working on 108_208_13_1_20250114_162512_traffic_ground.wav
working on 108_208_14_1_20250114_162535_zipper_ground.wav
working on 108_208_15_1_20250114_162542_gravy_ground.wav
working on 108_208_16_1_20250114_162559_siren_ground.wav
working on 108_208_17_1_20250114_162610_cherry_ground.wav
working on 108_208_18_1_20250114_162618_coffee_ground.wav
working on 108_208_19_1_20250114_162700_cologne_ground.wav
working on 108_208_20_1_20250114_162723_bitter_ground.wav
working on 108_208_21_1_20250114_162731_sugar_ground.wav
working on 108_208_23_1_20250114_162817_tiger_ground.wav
working on 108_208_24_1_20250114_162831_exam_ground.wav
working on 108_208_25_1_20250114_162844_puppy_ground.wav
working on 108_208_26_1_20250114_162904_hamster_ground.wav
working on 108_208_27_1_20250114_162936_bacon_ground.wav
working on 108_208_28_1_20250114_162956_river_ground.wav
working on 108_208_29_1_20250114_163011_walnut_ground.wav
working on 108_208_30_1_20250114_163024_butter_ground.wav
working on 108_208_31_1_20250114_163033_leather_ground.wav
working on 108_208_32_1_20250114_163057_chlorine_ground.wav
working on 108_208_34_1_20250114_163148_movie_board.wav
working on 108_208_35_1_20250114_163156_pasta_board.wav
working on 108_208_36_1_20250114_163248_balloon_board.wav
working on 108_208_37_1_20250114_163259_muscle_board.wav
working on 108_208_38_1_20250114_163310_birthday_board.wav
working on 108_208_39_1_20250114_163319_beauty_board.wav
working on 108_208_40_1_20250114_163328_christmas_board.wav
working on 108_208_41_1_20250114_163344_trumpet_board.wav
working on 108_208_42_1_20250114_163352_music_board.wav
working on 108_208_43_1_20250114_163408_hunger_board.wav
working on 108_208_45_1_20250114_163506_sushi_board.wav
working on 108_208_46_1_20250114_163523_jungle_board.wav
working on 108_208_47_1_20250114_163536_yellow_board.wav
working on 108_208_48_1_20250114_163634_flannel_board.wav
working on 108_208_49_1_20250114_163641_doughnut_board.wav
working on 108_208_50_1_20250114_163655_picture_board.wav
working on 108_208_51_1_20250114_163806_spinach_board.wav
working on 108_208_52_1_20250114_163842_blanket_board.wav
working on 108_208_53_1_20250114_163852_pillow_board.wav
working on 108_208_54_1_20250114_163903_soda_board.wav
Now we will collect envelope min and max in order to min-max normalize within participant such that all values are between 0 and 1.
if os.path.exists(os.path.join(ACfolder_processed, 'env_norm_dict.pkl')):withopen(os.path.join(ACfolder_processed, 'env_norm_dict.pkl'), 'rb') as f: env_norm_dict = pickle.load(f)else: env_error = [] env_norm_dict = {}# Loop over wav filesfor audiofile in wavtotrack: basename = os.path.basename(audiofile)print('working on '+ basename)# get the envelope values df = pd.read_csv(os.path.join(ACfolder_processed, 'env_'+ basename +'_nenorm.csv')) ampv = df['envelope'].values# calculate mean, sd, min and max mean_env = np.mean(ampv) sd_env = np.std(ampv) min_env = np.min(ampv) max_env = np.max(ampv)# store in dict env_norm_dict[basename] = {'mean': mean_env,'sd': sd_env,'min': min_env,'max': max_env}# save the dictwithopen(os.path.join(ACfolder_processed, 'env_norm_dict.pkl'), 'wb') as f: pickle.dump(env_norm_dict, f)
# reorganize the dictionary to get min-max per session & participantdf_norm = pd.DataFrame.from_dict(env_norm_dict, orient='index')df_norm['filename'] = df_norm.indexparts = df_norm['filename'].str.split('.').str[0].str.split('_', expand=True)df_norm['pair_id'] = parts[0] +'_'+ parts[1]df_norm['condition'] = parts[7] df_norm['target'] = parts[6]# find row in balancemeta with matching pair_id, condition, and targetfor idx, row in df_norm.iterrows(): matching_row = balancemeta[ (balancemeta['pair_id'] == row['pair_id']) & (balancemeta['clue_giver_condition'] == row['condition']) & (balancemeta['target_word'] == row['target']) ]ifnot matching_row.empty: cluegiver_id = matching_row['clue_giver_id'].values[0]# add to the current row df_norm.at[idx, 'clue_giver_id'] = cluegiver_id# now get min and max per session_participantgroup_minmax = ( df_norm .groupby('clue_giver_id') .agg( group_min=('min', 'min'), group_max=('max', 'max') ) .reset_index())group_minmax.head()
clue_giver_id
group_min
group_max
0
103.0
-0.000256
0.009786
1
108.0
-0.000315
0.015145
2
203.0
-0.001902
0.052426
3
208.0
-0.000818
0.038379
Now we normalize by min-max
envtotrack = glob.glob(os.path.join(ACfolder_processed, "env_*_nenorm.csv"))sr =500# Adapt if necessary# loop, normalize by min-maxforfilein envtotrack:print('Normalizing '+file) df = pd.read_csv(file) filename = df['filename'].values[0] parts = filename.split('.')[0].split('_') pair_id = parts[0] +'_'+ parts[1] condition = parts[7] target = parts[6]# find matching row from balancemeta matchingrow = balancemeta[ (balancemeta['pair_id'] == pair_id) & (balancemeta['clue_giver_condition'] == condition) & (balancemeta['target_word'] == target) ] cluegiver = matchingrow['clue_giver_id'].values[0]# get the group min and max group_min = group_minmax[group_minmax['clue_giver_id'] == cluegiver]['group_min'].values[0] group_max = group_minmax[group_minmax['clue_giver_id'] == cluegiver]['group_max'].values[0]# normalize the envelope df['envelope_norm'] = (df['envelope'] - group_min) / (group_max - group_min)# perform also envelope change df['envelope_change'] = np.insert(np.diff(df['envelope']), 0, 0)# smooth df['envelope_change'] = butter_lowpass_filtfilt(np.abs(df['envelope_change']), 10, sr, order=2)# multiply by sr to get in units in s (not per frame) df['envelope_change'] = df['envelope_change'] * sr# save the normalized envelope df.to_csv(os.path.join(ACfolder_processed, 'env_'+ filename +'_norm.csv'), index=False)# plot both envelopes for checking plt.figure(figsize=(10, 5)) plt.plot(df['time'], df['envelope'], label='Original Envelope') plt.plot(df['time'], df['envelope_norm'], label='Normalized Envelope') plt.title(f'Envelope Normalization Check for {filename}') plt.xlabel('Time (ms)') plt.ylabel('Amplitude') plt.legend() plt.show()
Now we extract pitch using the parselmouth library [@jadoul_etal18].
Code with function to extract the fundamental frequency
def extract_f0(locationsound): # Extract fundamental frequency from audio file""" Extracts the fundamental frequency (F0) from an audio file. Parameters: - locationsound (str): Path to the audio file - sex (str): 'vrouw' for female, other for male (determines F0 range) Returns: - tuple: (Sound object, numpy array of F0 values) """# read the sound file as numpy array audio, sr = librosa.load(locationsound, sr=48000)# read the sound file into Python snd = parselmouth.Sound(audio, sampling_frequency=sr) f0min =80 f0max =335# based on the Balance paper# Note that we could also set different min-max for female & male, given that we are not,# it is recommended (if not necessary) to have speaker-specific intercept in modelling pitch = snd.to_pitch(time_step =0.002, pitch_floor=f0min, pitch_ceiling=f0max) # time_step to get 500Hz f0_values = pitch.selected_array['frequency']return snd, f0_values
Now we loop over all audio files and extract f0 from each. Resulting f0 contours were smoothed with a Savitzky-Golay 3rd-polynomial filter with a span of 50 ms [following @fuchsetal2016] applied to continuous runs of phonated vocalization to maintain discontinuities typical of the f0 signal.
freq=44100f0_error = []# Loop over wav filesfor audiofile in wavtotrack: basename = os.path.basename(audiofile)print('working on '+ basename)# apply the function snd, f0 = extract_f0(audiofile) length =len(f0)# replace 0 values with NaN f0 = np.where(f0 ==0, np.nan, f0)# create time vector F0_time = np.linspace(0, snd.duration, len(f0)) *1000# Generate time vector# create df f0_df = pd.DataFrame({'time_ms': F0_time, 'f0': f0, 'ID': basename})# Smooth the f0 valuestry: f0_df = chunk_and_smooth(f0_df, 'f0') # do it with window 25exceptValueError:try:# unless there is only tiny chunk of f0 and then we need window of 5print('ValueError: '+ basename +', f0 trace is smaller than window length, resuming to window=5') f0_df = chunk_and_smooth(f0_df, 'f0', window=5)exceptValueError:print('ValueError: '+ basename +', even with window=5, skipping f0 extraction') f0_error.append(basename)continue# write as csv f0_df.to_csv(os.path.join(ACfolder_processed, 'f0_'+ basename +'.csv'), index=False)
working on 103_203_12_1_20250113_152455_doughnut_board.wav
ValueError: 103_203_12_1_20250113_152455_doughnut_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_13_1_20250113_152513_spinach_board.wav
ValueError: 103_203_13_1_20250113_152513_spinach_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_14_1_20250113_152536_balloon_board.wav
ValueError: 103_203_14_1_20250113_152536_balloon_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_15_1_20250113_152557_bacon_board.wav
ValueError: 103_203_15_1_20250113_152557_bacon_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_16_1_20250113_152613_chlorine_board.wav
ValueError: 103_203_16_1_20250113_152613_chlorine_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_17_1_20250113_152643_leather_board.wav
ValueError: 103_203_17_1_20250113_152643_leather_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_18_1_20250113_152657_pillow_board.wav
ValueError: 103_203_18_1_20250113_152657_pillow_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_19_1_20250113_152712_traffic_board.wav
ValueError: 103_203_19_1_20250113_152712_traffic_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_20_1_20250113_152735_bitter_board.wav
ValueError: 103_203_20_1_20250113_152735_bitter_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_21_1_20250113_152824_tiger_board.wav
ValueError: 103_203_21_1_20250113_152824_tiger_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_23_1_20250113_152946_trumpet_ground.wav
ValueError: 103_203_23_1_20250113_152946_trumpet_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_24_1_20250113_152957_pasta_ground.wav
ValueError: 103_203_24_1_20250113_152957_pasta_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_25_1_20250113_153028_hamster_ground.wav
ValueError: 103_203_25_1_20250113_153028_hamster_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_26_1_20250113_153140_siren_ground.wav
ValueError: 103_203_26_1_20250113_153140_siren_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_27_1_20250113_153217_jungle_ground.wav
ValueError: 103_203_27_1_20250113_153217_jungle_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_28_1_20250113_153235_birthday_ground.wav
working on 103_203_29_1_20250113_153255_cherry_ground.wav
ValueError: 103_203_29_1_20250113_153255_cherry_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_30_1_20250113_153310_exam_ground.wav
ValueError: 103_203_30_1_20250113_153310_exam_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_31_1_20250113_153333_flower_ground.wav
ValueError: 103_203_31_1_20250113_153333_flower_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_32_1_20250113_153341_coffee_ground.wav
ValueError: 103_203_32_1_20250113_153341_coffee_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_34_1_20250113_153427_picture_ground.wav
ValueError: 103_203_34_1_20250113_153427_picture_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_35_1_20250113_153443_yellow_ground.wav
ValueError: 103_203_35_1_20250113_153443_yellow_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_36_1_20250113_153454_flannel_ground.wav
working on 103_203_37_1_20250113_153517_movie_ground.wav
ValueError: 103_203_37_1_20250113_153517_movie_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_38_1_20250113_153528_puppy_ground.wav
ValueError: 103_203_38_1_20250113_153528_puppy_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_39_1_20250113_153540_muscle_ground.wav
ValueError: 103_203_39_1_20250113_153540_muscle_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_40_1_20250113_153555_beauty_ground.wav
ValueError: 103_203_40_1_20250113_153555_beauty_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_41_1_20250113_153604_sugar_ground.wav
ValueError: 103_203_41_1_20250113_153604_sugar_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_42_1_20250113_153614_sushi_ground.wav
ValueError: 103_203_42_1_20250113_153614_sushi_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_43_1_20250113_153626_gravy_ground.wav
working on 103_203_45_1_20250113_153745_walnut_board.wav
ValueError: 103_203_45_1_20250113_153745_walnut_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_46_1_20250113_153759_blanket_board.wav
ValueError: 103_203_46_1_20250113_153759_blanket_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_47_1_20250113_153811_soda_board.wav
ValueError: 103_203_47_1_20250113_153811_soda_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_48_1_20250113_153828_hunger_board.wav
ValueError: 103_203_48_1_20250113_153828_hunger_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_49_1_20250113_153839_christmas_board.wav
ValueError: 103_203_49_1_20250113_153839_christmas_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_50_1_20250113_153852_butter_board.wav
ValueError: 103_203_50_1_20250113_153852_butter_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_51_1_20250113_153926_cologne_board.wav
ValueError: 103_203_51_1_20250113_153926_cologne_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_52_1_20250113_154037_zipper_board.wav
ValueError: 103_203_52_1_20250113_154037_zipper_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 103_203_53_1_20250113_154055_river_board.wav
working on 103_203_54_1_20250113_154107_music_board.wav
ValueError: 103_203_54_1_20250113_154107_music_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_12_1_20250114_162456_flower_ground.wav
ValueError: 108_208_12_1_20250114_162456_flower_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_13_1_20250114_162512_traffic_ground.wav
ValueError: 108_208_13_1_20250114_162512_traffic_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_14_1_20250114_162535_zipper_ground.wav
ValueError: 108_208_14_1_20250114_162535_zipper_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_15_1_20250114_162542_gravy_ground.wav
ValueError: 108_208_15_1_20250114_162542_gravy_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_16_1_20250114_162559_siren_ground.wav
ValueError: 108_208_16_1_20250114_162559_siren_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_17_1_20250114_162610_cherry_ground.wav
ValueError: 108_208_17_1_20250114_162610_cherry_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_18_1_20250114_162618_coffee_ground.wav
ValueError: 108_208_18_1_20250114_162618_coffee_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_19_1_20250114_162700_cologne_ground.wav
ValueError: 108_208_19_1_20250114_162700_cologne_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_20_1_20250114_162723_bitter_ground.wav
ValueError: 108_208_20_1_20250114_162723_bitter_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_21_1_20250114_162731_sugar_ground.wav
ValueError: 108_208_21_1_20250114_162731_sugar_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_23_1_20250114_162817_tiger_ground.wav
ValueError: 108_208_23_1_20250114_162817_tiger_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_24_1_20250114_162831_exam_ground.wav
ValueError: 108_208_24_1_20250114_162831_exam_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_25_1_20250114_162844_puppy_ground.wav
ValueError: 108_208_25_1_20250114_162844_puppy_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_26_1_20250114_162904_hamster_ground.wav
ValueError: 108_208_26_1_20250114_162904_hamster_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_27_1_20250114_162936_bacon_ground.wav
ValueError: 108_208_27_1_20250114_162936_bacon_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_28_1_20250114_162956_river_ground.wav
ValueError: 108_208_28_1_20250114_162956_river_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_29_1_20250114_163011_walnut_ground.wav
ValueError: 108_208_29_1_20250114_163011_walnut_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_30_1_20250114_163024_butter_ground.wav
ValueError: 108_208_30_1_20250114_163024_butter_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_31_1_20250114_163033_leather_ground.wav
ValueError: 108_208_31_1_20250114_163033_leather_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_32_1_20250114_163057_chlorine_ground.wav
ValueError: 108_208_32_1_20250114_163057_chlorine_ground.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_34_1_20250114_163148_movie_board.wav
ValueError: 108_208_34_1_20250114_163148_movie_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_35_1_20250114_163156_pasta_board.wav
ValueError: 108_208_35_1_20250114_163156_pasta_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_36_1_20250114_163248_balloon_board.wav
ValueError: 108_208_36_1_20250114_163248_balloon_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_37_1_20250114_163259_muscle_board.wav
ValueError: 108_208_37_1_20250114_163259_muscle_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_38_1_20250114_163310_birthday_board.wav
ValueError: 108_208_38_1_20250114_163310_birthday_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_39_1_20250114_163319_beauty_board.wav
ValueError: 108_208_39_1_20250114_163319_beauty_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_40_1_20250114_163328_christmas_board.wav
ValueError: 108_208_40_1_20250114_163328_christmas_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_41_1_20250114_163344_trumpet_board.wav
ValueError: 108_208_41_1_20250114_163344_trumpet_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_42_1_20250114_163352_music_board.wav
ValueError: 108_208_42_1_20250114_163352_music_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_43_1_20250114_163408_hunger_board.wav
ValueError: 108_208_43_1_20250114_163408_hunger_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_45_1_20250114_163506_sushi_board.wav
ValueError: 108_208_45_1_20250114_163506_sushi_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_46_1_20250114_163523_jungle_board.wav
ValueError: 108_208_46_1_20250114_163523_jungle_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_47_1_20250114_163536_yellow_board.wav
working on 108_208_48_1_20250114_163634_flannel_board.wav
ValueError: 108_208_48_1_20250114_163634_flannel_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_49_1_20250114_163641_doughnut_board.wav
ValueError: 108_208_49_1_20250114_163641_doughnut_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_50_1_20250114_163655_picture_board.wav
ValueError: 108_208_50_1_20250114_163655_picture_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_51_1_20250114_163806_spinach_board.wav
ValueError: 108_208_51_1_20250114_163806_spinach_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_52_1_20250114_163842_blanket_board.wav
working on 108_208_53_1_20250114_163852_pillow_board.wav
ValueError: 108_208_53_1_20250114_163852_pillow_board.wav, f0 trace is smaller than window length, resuming to window=5
working on 108_208_54_1_20250114_163903_soda_board.wav
Here is an example of a file
time_ms
f0
ID
0
0.000000
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
1
2.004487
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
2
4.008973
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
3
6.013460
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
4
8.017946
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
5
10.022433
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
6
12.026919
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
7
14.031406
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
8
16.035892
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
9
18.040379
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
10
20.044866
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
11
22.049352
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
12
24.053839
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
13
26.058325
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
14
28.062812
NaN
103_203_12_1_20250113_152455_doughnut_board.wav
And here visualized
Extracting spectral centroid
To extract the spectral center of gravity (CoG), we first compute a cepstral envelope to smooth the magnitude spectrum while preserving its overall shape. The envelope is obtained via low-time liftering in the cepstral domain (keeping only the lowest frequencies), which suppresses fine harmonic structure without distorting the spectral contour. The CoG is then computed as the power-weighted mean frequency of this envelope — equivalent to a spectral centroid, but applied to the smoothed rather than raw spectrum.
Functions to compute cepstral envelope and center of gravity
# function to compute cepstral envelopedef cepstral_envelope(mag, lifter_cutoff=30):"""Compute cepstral envelope from magnitude spectrum. Args: mag: Magnitude spectrum (positive frequencies only) lifter_cutoff: Number of low quefrencies to keep (default: 30) Returns: envelope: Smoothed magnitude spectrum (positive frequencies only) """# Avoid log(0) mag_safe = np.maximum(mag, 1e-12)# Log-magnitude spectrum log_mag = np.log(mag_safe)# We assume `mag` comes from an rFFT of length N_fft = 2*(N_bins-1) N_bins = mag_safe.shape[0] N_fft =2* (N_bins -1)# Real cepstrum cepstrum = np.fft.irfft(log_mag, n=N_fft)# Low-time liftering: keep only low quefrencies liftered = np.zeros_like(cepstrum) liftered[:lifter_cutoff] = cepstrum[:lifter_cutoff]# Back to (smoothed) log spectrum (positive freqs only) smooth_log_mag = np.fft.rfft(liftered)# Envelope magnitude envelope = np.exp(np.real(smooth_log_mag))# Safety: ensure same length envelope = envelope[:N_bins]return envelope# function to compute center of gravitydef cog_from_envelope(freqs, envelope):"""Compute center of gravity from frequency envelope. Args: freqs: Frequency bins (positive frequencies only) envelope: Smoothed magnitude spectrum (positive frequencies only) Returns: cog: Center of gravity frequency (Hz) """ power_spec = envelope **2 total_power = np.sum(power_spec)if total_power <=0:return np.nanreturn np.sum(freqs * power_spec) / total_power
window_length =0.03# 30 ms analysis windowfor audiofile in wavtotrack: basename = os.path.basename(audiofile)print(f'Working on {basename}')# Load sound snd = parselmouth.Sound(audiofile)# Get sampling rate fs = snd.sampling_frequency filtered_sound = snd # use the original sound# Compute spectrogram spectrogram = filtered_sound.to_spectrogram(window_length=window_length)# Time points (s) times = spectrogram.xs() cog_values = []for t in times:# Spectrum slice at time t (parselmouth.Spectrum) spec_slice = spectrogram.to_spectrum_slice(time=t)# Frequencies (Hz) and magnitude spectrum freqs = spec_slice.xs() # shape (N_bins,) mag = np.abs(spec_slice.values[0, :]) # shape (N_bins,)# Smoothed spectral envelope (Option C) envelope = cepstral_envelope(mag, lifter_cutoff=30)# CoG of the envelope cog = cog_from_envelope(freqs, envelope) cog_values.append(cog)# Convert to numpy time_cog = np.array(times) *1000# ms cog_values = np.array(cog_values)# Create DataFrame cog_df = pd.DataFrame({'time': time_cog, 'CoG': cog_values, 'filename': basename})# Replace zeros with NaN (if any silent frames ended up as 0) cog_df['CoG'] = cog_df['CoG'].replace(0, np.nan)# Smooth the CoG tracetry: cog_df = chunk_and_smooth(cog_df, 'CoG')exceptValueError:print(f'ValueError: {basename}, CoG trace is smaller than window length, using window=5') cog_df = chunk_and_smooth(cog_df, 'CoG', window=5)# Save output_path = os.path.join(ACfolder_processed, f'cog_{basename}.csv') cog_df.to_csv(output_path, index=False)
Working on 103_203_12_1_20250113_152455_doughnut_board.wav
Working on 103_203_13_1_20250113_152513_spinach_board.wav
Working on 103_203_14_1_20250113_152536_balloon_board.wav
Working on 103_203_15_1_20250113_152557_bacon_board.wav
Working on 103_203_16_1_20250113_152613_chlorine_board.wav
Working on 103_203_17_1_20250113_152643_leather_board.wav
Working on 103_203_18_1_20250113_152657_pillow_board.wav
Working on 103_203_19_1_20250113_152712_traffic_board.wav
Working on 103_203_20_1_20250113_152735_bitter_board.wav
Working on 103_203_21_1_20250113_152824_tiger_board.wav
Working on 103_203_23_1_20250113_152946_trumpet_ground.wav
Working on 103_203_24_1_20250113_152957_pasta_ground.wav
Working on 103_203_25_1_20250113_153028_hamster_ground.wav
Working on 103_203_26_1_20250113_153140_siren_ground.wav
Working on 103_203_27_1_20250113_153217_jungle_ground.wav
Working on 103_203_28_1_20250113_153235_birthday_ground.wav
Working on 103_203_29_1_20250113_153255_cherry_ground.wav
Working on 103_203_30_1_20250113_153310_exam_ground.wav
Working on 103_203_31_1_20250113_153333_flower_ground.wav
Working on 103_203_32_1_20250113_153341_coffee_ground.wav
Working on 103_203_34_1_20250113_153427_picture_ground.wav
Working on 103_203_35_1_20250113_153443_yellow_ground.wav
Working on 103_203_36_1_20250113_153454_flannel_ground.wav
Working on 103_203_37_1_20250113_153517_movie_ground.wav
Working on 103_203_38_1_20250113_153528_puppy_ground.wav
Working on 103_203_39_1_20250113_153540_muscle_ground.wav
Working on 103_203_40_1_20250113_153555_beauty_ground.wav
Working on 103_203_41_1_20250113_153604_sugar_ground.wav
Working on 103_203_42_1_20250113_153614_sushi_ground.wav
Working on 103_203_43_1_20250113_153626_gravy_ground.wav
Working on 103_203_45_1_20250113_153745_walnut_board.wav
Working on 103_203_46_1_20250113_153759_blanket_board.wav
Working on 103_203_47_1_20250113_153811_soda_board.wav
Working on 103_203_48_1_20250113_153828_hunger_board.wav
Working on 103_203_49_1_20250113_153839_christmas_board.wav
Working on 103_203_50_1_20250113_153852_butter_board.wav
Working on 103_203_51_1_20250113_153926_cologne_board.wav
Working on 103_203_52_1_20250113_154037_zipper_board.wav
Working on 103_203_53_1_20250113_154055_river_board.wav
Working on 103_203_54_1_20250113_154107_music_board.wav
Working on 108_208_12_1_20250114_162456_flower_ground.wav
Working on 108_208_13_1_20250114_162512_traffic_ground.wav
Working on 108_208_14_1_20250114_162535_zipper_ground.wav
Working on 108_208_15_1_20250114_162542_gravy_ground.wav
Working on 108_208_16_1_20250114_162559_siren_ground.wav
Working on 108_208_17_1_20250114_162610_cherry_ground.wav
Working on 108_208_18_1_20250114_162618_coffee_ground.wav
Working on 108_208_19_1_20250114_162700_cologne_ground.wav
Working on 108_208_20_1_20250114_162723_bitter_ground.wav
Working on 108_208_21_1_20250114_162731_sugar_ground.wav
Working on 108_208_23_1_20250114_162817_tiger_ground.wav
Working on 108_208_24_1_20250114_162831_exam_ground.wav
Working on 108_208_25_1_20250114_162844_puppy_ground.wav
Working on 108_208_26_1_20250114_162904_hamster_ground.wav
Working on 108_208_27_1_20250114_162936_bacon_ground.wav
Working on 108_208_28_1_20250114_162956_river_ground.wav
Working on 108_208_29_1_20250114_163011_walnut_ground.wav
Working on 108_208_30_1_20250114_163024_butter_ground.wav
Working on 108_208_31_1_20250114_163033_leather_ground.wav
Working on 108_208_32_1_20250114_163057_chlorine_ground.wav
Working on 108_208_34_1_20250114_163148_movie_board.wav
Working on 108_208_35_1_20250114_163156_pasta_board.wav
Working on 108_208_36_1_20250114_163248_balloon_board.wav
Working on 108_208_37_1_20250114_163259_muscle_board.wav
Working on 108_208_38_1_20250114_163310_birthday_board.wav
Working on 108_208_39_1_20250114_163319_beauty_board.wav
Working on 108_208_40_1_20250114_163328_christmas_board.wav
Working on 108_208_41_1_20250114_163344_trumpet_board.wav
Working on 108_208_42_1_20250114_163352_music_board.wav
Working on 108_208_43_1_20250114_163408_hunger_board.wav
Working on 108_208_45_1_20250114_163506_sushi_board.wav
Working on 108_208_46_1_20250114_163523_jungle_board.wav
Working on 108_208_47_1_20250114_163536_yellow_board.wav
Working on 108_208_48_1_20250114_163634_flannel_board.wav
Working on 108_208_49_1_20250114_163641_doughnut_board.wav
Working on 108_208_50_1_20250114_163655_picture_board.wav
Working on 108_208_51_1_20250114_163806_spinach_board.wav
Working on 108_208_52_1_20250114_163842_blanket_board.wav
Working on 108_208_53_1_20250114_163852_pillow_board.wav
Working on 108_208_54_1_20250114_163903_soda_board.wav
