merged

segment_data.py

@@ -0,0 +1,11 @@
+import pandas as pd
+
+def fix_csv(collection_name = 'test'):
+    seg_data = pd.read_csv('./outputs/'+collection_name+'.csv',names=['phrase','filename'
+                ,'start_phoneme','end_phoneme','start_time','end_time'])
+    seg_data.to_csv('./outputs/'+collection_name+'.fixed.csv')
+
+
+def segment_data_gen(collection_name = 'test'):
+    # collection_name = 'test'
+    seg_data = pd.read_csv('./outputs/'+collection_name+'.fixed.csv',index_col=0)

segment_model.py

@@ -0,0 +1,108 @@
+from __future__ import absolute_import
+from __future__ import print_function
+import numpy as np
+from keras.models import Model,load_model,model_from_yaml
+from keras.layers import Input,Concatenate,Lambda, BatchNormalization, Dropout
+from keras.layers import Dense, LSTM, Bidirectional, GRU
+from keras.losses import categorical_crossentropy
+from keras.utils import to_categorical
+from keras.optimizers import RMSprop
+from keras.callbacks import TensorBoard, ModelCheckpoint
+from keras import backend as K
+from keras.utils import plot_model
+from speech_tools import create_dir,step_count
+from speech_data import segment_data_gen
+
+
+def accuracy(y_true, y_pred):
+    '''Compute classification accuracy with a fixed threshold on distances.
+    '''
+    return K.mean(K.equal(y_true, K.cast(y_pred > 0.5, y_true.dtype)))
+
+def dense_classifier(processed):
+    conc_proc = Concatenate()(processed)
+    d1 = Dense(64, activation='relu')(conc_proc)
+    # dr1 = Dropout(0.1)(d1)
+    # d2 = Dense(128, activation='relu')(d1)
+    d3 = Dense(8, activation='relu')(d1)
+    # dr2 = Dropout(0.1)(d2)
+    return Dense(2, activation='softmax')(d3)
+
+def segment_model(input_dim):
+    inp = Input(shape=input_dim)
+    # ls0 = LSTM(512, return_sequences=True)(inp)
+    ls1 = LSTM(128, return_sequences=True)(inp)
+    ls2 = LSTM(64, return_sequences=True)(ls1)
+    # ls3 = LSTM(32, return_sequences=True)(ls2)
+    ls4 = LSTM(32)(ls2)
+    d1 = Dense(64, activation='relu')(ls4)
+    d3 = Dense(8, activation='relu')(d1)
+    oup = Dense(2, activation='softmax')(d3)
+    return Model(inp, oup)
+
+def write_model_arch(mod,mod_file):
+    model_f = open(mod_file,'w')
+    model_f.write(mod.to_yaml())
+    model_f.close()
+
+def load_model_arch(mod_file):
+    model_f = open(mod_file,'r')
+    mod = model_from_yaml(model_f.read())
+    model_f.close()
+    return mod
+
+def train_segment(collection_name = 'test'):
+    batch_size = 128
+    model_dir = './models/segment/'+collection_name
+    create_dir(model_dir)
+    log_dir = './logs/segment/'+collection_name
+    create_dir(log_dir)
+    tr_gen_fn = segment_data_gen()
+    tr_gen = tr_gen_fn()
+    input_dim = (n_step, n_features)
+
+    model = segment_model(input_dim)
+    plot_model(model,show_shapes=True, to_file=model_dir+'/model.png')
+
+    tb_cb = TensorBoard(
+        log_dir=log_dir,
+        histogram_freq=1,
+        batch_size=32,
+        write_graph=True,
+        write_grads=True,
+        write_images=True,
+        embeddings_freq=0,
+        embeddings_layer_names=None,
+        embeddings_metadata=None)
+    cp_file_fmt = model_dir+'/siamese_speech_model-{epoch:02d}-epoch-{val_loss:0.2f}\
+-acc.h5'
+
+    cp_cb = ModelCheckpoint(
+        cp_file_fmt,
+        monitor='val_loss',
+        verbose=0,
+        save_best_only=True,
+        save_weights_only=True,
+        mode='auto',
+        period=1)
+    # train
+    rms = RMSprop()
+    model.compile(loss=categorical_crossentropy, optimizer=rms, metrics=[accuracy])
+    write_model_arch(model,model_dir+'/siamese_speech_model_arch.yaml')
+    epoch_n_steps = step_count(n_records,batch_size)
+    model.fit_generator(tr_gen
+                        , epochs=1000
+                        , steps_per_epoch=epoch_n_steps
+                        , validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y)
+                        , max_queue_size=32
+                        , callbacks=[tb_cb, cp_cb])
+    model.save(model_dir+'/speech_segment_model-final.h5')
+
+    y_pred = model.predict([te_pairs[:, 0], te_pairs[:, 1]])
+    te_acc = compute_accuracy(te_y, y_pred)
+    print('* Accuracy on test set: %0.2f%%' % (100 * te_acc))
+
+
+
+if __name__ == '__main__':
+    train_segment('test')

speech_model.py

@@ -3,7 +3,8 @@ from __future__ import print_function
 import numpy as np
 from speech_data import read_siamese_tfrecords_generator
 from keras.models import Model,load_model,model_from_yaml
-from keras.layers import Input, Dense, Dropout, LSTM, Lambda, Concatenate, Bidirectional
+from keras.layers import Input,Concatenate,Lambda, BatchNormalization, Dropout
+from keras.layers import Dense, LSTM, Bidirectional, GRU
 from keras.losses import categorical_crossentropy
 from keras.utils import to_categorical
 from keras.optimizers import RMSprop

speech_pitch.py

@@ -5,36 +5,35 @@ import matplotlib.pyplot as plt
 import seaborn as sns
 sns.set() # Use seaborn's default style to make graphs more pretty
 
-def pitch_array(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
+
+def pm_snd(sample_file):
+    # sample_file = 'inputs/self-apple/apple-low1.aiff'
     samples, samplerate, _  = snd.read(sample_file)
-    sample_sound = pm.Sound(values=samples,sampling_frequency=samplerate)
+    return pm.Sound(values=samples,sampling_frequency=samplerate)
+
+def pitch_array(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
+    sample_sound = pm_snd(sample_file)
     sample_pitch = sample_sound.to_pitch()
     return sample_pitch.to_matrix().as_array()
 
 def intensity_array(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
-    sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'
+    sample_sound = pm_snd(sample_file)
-    samples, samplerate, _  = snd.read(sample_file)
-    sample_sound = pm.Sound(values=samples,sampling_frequency=samplerate)
     sample_intensity = sample_sound.to_mfcc()
     sample_intensity.as_array().shape
     return sample_pitch.to_matrix().as_array()
 
 def compute_mfcc(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
-    # sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'
+    sample_sound = pm_snd(sample_file)
-    samples, samplerate, _  = snd.read(sample_file)
-    sample_sound = pm.Sound(values=samples,sampling_frequency=samplerate)
     sample_mfcc = sample_sound.to_mfcc()
     # sample_mfcc.to_array().shape
     return sample_mfcc.to_array()
 
 def compute_formants(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
     sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'
-    samples, samplerate, _  = snd.read(sample_file)
+    sample_sound = pm_snd(sample_file)
-    sample_sound = pm.Sound(values=samples,sampling_frequency=samplerate)
     sample_formant = sample_sound.to_formant_burg()
-    sample_formant.x_bins()
+    # sample_formant.x_bins()
-    # sample_mfcc.to_array().shape
+    return sample_formant.x_bins()
-    return sample_mfcc.to_array()
 
 def draw_spectrogram(spectrogram, dynamic_range=70):
     X, Y = spectrogram.x_grid(), spectrogram.y_grid()
@@ -62,10 +61,18 @@ def draw_pitch(pitch):
     plt.ylim(0, pitch.ceiling)
     plt.ylabel("pitch [Hz]")
 
-def pm_snd(sample_file):
+def draw_formants(formant):
-    # sample_file = 'inputs/self-apple/apple-low1.aiff'
+    # Extract selected pitch contour, and
-    samples, samplerate, _  = snd.read(sample_file)
+    # replace unvoiced samples by NaN to not plot
-    return pm.Sound(values=samples,sampling_frequency=samplerate)
+    formant_values = formant.to_matrix().values
+    pitch_values[pitch_values==0] = np.nan
+    plt.plot(pitch.xs(), pitch_values, linewidth=3, color='w')
+    plt.plot(pitch.xs(), pitch_values, linewidth=1)
+    plt.grid(False)
+    plt.ylim(0, pitch.ceiling)
+    plt.ylabel("Formants [val]")
+
+
 def plot_sample_raw(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
     # %matplotlib inline
     # sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff
@@ -109,28 +116,8 @@ def plot_sample_pitch(sample_file='outputs/audio/sunflowers-Victoria-180-normal-
 
 
 if __name__ == '__main__':
-    # sunflowers_vic_180_norm = pitch_array('outputs/audio/sunflowers-Victoria-180-normal-870.aiff')
+    mom_snd = pm_snd('outputs/test/moms_are_engineers-7608.aiff')
-    # sunflowers_fred_180_norm = pitch_array('outputs/audio/sunflowers-Fred-180-normal-6515.aiff')
-    # sunflowers_vic_180_norm_mfcc = compute_mfcc('outputs/audio/sunflowers-Victoria-180-normal-870.aiff')
-    # fred_180_norm_mfcc = compute_mfcc('outputs/audio/sunflowers-Fred-180-normal-6515.aiff')
-    # alex_mfcc = compute_mfcc('outputs/audio/sunflowers-Alex-180-normal-4763.aiff')
-    # # # sunflowers_vic_180_norm.shape
-    # # # sunflowers_fred_180_norm.shape
-    # # alex_mfcc.shape
-    # # sunflowers_vic_180_norm_mfcc.shape
-    # # sunflowers_fred_180_norm_mfcc.shape
-    # from speech_spectrum import generate_aiff_spectrogram
-    # vic_spec = generate_aiff_spectrogram('outputs/audio/sunflowers-Victoria-180-normal-870.aiff')
-    # alex_spec = generate_aiff_spectrogram('outputs/audio/sunflowers-Alex-180-normal-4763.aiff')
-    # alex150spec = generate_aiff_spectrogram('outputs/audio/sunflowers-Alex-150-normal-589.aiff')
-    # vic_spec.shape
-    # alex_spec.shape
-    # alex150spec.shape
-    # alex_mfcc.shape
-    # fred_180_norm_mfcc.shape
     plot_sample_pitch('outputs/audio/sunflowers-Victoria-180-normal-870.aiff')
     plot_sample_pitch('inputs/self-apple/apple-low1.aiff')
     plot_sample_pitch('inputs/self-apple/apple-low2.aiff')
     plot_sample_pitch('inputs/self-apple/apple-medium1.aiff')
-    # pm.SoundFileFormat
-    # pm.Pitch.get_number_of_frames()

speech_segmentgen.py

@@ -184,7 +184,7 @@ def story_texts():
 
 def generate_audio():
     synthQ = SynthesizerQueue()
-    phrases = random.sample(story_texts(), 5)  # story_texts()
+    phrases = random.sample(story_texts(), 100)  # story_texts()
     f = open(csv_dest_file, 'w')
     s_csv_w = csv.writer(f, quoting=csv.QUOTE_MINIMAL)
     i = 0