segment_data.py

@@ -1,11 +0,0 @@
-import pandas as pd
-
-def fix_csv(collection_name = 'test'):
-    seg_data = pd.read_csv('./outputs/'+collection_name+'.csv',names=['phrase','filename'
-                ,'start_phoneme','end_phoneme','start_time','end_time'])
-    seg_data.to_csv('./outputs/'+collection_name+'.fixed.csv')
-
-
-def segment_data_gen(collection_name = 'test'):
-    # collection_name = 'test'
-    seg_data = pd.read_csv('./outputs/'+collection_name+'.fixed.csv',index_col=0)

segment_model.py

@@ -1,108 +0,0 @@
-from __future__ import absolute_import
-from __future__ import print_function
-import numpy as np
-from keras.models import Model,load_model,model_from_yaml
-from keras.layers import Input,Concatenate,Lambda, BatchNormalization, Dropout
-from keras.layers import Dense, LSTM, Bidirectional, GRU
-from keras.losses import categorical_crossentropy
-from keras.utils import to_categorical
-from keras.optimizers import RMSprop
-from keras.callbacks import TensorBoard, ModelCheckpoint
-from keras import backend as K
-from keras.utils import plot_model
-from speech_tools import create_dir,step_count
-from speech_data import segment_data_gen
-
-
-def accuracy(y_true, y_pred):
-    '''Compute classification accuracy with a fixed threshold on distances.
-    '''
-    return K.mean(K.equal(y_true, K.cast(y_pred > 0.5, y_true.dtype)))
-
-def dense_classifier(processed):
-    conc_proc = Concatenate()(processed)
-    d1 = Dense(64, activation='relu')(conc_proc)
-    # dr1 = Dropout(0.1)(d1)
-    # d2 = Dense(128, activation='relu')(d1)
-    d3 = Dense(8, activation='relu')(d1)
-    # dr2 = Dropout(0.1)(d2)
-    return Dense(2, activation='softmax')(d3)
-
-def segment_model(input_dim):
-    inp = Input(shape=input_dim)
-    # ls0 = LSTM(512, return_sequences=True)(inp)
-    ls1 = LSTM(128, return_sequences=True)(inp)
-    ls2 = LSTM(64, return_sequences=True)(ls1)
-    # ls3 = LSTM(32, return_sequences=True)(ls2)
-    ls4 = LSTM(32)(ls2)
-    d1 = Dense(64, activation='relu')(ls4)
-    d3 = Dense(8, activation='relu')(d1)
-    oup = Dense(2, activation='softmax')(d3)
-    return Model(inp, oup)
-
-def write_model_arch(mod,mod_file):
-    model_f = open(mod_file,'w')
-    model_f.write(mod.to_yaml())
-    model_f.close()
-
-def load_model_arch(mod_file):
-    model_f = open(mod_file,'r')
-    mod = model_from_yaml(model_f.read())
-    model_f.close()
-    return mod
-
-def train_segment(collection_name = 'test'):
-    batch_size = 128
-    model_dir = './models/segment/'+collection_name
-    create_dir(model_dir)
-    log_dir = './logs/segment/'+collection_name
-    create_dir(log_dir)
-    tr_gen_fn = segment_data_gen()
-    tr_gen = tr_gen_fn()
-    input_dim = (n_step, n_features)
-
-    model = segment_model(input_dim)
-    plot_model(model,show_shapes=True, to_file=model_dir+'/model.png')
-
-    tb_cb = TensorBoard(
-        log_dir=log_dir,
-        histogram_freq=1,
-        batch_size=32,
-        write_graph=True,
-        write_grads=True,
-        write_images=True,
-        embeddings_freq=0,
-        embeddings_layer_names=None,
-        embeddings_metadata=None)
-    cp_file_fmt = model_dir+'/siamese_speech_model-{epoch:02d}-epoch-{val_loss:0.2f}\
--acc.h5'
-
-    cp_cb = ModelCheckpoint(
-        cp_file_fmt,
-        monitor='val_loss',
-        verbose=0,
-        save_best_only=True,
-        save_weights_only=True,
-        mode='auto',
-        period=1)
-    # train
-    rms = RMSprop()
-    model.compile(loss=categorical_crossentropy, optimizer=rms, metrics=[accuracy])
-    write_model_arch(model,model_dir+'/siamese_speech_model_arch.yaml')
-    epoch_n_steps = step_count(n_records,batch_size)
-    model.fit_generator(tr_gen
-                        , epochs=1000
-                        , steps_per_epoch=epoch_n_steps
-                        , validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y)
-                        , max_queue_size=32
-                        , callbacks=[tb_cb, cp_cb])
-    model.save(model_dir+'/speech_segment_model-final.h5')
-
-    y_pred = model.predict([te_pairs[:, 0], te_pairs[:, 1]])
-    te_acc = compute_accuracy(te_y, y_pred)
-    print('* Accuracy on test set: %0.2f%%' % (100 * te_acc))
-
-
-
-if __name__ == '__main__':
-    train_segment('test')

speech_model.py

@@ -3,8 +3,7 @@ from __future__ import print_function
 import numpy as np
 from speech_data import read_siamese_tfrecords_generator
 from keras.models import Model,load_model,model_from_yaml
-from keras.layers import Input,Concatenate,Lambda, BatchNormalization, Dropout
-from keras.layers import Dense, LSTM, Bidirectional, GRU
+from keras.layers import Input, Dense, Dropout, LSTM, Lambda, Concatenate, Bidirectional
 from keras.losses import categorical_crossentropy
 from keras.utils import to_categorical
 from keras.optimizers import RMSprop

speech_pitch.py

@@ -5,35 +5,36 @@ import matplotlib.pyplot as plt
 import seaborn as sns
 sns.set() # Use seaborn's default style to make graphs more pretty
 
-
-def pm_snd(sample_file):
-    # sample_file = 'inputs/self-apple/apple-low1.aiff'
-    samples, samplerate, _  = snd.read(sample_file)
-    return pm.Sound(values=samples,sampling_frequency=samplerate)
-
 def pitch_array(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
-    sample_sound = pm_snd(sample_file)
+    samples, samplerate, _  = snd.read(sample_file)
+    sample_sound = pm.Sound(values=samples,sampling_frequency=samplerate)
     sample_pitch = sample_sound.to_pitch()
     return sample_pitch.to_matrix().as_array()
 
 def intensity_array(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
-    sample_sound = pm_snd(sample_file)
+    sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'
+    samples, samplerate, _  = snd.read(sample_file)
+    sample_sound = pm.Sound(values=samples,sampling_frequency=samplerate)
     sample_intensity = sample_sound.to_mfcc()
     sample_intensity.as_array().shape
     return sample_pitch.to_matrix().as_array()
 
 def compute_mfcc(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
-    sample_sound = pm_snd(sample_file)
+    # sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'
+    samples, samplerate, _  = snd.read(sample_file)
+    sample_sound = pm.Sound(values=samples,sampling_frequency=samplerate)
     sample_mfcc = sample_sound.to_mfcc()
     # sample_mfcc.to_array().shape
     return sample_mfcc.to_array()
 
 def compute_formants(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
     sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'
-    sample_sound = pm_snd(sample_file)
+    samples, samplerate, _  = snd.read(sample_file)
+    sample_sound = pm.Sound(values=samples,sampling_frequency=samplerate)
     sample_formant = sample_sound.to_formant_burg()
-    # sample_formant.x_bins()
-    return sample_formant.x_bins()
+    sample_formant.x_bins()
+    # sample_mfcc.to_array().shape
+    return sample_mfcc.to_array()
 
 def draw_spectrogram(spectrogram, dynamic_range=70):
     X, Y = spectrogram.x_grid(), spectrogram.y_grid()
@@ -61,18 +62,10 @@ def draw_pitch(pitch):
     plt.ylim(0, pitch.ceiling)
     plt.ylabel("pitch [Hz]")
 
-def draw_formants(formant):
-    # Extract selected pitch contour, and
-    # replace unvoiced samples by NaN to not plot
-    formant_values = formant.to_matrix().values
-    pitch_values[pitch_values==0] = np.nan
-    plt.plot(pitch.xs(), pitch_values, linewidth=3, color='w')
-    plt.plot(pitch.xs(), pitch_values, linewidth=1)
-    plt.grid(False)
-    plt.ylim(0, pitch.ceiling)
-    plt.ylabel("Formants [val]")
-
-
+def pm_snd(sample_file):
+    # sample_file = 'inputs/self-apple/apple-low1.aiff'
+    samples, samplerate, _  = snd.read(sample_file)
+    return pm.Sound(values=samples,sampling_frequency=samplerate)
 def plot_sample_raw(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
     # %matplotlib inline
     # sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff
@@ -116,8 +109,28 @@ def plot_sample_pitch(sample_file='outputs/audio/sunflowers-Victoria-180-normal-
 
 
 if __name__ == '__main__':
-    mom_snd = pm_snd('outputs/test/moms_are_engineers-7608.aiff')
+    # sunflowers_vic_180_norm = pitch_array('outputs/audio/sunflowers-Victoria-180-normal-870.aiff')
+    # sunflowers_fred_180_norm = pitch_array('outputs/audio/sunflowers-Fred-180-normal-6515.aiff')
+    # sunflowers_vic_180_norm_mfcc = compute_mfcc('outputs/audio/sunflowers-Victoria-180-normal-870.aiff')
+    # fred_180_norm_mfcc = compute_mfcc('outputs/audio/sunflowers-Fred-180-normal-6515.aiff')
+    # alex_mfcc = compute_mfcc('outputs/audio/sunflowers-Alex-180-normal-4763.aiff')
+    # # # sunflowers_vic_180_norm.shape
+    # # # sunflowers_fred_180_norm.shape
+    # # alex_mfcc.shape
+    # # sunflowers_vic_180_norm_mfcc.shape
+    # # sunflowers_fred_180_norm_mfcc.shape
+    # from speech_spectrum import generate_aiff_spectrogram
+    # vic_spec = generate_aiff_spectrogram('outputs/audio/sunflowers-Victoria-180-normal-870.aiff')
+    # alex_spec = generate_aiff_spectrogram('outputs/audio/sunflowers-Alex-180-normal-4763.aiff')
+    # alex150spec = generate_aiff_spectrogram('outputs/audio/sunflowers-Alex-150-normal-589.aiff')
+    # vic_spec.shape
+    # alex_spec.shape
+    # alex150spec.shape
+    # alex_mfcc.shape
+    # fred_180_norm_mfcc.shape
     plot_sample_pitch('outputs/audio/sunflowers-Victoria-180-normal-870.aiff')
     plot_sample_pitch('inputs/self-apple/apple-low1.aiff')
     plot_sample_pitch('inputs/self-apple/apple-low2.aiff')
     plot_sample_pitch('inputs/self-apple/apple-medium1.aiff')
+    # pm.SoundFileFormat
+    # pm.Pitch.get_number_of_frames()

speech_tts_queue.py

@@ -184,7 +184,7 @@ def story_texts():
 
 def generate_audio():
     synthQ = SynthesizerQueue()
-    phrases = random.sample(story_texts(), 100)  # story_texts()
+    phrases = random.sample(story_texts(), 5)  # story_texts()
     f = open(csv_dest_file, 'w')
     s_csv_w = csv.writer(f, quoting=csv.QUOTE_MINIMAL)
     i = 0