generating spectrogram parallelly

pandas_parallel.py

@@ -0,0 +1,25 @@
+import multiprocessing
+import pandas as pd
+import numpy as np
+
+
+
+def _apply_df(args):
+    df, func, num, kwargs = args
+    return num, df.apply(func, **kwargs)
+
+def apply_by_multiprocessing(df,func,**kwargs):
+  cores = multiprocessing.cpu_count()
+  workers=kwargs.pop('workers') if 'workers' in kwargs else cores
+  pool = multiprocessing.Pool(processes=workers)
+  result = pool.map(_apply_df, [(d, func, i, kwargs) for i,d in enumerate(np.array_split(df, workers))])
+  pool.close()
+  result=sorted(result,key=lambda x:x[0])
+  return pd.concat([i[1] for i in result])
+
+def square(x):
+    return x**x
+
+if __name__ == '__main__':
+    df = pd.DataFrame({'a':range(10), 'b':range(10)})
+    apply_by_multiprocessing(df, square, axis=1, workers=4)

speech_data.py

@@ -1,28 +1,46 @@
 import pandas as pd
+from pandas_parallel import apply_by_multiprocessing
+import dask as dd
+import dask.dataframe as ddf
 import numpy as np
 from spectro_gen import generate_aiff_spectrogram
 from sklearn.model_selection import train_test_split
 import itertools
+import random
+import csv
 import gc
 
 
 def get_siamese_pairs(groupF1, groupF2):
     group1 = [r for (i, r) in groupF1.iterrows()]
     group2 = [r for (i, r) in groupF2.iterrows()]
-    f = [(g1, g2) for g2 in group2 for g1 in group1]
+    diff = [(g1, g2) for g2 in group2 for g1 in group1]
-    t = [i for i in itertools.combinations(group1, 2)
+    same = [i for i in itertools.combinations(group1, 2)
          ] + [i for i in itertools.combinations(group2, 2)]
-    return (t, f)
+    random.shuffle(same)
+    random.shuffle(diff)
+    # return (random.sample(same,10), random.sample(diff,10))
+    return same[:10],diff[:10]
 
 
 def append_zeros(spgr, max_samples):
     return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)],
                       'median')
 
+def padd_zeros(spgr, max_samples):
+    return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)],
+                      'constant')
+
+def to_onehot(a,class_count=2):
+    # >>> a = np.array([1, 0, 3])
+    a_row_n = a.shape[0]
+    b = np.zeros((a_row_n, class_count))
+    b[np.arange(a_row_n), a] = 1
+    return b
 
 def create_pair(l, r, max_samples):
-    l_sample = append_zeros(l, max_samples)
+    l_sample = padd_zeros(l, max_samples)
-    r_sample = append_zeros(r, max_samples)
+    r_sample = padd_zeros(r, max_samples)
     return np.asarray([l_sample, r_sample])
 
 
@@ -36,73 +54,116 @@ def create_X(sp, max_samples):
     return create_pair(sp[0]['spectrogram'], sp[1]['spectrogram'], max_samples)
 
 
-def get_word_pairs_data(word, max_samples):
+# def get_word_pairs_data(word, max_samples):
-    audio_samples = pd.read_csv(
+#     audio_samples = pd.read_csv(
-        './outputs/audio.csv',
+#         './outputs/audio.csv',
-        names=['word', 'voice', 'rate', 'variant', 'file'])
+#         names=['word', 'voice', 'rate', 'variant', 'file'])
-    audio_samples = audio_samples.loc[audio_samples['word'] ==
+#     audio_samples = audio_samples.loc[audio_samples['word'] ==
-                                      word].reset_index(drop=True)
+#                                       word].reset_index(drop=True)
-    audio_samples.loc[:, 'spectrogram'] = audio_samples.loc[:, 'file'].apply(
+#     audio_samples.loc[:, 'spectrogram'] = audio_samples.loc[:, 'file'].apply(
-        lambda x: 'outputs/audio/' + x).apply(generate_aiff_spectrogram)
+#         lambda x: 'outputs/audio/' + x).apply(generate_aiff_spectrogram)
 #     max_samples = audio_samples['spectrogram'].apply(
 #         lambda x: x.shape[0]).max()
-    same_data, diff_data = [], []
+#     same_data, diff_data = [], []
-    for (w, g) in audio_samples.groupby(audio_samples['word']):
+#     for (w, g) in audio_samples.groupby(audio_samples['word']):
-        sample_norm = g.loc[audio_samples['variant'] == 'normal']
+#         sample_norm = g.loc[audio_samples['variant'] == 'normal']
-        sample_phon = g.loc[audio_samples['variant'] == 'phoneme']
+#         sample_phon = g.loc[audio_samples['variant'] == 'phoneme']
-        same, diff = get_siamese_pairs(sample_norm, sample_phon)
+#         same, diff = get_siamese_pairs(sample_norm, sample_phon)
-        same_data.extend([create_X(s, max_samples) for s in same[:10]])
+#         same_data.extend([create_X(s, max_samples) for s in same])
-        diff_data.extend([create_X(d, max_samples) for d in diff[:10]])
+#         diff_data.extend([create_X(d, max_samples) for d in diff])
-    Y = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))])
+#     Y = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))])
-    X = np.asarray(same_data + diff_data)
+#     X = np.asarray(same_data + diff_data)
-    # tr_pairs, te_pairs, tr_y, te_y = train_test_split(X, Y, test_size=0.1)
+#     # tr_pairs, te_pairs, tr_y, te_y = train_test_split(X, Y, test_size=0.1)
-    return (X, Y)
+#     return (X, Y)
 
 
 def create_spectrogram_data(audio_group='audio'):
-    audio_samples = pd.read_csv(
+    audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv'
-        './outputs/' + audio_group + '.csv',
+        , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
-        names=['word', 'voice', 'rate', 'variant', 'file'])
+        , quoting=csv.QUOTE_NONE)
     # audio_samples = audio_samples.loc[audio_samples['word'] ==
     #                                   'sunflowers'].reset_index(drop=True)
-    audio_samples.loc[:, 'spectrogram'] = audio_samples.loc[:, 'file'].apply(
+    file_names = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
-        lambda x: 'outputs/' + audio_group + '/' + x).apply(
+    audio_samples['spectrogram'] = apply_by_multiprocessing(file_names,generate_aiff_spectrogram)#.apply(
-            generate_aiff_spectrogram)
+            #generate_aiff_spectrogram)
-    audio_samples.to_pickle('outputs/spectrogram.pkl')
+    audio_samples['window_count'] = audio_samples.loc[:,'spectrogram'].apply(lambda x: x.shape[0])
+    audio_samples.to_pickle('outputs/{}-spectrogram.pkl'.format(audio_group))
 
 
-def create_speech_pairs_data(audio_group='audio'):
+def create_tagged_data(audio_samples):
-    audio_samples = pd.read_pickle('outputs/spectrogram.pkl')
-    max_samples = audio_samples['spectrogram'].apply(
-        lambda x: x.shape[0]).max()
-    # sample_size = audio_samples['spectrogram'][0].shape[1]
-
-    print('generating siamese speech pairs')
     same_data, diff_data = [], []
     for (w, g) in audio_samples.groupby(audio_samples['word']):
+        # sample_norm = g.loc[audio_samples['variant'] == 'low']
+        # sample_phon = g.loc[audio_samples['variant'] == 'medium']
         sample_norm = g.loc[audio_samples['variant'] == 'normal']
         sample_phon = g.loc[audio_samples['variant'] == 'phoneme']
         same, diff = get_siamese_pairs(sample_norm, sample_phon)
-        same_data.extend([create_X(s, max_samples) for s in same[:10]])
+        same_data.extend([create_X(s) for s in same])
-        diff_data.extend([create_X(d, max_samples) for d in diff[:10]])
+        diff_data.extend([create_X(d) for d in diff])
     print('creating all speech pairs')
-    Y = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))])
+    Y_f = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))])
+    Y = to_onehot(Y_f.astype(np.int8))
     print('casting as array speech pairs')
     X = np.asarray(same_data + diff_data)
-    print('pickling X/Y')
+    # del same_data
-    np.save('outputs/X.npy', X)
+    # del diff_data
-    np.save('outputs/Y.npy', Y)
+    # gc.collect()
-    del same_data
+    return X,Y
-    del diff_data
-    gc.collect()
-    print('train/test splitting speech pairs')
-    tr_pairs, te_pairs, tr_y, te_y = train_test_split(X, Y, test_size=0.1)
-    print('pickling train/test')
-    np.save('outputs/tr_pairs.npy', tr_pairs)
-    np.save('outputs/te_pairs.npy', te_pairs)
-    np.save('outputs/tr_y.npy', tr_y)
-    np.save('outputs/te_y.npy', te_y)
 
+# def create_padded_spectrogram(audio_group='audio'):
+#     audio_samples = pd.read_pickle('outputs/{}-spectrogram.pkl'.format(audio_group))
+#     daf_audio_samples = ddf.from_pandas(audio_samples,npartitions=4)
+#     def spec_size(s):
+#         return s['spectrogram'].shape[0]
+#     max_samples = daf_audio_samples.apply(spec_size,axis=1, meta=('x', 'i8')).max().compute()
+#     print('max sample count is ',max_samples)
+#     def padd_zeros_fixed(sp):
+#         x = sp['spectrogram']
+#         bounds = [(0, max_samples - x.shape[0]), (0, 0)]
+#         sp['spectrogram'] = np.lib.pad(x,bounds,'constant')
+#         return sp
+#     padded_audio_samples = daf_audio_samples.apply(padd_zeros_fixed,axis=1, meta=audio_samples).compute()#,max_samples=max_samples)
+#     import pdb; pdb.set_trace()
+#     padded_spectrogram = np.asarray(padded_audio_samples['spectrogram'].tolist())
+#     np.save('outputs/{}-padded_spectrogram.npy'.format(audio_group),padded_spectrogram)
+
+def create_speech_pairs_data(audio_group='audio'):
+    audio_samples = pd.read_pickle('outputs/{}-spectrogram.pkl'.format(audio_group))
+    # sample_size = audio_samples['spectrogram'][0].shape[1]
+    tr_audio_samples,te_audio_samples = train_test_split(audio_samples, test_size=0.1)
+    def save_samples_for(sample_name,samples):
+        print('generating {} siamese speech pairs'.format(sample_name))
+        X,Y = create_tagged_data(samples)
+        print('shuffling array speech pairs')
+        rng_state = np.random.get_state()
+        np.random.shuffle(X)
+        np.random.set_state(rng_state)
+        np.random.shuffle(Y)
+        # p = np.random.permutation(len(X))
+        # X = X_i[p]
+        # Y = Y_i[p]
+        print('pickling X/Y')
+        np.save('outputs/{}-train-X.npy'.format(audio_group), X)
+        np.save('outputs/{}-train-Y.npy'.format(audio_group), Y)
+    save_samples_for('train',tr_audio_samples)
+    save_samples_for('test',te_audio_samples)
+    # print('generating test siamese speech pairs ')
+    # X,Y = create_tagged_data(te_audio_samples,max_samples)
+    # print('pickling X/Y')
+    # np.save('outputs/{}-test-X.npy'.format(audio_group), X)
+    # np.save('outputs/{}-test-Y.npy'.format(audio_group), Y)
+    # # print('train/test splitting speech pairs')
+    # # tr_pairs, te_pairs, tr_y, te_y = train_test_split(X, Y, test_size=0.1)
+    # # print('pickling train/test')
+    # # np.save('outputs/{}-tr_pairs.npy'.format(audio_group), tr_pairs)
+    # # np.save('outputs/{}-te_pairs.npy'.format(audio_group), te_pairs)
+    # # np.save('outputs/{}-tr_y.npy'.format(audio_group), tr_y)
+    # # np.save('outputs/{}-te_y.npy'.format(audio_group), te_y)
+
+def speech_data(audio_group='audio'):
+    X = np.load('outputs/{}-X.npy'.format(audio_group)) / 255.0
+    Y = np.load('outputs/{}-Y.npy'.format(audio_group))
+    return (X,Y)
 
 def speech_model_data():
     tr_pairs = np.load('outputs/tr_pairs.npy') / 255.0
@@ -117,5 +178,7 @@ def speech_model_data():
 if __name__ == '__main__':
     # sunflower_pairs_data()
     # create_spectrogram_data()
-    create_speech_pairs_data()
+    create_spectrogram_data('story_words')
+    # create_padded_spectrogram()
+    # create_speech_pairs_data()
     # print(speech_model_data())

test_siamese.py

@@ -1,11 +1,10 @@
 from speech_siamese import siamese_model
 from record_mic_speech import record_spectrogram
 from importlib import reload
-import speech_data
+# import speech_data
-reload(speech_data)
+# reload(speech_data)
-from speech_data import create_test_pair,get_word_pairs_data
+from speech_data import create_test_pair,get_word_pairs_data,speech_data
 import numpy as np
-from keras.utils import to_categorical
 
 model = siamese_model((15, 1654))
 model.load_weights('./models/siamese_speech_model-final.h5')
@@ -16,9 +15,16 @@ def predict_recording_with(m,sample_size=15):
     inp = create_test_pair(spec1,spec2,sample_size)
     return m.predict([inp[:, 0], inp[:, 1]])
 
-while(True):
+# while(True):
-    print(predict_recording_with(model))
+#     print(predict_recording_with(model))
 
+
+def test_with(audio_group):
+    X,Y = speech_data(audio_group)
+    print(np.argmax(model.predict([X[:, 0], X[:, 1]]),axis=1))
+    print(Y.astype(np.int8))
+
+test_with('rand_edu')
 # sunflower_data,sunflower_result = get_word_pairs_data('sweater',15)
 # print(np.argmax(model.predict([sunflower_data[:, 0], sunflower_data[:, 1]]),axis=1))
 # print(sunflower_result)