From 15f29895d4680fc04cdc836dbf5ac173a89d2aa8 Mon Sep 17 00:00:00 2001 From: Malar Kannan Date: Tue, 7 Nov 2017 00:10:23 +0530 Subject: [PATCH] implemented tfrecord reader and model refactor wip --- speech_data.py | 65 ++++++++++++++++++++++++++++++----------------- speech_siamese.py | 12 +++++---- 2 files changed, 48 insertions(+), 29 deletions(-) diff --git a/speech_data.py b/speech_data.py index 8876480..b9b822f 100644 --- a/speech_data.py +++ b/speech_data.py @@ -31,7 +31,8 @@ def siamese_pairs(rightGroup, wrongGroup): random.shuffle(rightWrongPairs) random.shuffle(rightRightPairs) # return (random.sample(same,10), random.sample(diff,10)) - return rightRightPairs[:10],rightWrongPairs[:10] + # return rightRightPairs[:10],rightWrongPairs[:10] + return rightRightPairs,rightWrongPairs def append_zeros(spgr, max_samples): return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)], @@ -42,7 +43,6 @@ def padd_zeros(spgr, max_samples): 'constant') def to_onehot(a,class_count=2): - # >>> a = np.array([1, 0, 3]) a_row_n = a.shape[0] b = np.zeros((a_row_n, class_count)) b[np.arange(a_row_n), a] = 1 @@ -101,6 +101,10 @@ def create_spectrogram_data(audio_group='audio'): audio_samples.to_pickle('outputs/{}-spectrogram.pkl'.format(audio_group)) def create_spectrogram_tfrecords(audio_group='audio'): + ''' + http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/ + http://www.machinelearninguru.com/deep_learning/tensorflow/basics/tfrecord/tfrecord.html + ''' audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv' , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file'] , quoting=csv.QUOTE_NONE) @@ -120,7 +124,6 @@ def create_spectrogram_tfrecords(audio_group='audio'): return tf.train.Feature(bytes_list=tf.train.BytesList(value=value)) writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '.tfrecords') - # audio_samples = audio_samples[:100] for (w, word_group) in audio_samples.groupby(audio_samples['word']): g = word_group.reset_index() g['spectrogram'] = apply_by_multiprocessing(g['file_path'],generate_aiff_spectrogram) @@ -160,22 +163,28 @@ def create_spectrogram_tfrecords(audio_group='audio'): writer.write(example.SerializeToString()) writer.close() -def create_tagged_data(audio_samples): - same_data, diff_data = [], [] - for (w, g) in audio_samples.groupby(audio_samples['word']): - # sample_norm = g.loc[audio_samples['variant'] == 'low'] - # sample_phon = g.loc[audio_samples['variant'] == 'medium'] - sample_norm = g.loc[audio_samples['variant'] == 'normal'] - sample_phon = g.loc[audio_samples['variant'] == 'phoneme'] - same, diff = get_siamese_pairs(sample_norm, sample_phon) - same_data.extend([create_X(s) for s in same]) - diff_data.extend([create_X(d) for d in diff]) - print('creating all speech pairs') - Y_f = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))]) - Y = to_onehot(Y_f.astype(np.int8)) - print('casting as array speech pairs') - X = np.asarray(same_data + diff_data) - return X,Y +def read_siamese_tfrecords(audio_group='audio'): + records_file = os.path.join('./outputs',audio_group+'.tfrecords') + record_iterator = tf.python_io.tf_record_iterator(path=records_file) + input_pairs = [] + output_class = [] + input_words = [] + for string_record in record_iterator: + example = tf.train.Example() + example.ParseFromString(string_record) + word = example.features.feature['word'].bytes_list.value[0] + input_words.append(word) + example.features.feature['spec2'].float_list.value[0] + spec_n1 = example.features.feature['spec_n1'].int64_list.value[0] + spec_n2 = example.features.feature['spec_n2'].int64_list.value[0] + spec_w1 = example.features.feature['spec_w1'].int64_list.value[0] + spec_w2 = example.features.feature['spec_w2'].int64_list.value[0] + spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(spec_n1,spec_w1) + spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(spec_n2,spec_w2) + input_pairs.append([spec1,spec2]) + output = example.features.feature['output'].int64_list.value + output_class.append(output) + return input_pairs,output_class def create_speech_pairs_data(audio_group='audio'): audio_samples = pd.read_pickle('outputs/{}-spectrogram.pkl'.format(audio_group)) @@ -195,10 +204,17 @@ def create_speech_pairs_data(audio_group='audio'): save_samples_for('train',tr_audio_samples) save_samples_for('test',te_audio_samples) -def speech_data(audio_group='audio'): - X = np.load('outputs/{}-X.npy'.format(audio_group)) / 255.0 - Y = np.load('outputs/{}-Y.npy'.format(audio_group)) - return (X,Y) +def audio_samples_word_count(audio_group='audio'): + audio_group = 'story_all' + audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv' + , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file'] + , quoting=csv.QUOTE_NONE) + # audio_samples = audio_samples.loc[audio_samples['word'] == + # 'sunflowers'].reset_index(drop=True) + audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x) + audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists) + audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index() + return len(audio_samples.groupby(audio_samples['word'])) def speech_model_data(): tr_pairs = np.load('outputs/tr_pairs.npy') / 255.0 @@ -214,7 +230,8 @@ if __name__ == '__main__': # sunflower_pairs_data() # create_spectrogram_data() # create_spectrogram_data('story_words') - create_spectrogram_tfrecords('story_words') + # create_spectrogram_tfrecords('story_words') + create_spectrogram_tfrecords('story_all') # create_padded_spectrogram() # create_speech_pairs_data() # print(speech_model_data()) diff --git a/speech_siamese.py b/speech_siamese.py index e07c6cf..64a28fb 100644 --- a/speech_siamese.py +++ b/speech_siamese.py @@ -1,7 +1,8 @@ from __future__ import absolute_import from __future__ import print_function import numpy as np -from speech_data import speech_model_data +# from speech_data import speech_model_data +from speech_data import read_siamese_tfrecords from keras.models import Model,load_model from keras.layers import Input, Dense, Dropout, LSTM, Lambda, Concatenate from keras.losses import categorical_crossentropy @@ -80,10 +81,11 @@ def siamese_model(input_dim): def train_siamese(): # the data, shuffled and split between train and test sets - tr_pairs, te_pairs, tr_y_e, te_y_e = speech_model_data() - tr_y = to_categorical(tr_y_e, num_classes=2) - te_y = to_categorical(te_y_e, num_classes=2) - input_dim = (tr_pairs.shape[2], tr_pairs.shape[3]) + # tr_pairs, te_pairs, tr_y_e, te_y_e = speech_model_data() + pairs,y = read_siamese_tfrecords('story_words') + # tr_y = to_categorical(tr_y_e, num_classes=2) + # te_y = to_categorical(te_y_e, num_classes=2) + input_dim = (None, 1654) model = siamese_model(input_dim)