malar
/
speech-scoring
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

implemented tfrecord reader and model refactor wip

master
Malar Kannan 2017-11-07 00:10:23 +05:30
parent 5b682c78b8
commit 15f29895d4
2 changed files with 48 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
speech_data.py
View File

@ -31,7 +31,8 @@ def siamese_pairs(rightGroup, wrongGroup):
random.shuffle(rightWrongPairs) random.shuffle(rightWrongPairs)
random.shuffle(rightRightPairs) random.shuffle(rightRightPairs)
# return (random.sample(same,10), random.sample(diff,10)) # return (random.sample(same,10), random.sample(diff,10))
return rightRightPairs[:10],rightWrongPairs[:10] # return rightRightPairs[:10],rightWrongPairs[:10]
return rightRightPairs,rightWrongPairs
def append_zeros(spgr, max_samples): def append_zeros(spgr, max_samples):
return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)], return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)],
@ -42,7 +43,6 @@ def padd_zeros(spgr, max_samples):
'constant') 'constant')
def to_onehot(a,class_count=2): def to_onehot(a,class_count=2):
# >>> a = np.array([1, 0, 3])
a_row_n = a.shape[0] a_row_n = a.shape[0]
b = np.zeros((a_row_n, class_count)) b = np.zeros((a_row_n, class_count))
b[np.arange(a_row_n), a] = 1 b[np.arange(a_row_n), a] = 1
@ -101,6 +101,10 @@ def create_spectrogram_data(audio_group='audio'):
audio_samples.to_pickle('outputs/{}-spectrogram.pkl'.format(audio_group)) audio_samples.to_pickle('outputs/{}-spectrogram.pkl'.format(audio_group))
def create_spectrogram_tfrecords(audio_group='audio'): def create_spectrogram_tfrecords(audio_group='audio'):
'''
http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
http://www.machinelearninguru.com/deep_learning/tensorflow/basics/tfrecord/tfrecord.html
'''
audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv' audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv'
, names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file'] , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
, quoting=csv.QUOTE_NONE) , quoting=csv.QUOTE_NONE)
@ -120,7 +124,6 @@ def create_spectrogram_tfrecords(audio_group='audio'):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=value)) return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '.tfrecords') writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '.tfrecords')
# audio_samples = audio_samples[:100]
for (w, word_group) in audio_samples.groupby(audio_samples['word']): for (w, word_group) in audio_samples.groupby(audio_samples['word']):
g = word_group.reset_index() g = word_group.reset_index()
g['spectrogram'] = apply_by_multiprocessing(g['file_path'],generate_aiff_spectrogram) g['spectrogram'] = apply_by_multiprocessing(g['file_path'],generate_aiff_spectrogram)
@ -160,22 +163,28 @@ def create_spectrogram_tfrecords(audio_group='audio'):
writer.write(example.SerializeToString()) writer.write(example.SerializeToString())
writer.close() writer.close()
def create_tagged_data(audio_samples): def read_siamese_tfrecords(audio_group='audio'):
same_data, diff_data = [], [] records_file = os.path.join('./outputs',audio_group+'.tfrecords')
for (w, g) in audio_samples.groupby(audio_samples['word']): record_iterator = tf.python_io.tf_record_iterator(path=records_file)
# sample_norm = g.loc[audio_samples['variant'] == 'low'] input_pairs = []
# sample_phon = g.loc[audio_samples['variant'] == 'medium'] output_class = []
sample_norm = g.loc[audio_samples['variant'] == 'normal'] input_words = []
sample_phon = g.loc[audio_samples['variant'] == 'phoneme'] for string_record in record_iterator:
same, diff = get_siamese_pairs(sample_norm, sample_phon) example = tf.train.Example()
same_data.extend([create_X(s) for s in same]) example.ParseFromString(string_record)
diff_data.extend([create_X(d) for d in diff]) word = example.features.feature['word'].bytes_list.value[0]
print('creating all speech pairs') input_words.append(word)
Y_f = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))]) example.features.feature['spec2'].float_list.value[0]
Y = to_onehot(Y_f.astype(np.int8)) spec_n1 = example.features.feature['spec_n1'].int64_list.value[0]
print('casting as array speech pairs') spec_n2 = example.features.feature['spec_n2'].int64_list.value[0]
X = np.asarray(same_data + diff_data) spec_w1 = example.features.feature['spec_w1'].int64_list.value[0]
return X,Y spec_w2 = example.features.feature['spec_w2'].int64_list.value[0]
spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(spec_n1,spec_w1)
spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(spec_n2,spec_w2)
input_pairs.append([spec1,spec2])
output = example.features.feature['output'].int64_list.value
output_class.append(output)
return input_pairs,output_class
def create_speech_pairs_data(audio_group='audio'): def create_speech_pairs_data(audio_group='audio'):
audio_samples = pd.read_pickle('outputs/{}-spectrogram.pkl'.format(audio_group)) audio_samples = pd.read_pickle('outputs/{}-spectrogram.pkl'.format(audio_group))
@ -195,10 +204,17 @@ def create_speech_pairs_data(audio_group='audio'):
save_samples_for('train',tr_audio_samples) save_samples_for('train',tr_audio_samples)
save_samples_for('test',te_audio_samples) save_samples_for('test',te_audio_samples)
def speech_data(audio_group='audio'): def audio_samples_word_count(audio_group='audio'):
X = np.load('outputs/{}-X.npy'.format(audio_group)) / 255.0 audio_group = 'story_all'
Y = np.load('outputs/{}-Y.npy'.format(audio_group)) audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv'
return (X,Y) , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
, quoting=csv.QUOTE_NONE)
# audio_samples = audio_samples.loc[audio_samples['word'] ==
# 'sunflowers'].reset_index(drop=True)
audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists)
audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index()
return len(audio_samples.groupby(audio_samples['word']))
def speech_model_data(): def speech_model_data():
tr_pairs = np.load('outputs/tr_pairs.npy') / 255.0 tr_pairs = np.load('outputs/tr_pairs.npy') / 255.0
@ -214,7 +230,8 @@ if __name__ == '__main__':
# sunflower_pairs_data() # sunflower_pairs_data()
# create_spectrogram_data() # create_spectrogram_data()
# create_spectrogram_data('story_words') # create_spectrogram_data('story_words')
create_spectrogram_tfrecords('story_words') # create_spectrogram_tfrecords('story_words')
create_spectrogram_tfrecords('story_all')
# create_padded_spectrogram() # create_padded_spectrogram()
# create_speech_pairs_data() # create_speech_pairs_data()
# print(speech_model_data()) # print(speech_model_data())

12
speech_siamese.py
View File

@ -1,7 +1,8 @@
from __future__ import absolute_import from __future__ import absolute_import
from __future__ import print_function from __future__ import print_function
import numpy as np import numpy as np
from speech_data import speech_model_data # from speech_data import speech_model_data
from speech_data import read_siamese_tfrecords
from keras.models import Model,load_model from keras.models import Model,load_model
from keras.layers import Input, Dense, Dropout, LSTM, Lambda, Concatenate from keras.layers import Input, Dense, Dropout, LSTM, Lambda, Concatenate
from keras.losses import categorical_crossentropy from keras.losses import categorical_crossentropy
@ -80,10 +81,11 @@ def siamese_model(input_dim):
def train_siamese(): def train_siamese():
# the data, shuffled and split between train and test sets # the data, shuffled and split between train and test sets
tr_pairs, te_pairs, tr_y_e, te_y_e = speech_model_data() # tr_pairs, te_pairs, tr_y_e, te_y_e = speech_model_data()
tr_y = to_categorical(tr_y_e, num_classes=2) pairs,y = read_siamese_tfrecords('story_words')
te_y = to_categorical(te_y_e, num_classes=2) # tr_y = to_categorical(tr_y_e, num_classes=2)
input_dim = (tr_pairs.shape[2], tr_pairs.shape[3]) # te_y = to_categorical(te_y_e, num_classes=2)
input_dim = (None, 1654)
model = siamese_model(input_dim) model = siamese_model(input_dim)