diff --git a/speech_data.py b/speech_data.py index b254bee..8876480 100644 --- a/speech_data.py +++ b/speech_data.py @@ -23,6 +23,15 @@ def get_siamese_pairs(groupF1, groupF2): # return (random.sample(same,10), random.sample(diff,10)) return same[:10],diff[:10] +def siamese_pairs(rightGroup, wrongGroup): + group1 = [r for (i, r) in rightGroup.iterrows()] + group2 = [r for (i, r) in wrongGroup.iterrows()] + rightWrongPairs = [(g1, g2) for g2 in group2 for g1 in group1] + rightRightPairs = [i for i in itertools.combinations(group1, 2)] + random.shuffle(rightWrongPairs) + random.shuffle(rightRightPairs) + # return (random.sample(same,10), random.sample(diff,10)) + return rightRightPairs[:10],rightWrongPairs[:10] def append_zeros(spgr, max_samples): return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)], @@ -92,7 +101,6 @@ def create_spectrogram_data(audio_group='audio'): audio_samples.to_pickle('outputs/{}-spectrogram.pkl'.format(audio_group)) def create_spectrogram_tfrecords(audio_group='audio'): - # audio_group = 'story_words' audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv' , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file'] , quoting=csv.QUOTE_NONE) @@ -101,9 +109,6 @@ def create_spectrogram_tfrecords(audio_group='audio'): audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x) audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists) audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index() - # audio_samples['spectrogram'] = apply_by_multiprocessing(audio_samples['file_paths'],generate_aiff_spectrogram)#.apply( - # audio_samples['window_count'] = audio_samples.loc[:,'spectrogram'].apply(lambda x: x.shape[0]) - # audio_samples.to_pickle('outputs/{}-spectrogram.pkl'.format(audio_group)) def _float_feature(value): return tf.train.Feature(float_list=tf.train.FloatList(value=value)) @@ -115,27 +120,44 @@ def create_spectrogram_tfrecords(audio_group='audio'): return tf.train.Feature(bytes_list=tf.train.BytesList(value=value)) writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '.tfrecords') - audio_samples = audio_samples[:100] - for (i,sample) in audio_samples.iterrows(): - spectrogram = generate_aiff_spectrogram(sample['file_path']) - spec_n = spectrogram.shape[0] - spec_w = spectrogram.shape[1] - spec = spectrogram.reshape(-1) - example = tf.train.Example(features=tf.train.Features( - feature={ - 'word': _bytes_feature([sample['word'].encode('utf-8')]), - 'phoneme': _bytes_feature([sample['phonemes'].encode('utf-8')]), - 'voice': _bytes_feature([sample['voice'].encode('utf-8')]), - 'language': _bytes_feature([sample['language'].encode('utf-8')]), - 'rate':_int64_feature([sample['rate']]), - 'variant': _bytes_feature([sample['variant'].encode('utf-8')]), - 'file': _bytes_feature([sample['file'].encode('utf-8')]), - 'spec':_float_feature(spec), - 'spec_n':_int64_feature([spec_n]), - 'spec_w':_int64_feature([spec_w]) - } - )) - writer.write(example.SerializeToString()) + # audio_samples = audio_samples[:100] + for (w, word_group) in audio_samples.groupby(audio_samples['word']): + g = word_group.reset_index() + g['spectrogram'] = apply_by_multiprocessing(g['file_path'],generate_aiff_spectrogram) + sample_right = g.loc[audio_samples['variant'] == 'low'] + sample_wrong = g.loc[audio_samples['variant'] == 'medium'] + same, diff = siamese_pairs(sample_right, sample_wrong) + groups = [([0,1],same),([1,0],diff)] + for (output,group) in groups: + for sample1,sample2 in group: + spectro1,spectro2 = sample1['spectrogram'],sample2['spectrogram'] + spec_n1,spec_n2 = spectro1.shape[0],spectro2.shape[0] + spec_w1,spec_w2 = spectro1.shape[1],spectro2.shape[1] + spec1,spec2 = spectro1.reshape(-1),spectro2.reshape(-1) + example = tf.train.Example(features=tf.train.Features( + feature={ + 'word': _bytes_feature([w.encode('utf-8')]), + 'phoneme1': _bytes_feature([sample1['phonemes'].encode('utf-8')]), + 'phoneme2': _bytes_feature([sample2['phonemes'].encode('utf-8')]), + 'voice1': _bytes_feature([sample1['voice'].encode('utf-8')]), + 'voice2': _bytes_feature([sample2['voice'].encode('utf-8')]), + 'language': _bytes_feature([sample1['language'].encode('utf-8')]), + 'rate1':_int64_feature([sample1['rate']]), + 'rate2':_int64_feature([sample2['rate']]), + 'variant1': _bytes_feature([sample1['variant'].encode('utf-8')]), + 'variant2': _bytes_feature([sample2['variant'].encode('utf-8')]), + 'file1': _bytes_feature([sample1['file'].encode('utf-8')]), + 'file2': _bytes_feature([sample2['file'].encode('utf-8')]), + 'spec1':_float_feature(spec1), + 'spec2':_float_feature(spec2), + 'spec_n1':_int64_feature([spec_n1]), + 'spec_w1':_int64_feature([spec_w1]), + 'spec_n2':_int64_feature([spec_n2]), + 'spec_w2':_int64_feature([spec_w2]), + 'output':_int64_feature(output) + } + )) + writer.write(example.SerializeToString()) writer.close() def create_tagged_data(audio_samples):