import pandas as pd from pandas_parallel import apply_by_multiprocessing # import dask as dd # import dask.dataframe as ddf import tensorflow as tf import numpy as np from spectro_gen import generate_aiff_spectrogram from sklearn.model_selection import train_test_split import itertools import os import random import csv import gc def get_siamese_pairs(groupF1, groupF2): group1 = [r for (i, r) in groupF1.iterrows()] group2 = [r for (i, r) in groupF2.iterrows()] diff = [(g1, g2) for g2 in group2 for g1 in group1] same = [i for i in itertools.combinations(group1, 2) ] + [i for i in itertools.combinations(group2, 2)] random.shuffle(same) random.shuffle(diff) # return (random.sample(same,10), random.sample(diff,10)) return same[:10],diff[:10] def siamese_pairs(rightGroup, wrongGroup): group1 = [r for (i, r) in rightGroup.iterrows()] group2 = [r for (i, r) in wrongGroup.iterrows()] rightWrongPairs = [(g1, g2) for g2 in group2 for g1 in group1] rightRightPairs = [i for i in itertools.combinations(group1, 2)] random.shuffle(rightWrongPairs) random.shuffle(rightRightPairs) # return (random.sample(same,10), random.sample(diff,10)) return rightRightPairs[:10],rightWrongPairs[:10] def append_zeros(spgr, max_samples): return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)], 'median') def padd_zeros(spgr, max_samples): return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)], 'constant') def to_onehot(a,class_count=2): # >>> a = np.array([1, 0, 3]) a_row_n = a.shape[0] b = np.zeros((a_row_n, class_count)) b[np.arange(a_row_n), a] = 1 return b def create_pair(l, r, max_samples): l_sample = padd_zeros(l, max_samples) r_sample = padd_zeros(r, max_samples) return np.asarray([l_sample, r_sample]) def create_test_pair(l, r, max_samples): l_sample = append_zeros(l, max_samples) r_sample = append_zeros(r, max_samples) return np.asarray([[l_sample, r_sample]]) def create_X(sp, max_samples): return create_pair(sp[0]['spectrogram'], sp[1]['spectrogram'], max_samples) # def get_word_pairs_data(word, max_samples): # audio_samples = pd.read_csv( # './outputs/audio.csv', # names=['word', 'voice', 'rate', 'variant', 'file']) # audio_samples = audio_samples.loc[audio_samples['word'] == # word].reset_index(drop=True) # audio_samples.loc[:, 'spectrogram'] = audio_samples.loc[:, 'file'].apply( # lambda x: 'outputs/audio/' + x).apply(generate_aiff_spectrogram) # max_samples = audio_samples['spectrogram'].apply( # lambda x: x.shape[0]).max() # same_data, diff_data = [], [] # for (w, g) in audio_samples.groupby(audio_samples['word']): # sample_norm = g.loc[audio_samples['variant'] == 'normal'] # sample_phon = g.loc[audio_samples['variant'] == 'phoneme'] # same, diff = get_siamese_pairs(sample_norm, sample_phon) # same_data.extend([create_X(s, max_samples) for s in same]) # diff_data.extend([create_X(d, max_samples) for d in diff]) # Y = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))]) # X = np.asarray(same_data + diff_data) # # tr_pairs, te_pairs, tr_y, te_y = train_test_split(X, Y, test_size=0.1) # return (X, Y) def create_spectrogram_data(audio_group='audio'): audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv' , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file'] , quoting=csv.QUOTE_NONE) # audio_samples = audio_samples.loc[audio_samples['word'] == # 'sunflowers'].reset_index(drop=True) audio_samples['file_paths'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x) audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_paths'], os.path.exists) audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index() audio_samples['spectrogram'] = apply_by_multiprocessing(audio_samples['file_paths'],generate_aiff_spectrogram)#.apply( audio_samples['window_count'] = audio_samples.loc[:,'spectrogram'].apply(lambda x: x.shape[0]) audio_samples.to_pickle('outputs/{}-spectrogram.pkl'.format(audio_group)) def create_spectrogram_tfrecords(audio_group='audio'): audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv' , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file'] , quoting=csv.QUOTE_NONE) # audio_samples = audio_samples.loc[audio_samples['word'] == # 'sunflowers'].reset_index(drop=True) audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x) audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists) audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index() def _float_feature(value): return tf.train.Feature(float_list=tf.train.FloatList(value=value)) def _int64_feature(value): return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) def _bytes_feature(value): return tf.train.Feature(bytes_list=tf.train.BytesList(value=value)) writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '.tfrecords') # audio_samples = audio_samples[:100] for (w, word_group) in audio_samples.groupby(audio_samples['word']): g = word_group.reset_index() g['spectrogram'] = apply_by_multiprocessing(g['file_path'],generate_aiff_spectrogram) sample_right = g.loc[audio_samples['variant'] == 'low'] sample_wrong = g.loc[audio_samples['variant'] == 'medium'] same, diff = siamese_pairs(sample_right, sample_wrong) groups = [([0,1],same),([1,0],diff)] for (output,group) in groups: for sample1,sample2 in group: spectro1,spectro2 = sample1['spectrogram'],sample2['spectrogram'] spec_n1,spec_n2 = spectro1.shape[0],spectro2.shape[0] spec_w1,spec_w2 = spectro1.shape[1],spectro2.shape[1] spec1,spec2 = spectro1.reshape(-1),spectro2.reshape(-1) example = tf.train.Example(features=tf.train.Features( feature={ 'word': _bytes_feature([w.encode('utf-8')]), 'phoneme1': _bytes_feature([sample1['phonemes'].encode('utf-8')]), 'phoneme2': _bytes_feature([sample2['phonemes'].encode('utf-8')]), 'voice1': _bytes_feature([sample1['voice'].encode('utf-8')]), 'voice2': _bytes_feature([sample2['voice'].encode('utf-8')]), 'language': _bytes_feature([sample1['language'].encode('utf-8')]), 'rate1':_int64_feature([sample1['rate']]), 'rate2':_int64_feature([sample2['rate']]), 'variant1': _bytes_feature([sample1['variant'].encode('utf-8')]), 'variant2': _bytes_feature([sample2['variant'].encode('utf-8')]), 'file1': _bytes_feature([sample1['file'].encode('utf-8')]), 'file2': _bytes_feature([sample2['file'].encode('utf-8')]), 'spec1':_float_feature(spec1), 'spec2':_float_feature(spec2), 'spec_n1':_int64_feature([spec_n1]), 'spec_w1':_int64_feature([spec_w1]), 'spec_n2':_int64_feature([spec_n2]), 'spec_w2':_int64_feature([spec_w2]), 'output':_int64_feature(output) } )) writer.write(example.SerializeToString()) writer.close() def create_tagged_data(audio_samples): same_data, diff_data = [], [] for (w, g) in audio_samples.groupby(audio_samples['word']): # sample_norm = g.loc[audio_samples['variant'] == 'low'] # sample_phon = g.loc[audio_samples['variant'] == 'medium'] sample_norm = g.loc[audio_samples['variant'] == 'normal'] sample_phon = g.loc[audio_samples['variant'] == 'phoneme'] same, diff = get_siamese_pairs(sample_norm, sample_phon) same_data.extend([create_X(s) for s in same]) diff_data.extend([create_X(d) for d in diff]) print('creating all speech pairs') Y_f = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))]) Y = to_onehot(Y_f.astype(np.int8)) print('casting as array speech pairs') X = np.asarray(same_data + diff_data) return X,Y def create_speech_pairs_data(audio_group='audio'): audio_samples = pd.read_pickle('outputs/{}-spectrogram.pkl'.format(audio_group)) # sample_size = audio_samples['spectrogram'][0].shape[1] tr_audio_samples,te_audio_samples = train_test_split(audio_samples, test_size=0.1) def save_samples_for(sample_name,samples): print('generating {} siamese speech pairs'.format(sample_name)) X,Y = create_tagged_data(samples) print('shuffling array speech pairs') rng_state = np.random.get_state() np.random.shuffle(X) np.random.set_state(rng_state) np.random.shuffle(Y) print('pickling X/Y') np.save('outputs/{}-train-X.npy'.format(audio_group), X) np.save('outputs/{}-train-Y.npy'.format(audio_group), Y) save_samples_for('train',tr_audio_samples) save_samples_for('test',te_audio_samples) def speech_data(audio_group='audio'): X = np.load('outputs/{}-X.npy'.format(audio_group)) / 255.0 Y = np.load('outputs/{}-Y.npy'.format(audio_group)) return (X,Y) def speech_model_data(): tr_pairs = np.load('outputs/tr_pairs.npy') / 255.0 te_pairs = np.load('outputs/te_pairs.npy') / 255.0 tr_pairs[tr_pairs < 0] = 0 te_pairs[te_pairs < 0] = 0 tr_y = np.load('outputs/tr_y.npy') te_y = np.load('outputs/te_y.npy') return tr_pairs, te_pairs, tr_y, te_y if __name__ == '__main__': # sunflower_pairs_data() # create_spectrogram_data() # create_spectrogram_data('story_words') create_spectrogram_tfrecords('story_words') # create_padded_spectrogram() # create_speech_pairs_data() # print(speech_model_data())