import pandas as pd import numpy as np from spectro_gen import generate_aiff_spectrogram from sklearn.model_selection import train_test_split import itertools import pickle,gc def sunflower_data(): audio_samples = pd.read_csv('./outputs/audio.csv',names=['word','voice','rate','variant','file']) sunflowers = audio_samples.loc[audio_samples['word'] == 'sunflowers'].reset_index(drop=True) sunflowers.loc[:,'file'] = sunflowers.loc[:,'file'].apply(lambda x:'outputs/'+x).apply(generate_aiff_spectrogram) y_data = sunflowers['variant'].apply(lambda x:x=='normal').values max_samples = sunflowers['file'].apply(lambda x:x.shape[0]).max() sample_size = sunflowers['file'][0].shape[1] sample_count = sunflowers['file'].shape[0] sunflowers['file'][0].shape[0] def append_zeros(spgr): orig = spgr.shape[0] return np.lib.pad(spgr,[(0, max_samples-orig), (0,0)],'median') pad_sun = sunflowers['file'].apply(append_zeros).values x_data = np.vstack(pad_sun).reshape((sample_count,max_samples,sample_size,)) return train_test_split(x_data,y_data,test_size=0.33) def get_siamese_pairs(groupF1,groupF2): group1 = [r for (i,r) in groupF1.iterrows()] group2 = [r for (i,r) in groupF2.iterrows()] f = [(g1,g2) for g2 in group2 for g1 in group1] t = [i for i in itertools.combinations(group1,2)]+[i for i in itertools.combinations(group2,2)] return (t,f) def sunflower_pairs_data(): audio_samples = pd.read_csv('./outputs/audio.csv',names=['word','voice','rate','variant','file']) audio_samples = audio_samples.loc[audio_samples['word'] == 'sunflowers'].reset_index(drop=True) audio_samples.loc[:,'spectrogram'] = audio_samples.loc[:,'file'].apply(lambda x:'outputs/audio/'+x).apply(generate_aiff_spectrogram) max_samples = audio_samples['spectrogram'].apply(lambda x:x.shape[0]).max() sample_size = audio_samples['spectrogram'][0].shape[1] same_data,diff_data = [],[] for (w,g) in audio_samples.groupby(audio_samples['word']): sample_norm = g.loc[audio_samples['variant'] == 'normal'] sample_phon = g.loc[audio_samples['variant'] == 'phoneme'] same , diff = get_siamese_pairs(sample_norm,sample_phon) same_data.extend(same) diff_data.extend(diff) Y = np.hstack([np.ones(len(same_data)),np.zeros(len(diff_data))]) X_sample_pairs = same_data+diff_data def append_zeros(spgr): sample = np.lib.pad(spgr,[(0, max_samples-spgr.shape[0]), (0,0)],'median') return np.expand_dims(sample,axis=0) def create_X(sp): # sample_count = sp[0]['file'].shape[0] l_sample = append_zeros(sp[0]['spectrogram']) r_sample = append_zeros(sp[1]['spectrogram'])#.apply(append_zeros).values # x_data = np.vstack(pad_sun).reshape((sample_count,max_samples,sample_size)) return np.expand_dims(np.vstack([l_sample,r_sample]),axis=0) X_list = (create_X(sp) for sp in X_sample_pairs) X = np.vstack(X_list) tr_pairs,te_pairs,tr_y,te_y = train_test_split(X,Y,test_size=0.1) return train_test_split(X,Y,test_size=0.1) def create_spectrogram_data(audio_group='audio'): audio_samples = pd.read_csv('./outputs/'+audio_group+'.csv',names=['word','voice','rate','variant','file']) # audio_samples = audio_samples.loc[audio_samples['word'] == 'sunflowers'].reset_index(drop=True) audio_samples.loc[:,'spectrogram'] = audio_samples.loc[:,'file'].apply(lambda x:'outputs/'+audio_group+'/'+x).apply(generate_aiff_spectrogram) audio_samples.to_pickle('outputs/spectrogram.pkl') def create_speech_pairs_data(audio_group='audio'): audio_samples = pd.read_pickle('outputs/spectrogram.pkl') max_samples = audio_samples['spectrogram'].apply(lambda x:x.shape[0]).max() sample_size = audio_samples['spectrogram'][0].shape[1] def append_zeros(spgr): sample = np.lib.pad(spgr,[(0, max_samples-spgr.shape[0]), (0,0)],'median') return sample def create_X(sp): l_sample = append_zeros(sp[0]['spectrogram']) r_sample = append_zeros(sp[1]['spectrogram']) return np.asarray([l_sample,r_sample]) print('generating siamese speech pairs') same_data,diff_data = [],[] for (w,g) in audio_samples.groupby(audio_samples['word']): sample_norm = g.loc[audio_samples['variant'] == 'normal']#.reset_index(drop=True) sample_phon = g.loc[audio_samples['variant'] == 'phoneme']#.reset_index(drop=True) same , diff = get_siamese_pairs(sample_norm,sample_phon) same_data.extend([create_X(s) for s in same[:10]]) diff_data.extend([create_X(d) for d in diff[:10]]) print('creating all speech pairs') Y = np.hstack([np.ones(len(same_data)),np.zeros(len(diff_data))]) print('casting as array speech pairs') X = np.asarray(same_data+diff_data) print('pickling X/Y') np.save('outputs/X.npy',X) np.save('outputs/Y.npy',Y) del X gc.collect() print('train/test splitting speech pairs') tr_pairs,te_pairs,tr_y,te_y = train_test_split(X,Y,test_size=0.1) print('pickling train/test') np.save('outputs/tr_pairs.npy',tr_pairs) np.save('outputs/te_pairs.npy',te_pairs) np.save('outputs/tr_y.npy',tr_y) np.save('outputs/te_y.npy',te_y) # def create_speech_model_data(): # (max_samples,sample_size) = pickle.load(open('./spectrogram_vars.pkl','rb')) # x_data_pos = np.load('outputs/x_data_pos.npy') # x_data_neg = np.load('outputs/x_data_neg.npy') # x_pos_train, x_pos_test, x_neg_train, x_neg_test =train_test_split(x_data_pos,x_data_neg,test_size=0.1) # del x_data_pos # del x_data_neg # gc.collect() # print('split train and test') # tr_y = np.array(x_pos_train.shape[0]*[1]) # te_y = np.array(x_pos_test.shape[0]*[[1,0]]) # tr_pairs = np.array([x_pos_train,x_neg_train]).reshape(x_pos_train.shape[0],2,max_samples,sample_size) # te_pairs = np.array([x_pos_test,x_neg_test]).reshape(x_pos_test.shape[0],2,max_samples,sample_size) # print('reshaped to input dim') # np.save('outputs/tr_pairs.npy',tr_pairs) # np.save('outputs/te_pairs.npy',te_pairs) # np.save('outputs/tr_y.npy',tr_y) # np.save('outputs/te_y.npy',te_y) # print('pickled speech model data') def speech_model_data(): tr_pairs = np.load('outputs/tr_pairs.npy')/255.0 te_pairs = np.load('outputs/te_pairs.npy')/255.0 tr_pairs[tr_pairs < 0] = 0 te_pairs[te_pairs < 0] = 0 tr_y = np.load('outputs/tr_y.npy') te_y = np.load('outputs/te_y.npy') return tr_pairs,te_pairs,tr_y,te_y if __name__ == '__main__': # sunflower_pairs_data() #create_spectrogram_data() create_speech_pairs_data() # print(speech_model_data())