speech-scoring/speech_data.py

import pandas as pd
import numpy as np
from spectro_gen import generate_aiff_spectrogram
from sklearn.model_selection import train_test_split
import itertools
import pickle,gc

def sunflower_data():
    audio_samples = pd.read_csv('./outputs/audio.csv',names=['word','voice','rate','variant','file'])
    sunflowers = audio_samples.loc[audio_samples['word'] == 'sunflowers'].reset_index(drop=True)
    sunflowers.loc[:,'file'] = sunflowers.loc[:,'file'].apply(lambda x:'outputs/'+x).apply(generate_aiff_spectrogram)
    y_data = sunflowers['variant'].apply(lambda x:x=='normal').values
    max_samples = sunflowers['file'].apply(lambda x:x.shape[0]).max()
    sample_size = sunflowers['file'][0].shape[1]
    sample_count = sunflowers['file'].shape[0]
    sunflowers['file'][0].shape[0]
    def append_zeros(spgr):
        orig = spgr.shape[0]
        return np.lib.pad(spgr,[(0, max_samples-orig), (0,0)],'median')
    pad_sun = sunflowers['file'].apply(append_zeros).values
    x_data = np.vstack(pad_sun).reshape((sample_count,max_samples,sample_size,))
    return train_test_split(x_data,y_data,test_size=0.33)

def get_siamese_pairs(groupF1,groupF2):
    group1 = [r for (i,r) in groupF1.iterrows()]
    group2 = [r for (i,r) in groupF2.iterrows()]
    f = [(g1,g2) for g2 in group2 for g1 in group1]
    t = [i for i in itertools.combinations(group1,2)]+[i for i in itertools.combinations(group2,2)]
    return (t,f)

def sunflower_pairs_data():
    audio_samples = pd.read_csv('./outputs/audio.csv',names=['word','voice','rate','variant','file'])
    audio_samples = audio_samples.loc[audio_samples['word'] == 'sunflowers'].reset_index(drop=True)
    audio_samples.loc[:,'spectrogram'] = audio_samples.loc[:,'file'].apply(lambda x:'outputs/audio/'+x).apply(generate_aiff_spectrogram)
    max_samples = audio_samples['spectrogram'].apply(lambda x:x.shape[0]).max()
    sample_size = audio_samples['spectrogram'][0].shape[1]
    same_data,diff_data = [],[]
    for (w,g) in audio_samples.groupby(audio_samples['word']):
        sample_norm = g.loc[audio_samples['variant'] == 'normal']
        sample_phon = g.loc[audio_samples['variant'] == 'phoneme']
        same , diff = get_siamese_pairs(sample_norm,sample_phon)
        same_data.extend(same)
        diff_data.extend(diff)
    Y = np.hstack([np.ones(len(same_data)),np.zeros(len(diff_data))])
    X_sample_pairs = same_data+diff_data
    def append_zeros(spgr):
        sample = np.lib.pad(spgr,[(0, max_samples-spgr.shape[0]), (0,0)],'median')
        return np.expand_dims(sample,axis=0)
    def create_X(sp):
        # sample_count = sp[0]['file'].shape[0]
        l_sample = append_zeros(sp[0]['spectrogram'])
        r_sample = append_zeros(sp[1]['spectrogram'])#.apply(append_zeros).values
        # x_data = np.vstack(pad_sun).reshape((sample_count,max_samples,sample_size))
        return np.expand_dims(np.vstack([l_sample,r_sample]),axis=0)
    X_list = (create_X(sp) for sp in X_sample_pairs)
    X = np.vstack(X_list)
    tr_pairs,te_pairs,tr_y,te_y = train_test_split(X,Y,test_size=0.1)
    return train_test_split(X,Y,test_size=0.1)

def create_spectrogram_data(audio_group='audio'):
    audio_samples = pd.read_csv('./outputs/'+audio_group+'.csv',names=['word','voice','rate','variant','file'])
    # audio_samples = audio_samples.loc[audio_samples['word'] == 'sunflowers'].reset_index(drop=True)
    audio_samples.loc[:,'spectrogram'] = audio_samples.loc[:,'file'].apply(lambda x:'outputs/'+audio_group+'/'+x).apply(generate_aiff_spectrogram)
    audio_samples.to_pickle('outputs/spectrogram.pkl')

def create_speech_pairs_data(audio_group='audio'):
    audio_samples = pd.read_pickle('outputs/spectrogram.pkl')
    max_samples = audio_samples['spectrogram'].apply(lambda x:x.shape[0]).max()
    sample_size = audio_samples['spectrogram'][0].shape[1]

    def append_zeros(spgr):
        sample = np.lib.pad(spgr,[(0, max_samples-spgr.shape[0]), (0,0)],'median')
        return sample
    def create_X(sp):
        l_sample = append_zeros(sp[0]['spectrogram'])
        r_sample = append_zeros(sp[1]['spectrogram'])
        return np.asarray([l_sample,r_sample])

    print('generating siamese speech pairs')
    same_data,diff_data = [],[]
    for (w,g) in audio_samples.groupby(audio_samples['word']):
        sample_norm = g.loc[audio_samples['variant'] == 'normal']#.reset_index(drop=True)
        sample_phon = g.loc[audio_samples['variant'] == 'phoneme']#.reset_index(drop=True)
        same , diff = get_siamese_pairs(sample_norm,sample_phon)
        same_data.extend([create_X(s) for s in same[:10]])
        diff_data.extend([create_X(d) for d in diff[:10]])
    print('creating all speech pairs')
    Y = np.hstack([np.ones(len(same_data)),np.zeros(len(diff_data))])
    print('casting as array speech pairs')
    X = np.asarray(same_data+diff_data)
    print('pickling X/Y')
    np.save('outputs/X.npy',X)
    np.save('outputs/Y.npy',Y)
    del X
    gc.collect()
    print('train/test splitting speech pairs')
    tr_pairs,te_pairs,tr_y,te_y = train_test_split(X,Y,test_size=0.1)
    print('pickling train/test')
    np.save('outputs/tr_pairs.npy',tr_pairs)
    np.save('outputs/te_pairs.npy',te_pairs)
    np.save('outputs/tr_y.npy',tr_y)
    np.save('outputs/te_y.npy',te_y)

# def create_speech_model_data():
#     (max_samples,sample_size) = pickle.load(open('./spectrogram_vars.pkl','rb'))
#     x_data_pos = np.load('outputs/x_data_pos.npy')
#     x_data_neg = np.load('outputs/x_data_neg.npy')
#     x_pos_train, x_pos_test, x_neg_train, x_neg_test =train_test_split(x_data_pos,x_data_neg,test_size=0.1)
#     del x_data_pos
#     del x_data_neg
#     gc.collect()
#     print('split train and test')
#     tr_y = np.array(x_pos_train.shape[0]*[1])
#     te_y = np.array(x_pos_test.shape[0]*[[1,0]])
#     tr_pairs = np.array([x_pos_train,x_neg_train]).reshape(x_pos_train.shape[0],2,max_samples,sample_size)
#     te_pairs = np.array([x_pos_test,x_neg_test]).reshape(x_pos_test.shape[0],2,max_samples,sample_size)
#     print('reshaped to input dim')
#     np.save('outputs/tr_pairs.npy',tr_pairs)
#     np.save('outputs/te_pairs.npy',te_pairs)
#     np.save('outputs/tr_y.npy',tr_y)
#     np.save('outputs/te_y.npy',te_y)
#     print('pickled speech model data')

def speech_model_data():
    tr_pairs = np.load('outputs/tr_pairs.npy')/255.0
    te_pairs = np.load('outputs/te_pairs.npy')/255.0
    tr_pairs[tr_pairs < 0] = 0
    te_pairs[te_pairs < 0] = 0
    tr_y = np.load('outputs/tr_y.npy')
    te_y = np.load('outputs/te_y.npy')
    return tr_pairs,te_pairs,tr_y,te_y

if __name__ == '__main__':
    # sunflower_pairs_data()
    #create_spectrogram_data()
    create_speech_pairs_data()
    # print(speech_model_data())