speech-scoring/speech_data.py

import pandas as pd
from pandas_parallel import apply_by_multiprocessing
import dask as dd
import dask.dataframe as ddf
import numpy as np
from spectro_gen import generate_aiff_spectrogram
from sklearn.model_selection import train_test_split
import itertools
import os
import random
import csv
import gc

def get_siamese_pairs(groupF1, groupF2):
    group1 = [r for (i, r) in groupF1.iterrows()]
    group2 = [r for (i, r) in groupF2.iterrows()]
    diff = [(g1, g2) for g2 in group2 for g1 in group1]
    same = [i for i in itertools.combinations(group1, 2)
         ] + [i for i in itertools.combinations(group2, 2)]
    random.shuffle(same)
    random.shuffle(diff)
    # return (random.sample(same,10), random.sample(diff,10))
    return same[:10],diff[:10]


def append_zeros(spgr, max_samples):
    return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)],
                      'median')

def padd_zeros(spgr, max_samples):
    return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)],
                      'constant')

def to_onehot(a,class_count=2):
    # >>> a = np.array([1, 0, 3])
    a_row_n = a.shape[0]
    b = np.zeros((a_row_n, class_count))
    b[np.arange(a_row_n), a] = 1
    return b

def create_pair(l, r, max_samples):
    l_sample = padd_zeros(l, max_samples)
    r_sample = padd_zeros(r, max_samples)
    return np.asarray([l_sample, r_sample])


def create_test_pair(l, r, max_samples):
    l_sample = append_zeros(l, max_samples)
    r_sample = append_zeros(r, max_samples)
    return np.asarray([[l_sample, r_sample]])


def create_X(sp, max_samples):
    return create_pair(sp[0]['spectrogram'], sp[1]['spectrogram'], max_samples)


# def get_word_pairs_data(word, max_samples):
#     audio_samples = pd.read_csv(
#         './outputs/audio.csv',
#         names=['word', 'voice', 'rate', 'variant', 'file'])
#     audio_samples = audio_samples.loc[audio_samples['word'] ==
#                                       word].reset_index(drop=True)
#     audio_samples.loc[:, 'spectrogram'] = audio_samples.loc[:, 'file'].apply(
#         lambda x: 'outputs/audio/' + x).apply(generate_aiff_spectrogram)
#     max_samples = audio_samples['spectrogram'].apply(
#         lambda x: x.shape[0]).max()
#     same_data, diff_data = [], []
#     for (w, g) in audio_samples.groupby(audio_samples['word']):
#         sample_norm = g.loc[audio_samples['variant'] == 'normal']
#         sample_phon = g.loc[audio_samples['variant'] == 'phoneme']
#         same, diff = get_siamese_pairs(sample_norm, sample_phon)
#         same_data.extend([create_X(s, max_samples) for s in same])
#         diff_data.extend([create_X(d, max_samples) for d in diff])
#     Y = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))])
#     X = np.asarray(same_data + diff_data)
#     # tr_pairs, te_pairs, tr_y, te_y = train_test_split(X, Y, test_size=0.1)
#     return (X, Y)


def create_spectrogram_data(audio_group='audio'):
    audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv'
        , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
        , quoting=csv.QUOTE_NONE)
    # audio_samples = audio_samples.loc[audio_samples['word'] ==
    #                                   'sunflowers'].reset_index(drop=True)
    file_names = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
    audio_samples['file_exists'] = apply_by_multiprocessing(file_names,os.path.exists)
    audio_samples = audio_samples[audio_samples['file_exists'] == False]
    audio_samples['spectrogram'] = apply_by_multiprocessing(file_names,generate_aiff_spectrogram)#.apply(
            #generate_aiff_spectrogram)
    audio_samples['window_count'] = audio_samples.loc[:,'spectrogram'].apply(lambda x: x.shape[0])
    audio_samples.to_pickle('outputs/{}-spectrogram.pkl'.format(audio_group))


def create_tagged_data(audio_samples):
    same_data, diff_data = [], []
    for (w, g) in audio_samples.groupby(audio_samples['word']):
        # sample_norm = g.loc[audio_samples['variant'] == 'low']
        # sample_phon = g.loc[audio_samples['variant'] == 'medium']
        sample_norm = g.loc[audio_samples['variant'] == 'normal']
        sample_phon = g.loc[audio_samples['variant'] == 'phoneme']
        same, diff = get_siamese_pairs(sample_norm, sample_phon)
        same_data.extend([create_X(s) for s in same])
        diff_data.extend([create_X(d) for d in diff])
    print('creating all speech pairs')
    Y_f = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))])
    Y = to_onehot(Y_f.astype(np.int8))
    print('casting as array speech pairs')
    X = np.asarray(same_data + diff_data)
    return X,Y

def create_speech_pairs_data(audio_group='audio'):
    audio_samples = pd.read_pickle('outputs/{}-spectrogram.pkl'.format(audio_group))
    # sample_size = audio_samples['spectrogram'][0].shape[1]
    tr_audio_samples,te_audio_samples = train_test_split(audio_samples, test_size=0.1)
    def save_samples_for(sample_name,samples):
        print('generating {} siamese speech pairs'.format(sample_name))
        X,Y = create_tagged_data(samples)
        print('shuffling array speech pairs')
        rng_state = np.random.get_state()
        np.random.shuffle(X)
        np.random.set_state(rng_state)
        np.random.shuffle(Y)
        print('pickling X/Y')
        np.save('outputs/{}-train-X.npy'.format(audio_group), X)
        np.save('outputs/{}-train-Y.npy'.format(audio_group), Y)
    save_samples_for('train',tr_audio_samples)
    save_samples_for('test',te_audio_samples)

def speech_data(audio_group='audio'):
    X = np.load('outputs/{}-X.npy'.format(audio_group)) / 255.0
    Y = np.load('outputs/{}-Y.npy'.format(audio_group))
    return (X,Y)

def speech_model_data():
    tr_pairs = np.load('outputs/tr_pairs.npy') / 255.0
    te_pairs = np.load('outputs/te_pairs.npy') / 255.0
    tr_pairs[tr_pairs < 0] = 0
    te_pairs[te_pairs < 0] = 0
    tr_y = np.load('outputs/tr_y.npy')
    te_y = np.load('outputs/te_y.npy')
    return tr_pairs, te_pairs, tr_y, te_y


if __name__ == '__main__':
    # sunflower_pairs_data()
    # create_spectrogram_data()
    create_spectrogram_data('story_words')
    # create_padded_spectrogram()
    # create_speech_pairs_data()
    # print(speech_model_data())