speech-scoring/speech_data.py

import pandas as pd
from pandas_parallel import apply_by_multiprocessing
# import dask as dd
# import dask.dataframe as ddf
import tensorflow as tf
import numpy as np
from spectro_gen import generate_aiff_spectrogram
from sklearn.model_selection import train_test_split
import itertools
import os
import random
import csv
import gc
import progressbar

def prog_bar(title):
    widgets = [title, progressbar.Counter(), ' [', progressbar.Bar(), '] - ', progressbar.ETA()]
    return progressbar.ProgressBar(widgets=widgets)

def siamese_pairs(rightGroup, wrongGroup):
    group1 = [r for (i, r) in rightGroup.iterrows()]
    group2 = [r for (i, r) in wrongGroup.iterrows()]
    rightWrongPairs = [(g1, g2) for g2 in group2 for g1 in group1]
    rightRightPairs = [i for i in itertools.combinations(group1, 2)]
    random.shuffle(rightWrongPairs)
    random.shuffle(rightRightPairs)
    # return (random.sample(same,10), random.sample(diff,10))
    # return rightRightPairs[:10],rightWrongPairs[:10]
    return rightRightPairs[:32],rightWrongPairs[:32]
    # return rightRightPairs,rightWrongPairs

def create_spectrogram_tfrecords(audio_group='audio'):
    '''
    http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
    http://www.machinelearninguru.com/deep_learning/tensorflow/basics/tfrecord/tfrecord.html
    '''
    audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv'
        , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
        , quoting=csv.QUOTE_NONE)
    # audio_samples = audio_samples.loc[audio_samples['word'] ==
    #                                   'sunflowers'].reset_index(drop=True)
    audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
    audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists)
    audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index()
    audio_samples['rate_int'] = apply_by_multiprocessing(audio_samples['rate'], str.isdigit)
    audio_samples = audio_samples[audio_samples['rate_int'] == True].reset_index().drop(['level_0'],axis=1)
    audio_samples['rate'] = audio_samples['rate'].astype(int)
    def _float_feature(value):
      return tf.train.Feature(float_list=tf.train.FloatList(value=value))

    def _int64_feature(value):
      return tf.train.Feature(int64_list=tf.train.Int64List(value=value))

    def _bytes_feature(value):
      return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))

    writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '.tfrecords')
    prog = prog_bar('Generating siamese pairs : ')
    for (w, word_group) in prog(audio_samples.groupby(audio_samples['word'])):
        g = word_group.reset_index()
        g['spectrogram'] = apply_by_multiprocessing(g['file_path'],generate_aiff_spectrogram)
        sample_right = g.loc[audio_samples['variant'] == 'low']
        sample_wrong = g.loc[audio_samples['variant'] == 'medium']
        same, diff = siamese_pairs(sample_right, sample_wrong)
        groups = [([0,1],same),([1,0],diff)]
        for (output,group) in groups:
            for sample1,sample2 in group:
                spectro1,spectro2 = sample1['spectrogram'],sample2['spectrogram']
                spec_n1,spec_n2 = spectro1.shape[0],spectro2.shape[0]
                spec_w1,spec_w2 = spectro1.shape[1],spectro2.shape[1]
                spec1,spec2 = spectro1.reshape(-1),spectro2.reshape(-1)
                example = tf.train.Example(features=tf.train.Features(
                  feature={
                    'word': _bytes_feature([w.encode('utf-8')]),
                    'phoneme1': _bytes_feature([sample1['phonemes'].encode('utf-8')]),
                    'phoneme2': _bytes_feature([sample2['phonemes'].encode('utf-8')]),
                    'voice1': _bytes_feature([sample1['voice'].encode('utf-8')]),
                    'voice2': _bytes_feature([sample2['voice'].encode('utf-8')]),
                    'language': _bytes_feature([sample1['language'].encode('utf-8')]),
                    'rate1':_int64_feature([sample1['rate']]),
                    'rate2':_int64_feature([sample2['rate']]),
                    'variant1': _bytes_feature([sample1['variant'].encode('utf-8')]),
                    'variant2': _bytes_feature([sample2['variant'].encode('utf-8')]),
                    'file1': _bytes_feature([sample1['file'].encode('utf-8')]),
                    'file2': _bytes_feature([sample2['file'].encode('utf-8')]),
                    'spec1':_float_feature(spec1),
                    'spec2':_float_feature(spec2),
                    'spec_n1':_int64_feature([spec_n1]),
                    'spec_w1':_int64_feature([spec_w1]),
                    'spec_n2':_int64_feature([spec_n2]),
                    'spec_w2':_int64_feature([spec_w2]),
                    'output':_int64_feature(output)
                  }
                ))
                writer.write(example.SerializeToString())
    writer.close()

def read_siamese_tfrecords(audio_group='audio'):
    records_file  = os.path.join('./outputs',audio_group+'.tfrecords')
    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
    input_pairs = []
    output_class = []
    input_words = []
    for string_record in record_iterator:
        example = tf.train.Example()
        example.ParseFromString(string_record)
        word = example.features.feature['word'].bytes_list.value[0]
        input_words.append(word)
        example.features.feature['spec2'].float_list.value[0]
        spec_n1 = example.features.feature['spec_n1'].int64_list.value[0]
        spec_n2 = example.features.feature['spec_n2'].int64_list.value[0]
        spec_w1 = example.features.feature['spec_w1'].int64_list.value[0]
        spec_w2 = example.features.feature['spec_w2'].int64_list.value[0]
        spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(spec_n1,spec_w1)
        spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(spec_n2,spec_w2)
        input_pairs.append([spec1,spec2])
        output = example.features.feature['output'].int64_list.value
        output_class.append(output)
    return input_pairs,output_class

def audio_samples_word_count(audio_group='audio'):
    audio_group = 'story_all'
    audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv'
        , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
        , quoting=csv.QUOTE_NONE)
    # audio_samples = audio_samples.loc[audio_samples['word'] ==
    #                                   'sunflowers'].reset_index(drop=True)
    audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
    audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists)
    audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index()
    return len(audio_samples.groupby(audio_samples['word']))

def fix_csv(audio_group='audio'):
    audio_group = 'story_all'
    audio_samples = pd.read_csv( './outputs/story_words.csv'
        , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
        , quoting=csv.QUOTE_NONE)
    voice_set = set(audio_samples['voice'].unique().tolist())
    audio_csv_lines = open('./outputs/' + audio_group + '.csv','r').readlines()
    audio_csv_data = [i.strip().split(',') for i in audio_csv_lines]
    to_be_fixed = [i for i in audio_csv_data if len(i) > 7]
    def unite_words(entries):
        entries = to_be_fixed[0]
        word_entries = next(((entries[:i],entries[i:])  for (i,e) in enumerate(entries) if e in voice_set),'')
        word_entries[1]
        return
    to_be_fixed[0]
    entries = [unite_words for e in to_be_fixed]
    [i for i in entries if len(i) % 2 != 0]

if __name__ == '__main__':
    # sunflower_pairs_data()
    # create_spectrogram_data()
    # create_spectrogram_data('story_words')
    # create_spectrogram_tfrecords('story_words')
    create_spectrogram_tfrecords('story_all')
    # create_padded_spectrogram()
    # create_speech_pairs_data()
    # print(speech_model_data())