speech-scoring/speech_data.py

import pandas as pd
from pandas_parallel import apply_by_multiprocessing
# import dask as dd
# import dask.dataframe as ddf
import tensorflow as tf
from tensorflow.python.ops import data_flow_ops
import numpy as np
from spectro_gen import generate_aiff_spectrogram
from sklearn.model_selection import train_test_split
import itertools
import os
import random
import csv
import gc
import pickle
from tqdm import tqdm


def siamese_pairs(rightGroup, wrongGroup):
    group1 = [r for (i, r) in rightGroup.iterrows()]
    group2 = [r for (i, r) in wrongGroup.iterrows()]
    rightWrongPairs = [(g1, g2) for g2 in group2 for g1 in group1]
    rightRightPairs = [i for i in itertools.combinations(group1, 2)]
    random.shuffle(rightWrongPairs)
    random.shuffle(rightRightPairs)
    return rightRightPairs[:32],rightWrongPairs[:32]


def _float_feature(value):
  return tf.train.Feature(float_list=tf.train.FloatList(value=value))

def _int64_feature(value):
  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))

def _bytes_feature(value):
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))

def create_spectrogram_tfrecords(audio_group='audio'):
    '''
    http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
    http://www.machinelearninguru.com/deep_learning/tensorflow/basics/tfrecord/tfrecord.html
    '''
    audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv'
        , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
        , quoting=csv.QUOTE_NONE)
    audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
    audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists)
    audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index()

    n_records = n_spec = n_features = 0

    writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '.tfrecords')
    prog = tqdm(audio_samples.groupby(audio_samples['word']),desc='Computing spectrogram')
    for (w, word_group) in prog:
        prog.set_postfix(word=w)
        g = word_group.reset_index()
        g['spectrogram'] = apply_by_multiprocessing(g['file_path'],generate_aiff_spectrogram)
        sample_right = g.loc[g['variant'] == 'low']
        sample_wrong = g.loc[g['variant'] == 'medium']
        same, diff = siamese_pairs(sample_right, sample_wrong)
        groups = [([0,1],same),([1,0],diff)]
        for (output,group) in groups:
            group_prog = tqdm(group,desc='Writing Spectrogram')
            for sample1,sample2 in group_prog:
                group_prog.set_postfix(output=output
                    ,var1=sample1['variant']
                    ,var2=sample2['variant'])
                spectro1,spectro2 = sample1['spectrogram'],sample2['spectrogram']
                spec_n1,spec_n2 = spectro1.shape[0],spectro2.shape[0]
                spec_w1,spec_w2 = spectro1.shape[1],spectro2.shape[1]
                spec1,spec2 = spectro1.reshape(-1),spectro2.reshape(-1)

                n_spec = max([n_spec,spec_n1,spec_n2])
                n_features = spec_w1
                n_records+=1

                example = tf.train.Example(features=tf.train.Features(
                  feature={
                    'word': _bytes_feature([w.encode('utf-8')]),
                    'phoneme1': _bytes_feature([sample1['phonemes'].encode('utf-8')]),
                    'phoneme2': _bytes_feature([sample2['phonemes'].encode('utf-8')]),
                    'voice1': _bytes_feature([sample1['voice'].encode('utf-8')]),
                    'voice2': _bytes_feature([sample2['voice'].encode('utf-8')]),
                    'language': _bytes_feature([sample1['language'].encode('utf-8')]),
                    'rate1':_int64_feature([sample1['rate']]),
                    'rate2':_int64_feature([sample2['rate']]),
                    'variant1': _bytes_feature([sample1['variant'].encode('utf-8')]),
                    'variant2': _bytes_feature([sample2['variant'].encode('utf-8')]),
                    'file1': _bytes_feature([sample1['file'].encode('utf-8')]),
                    'file2': _bytes_feature([sample2['file'].encode('utf-8')]),
                    'spec1':_float_feature(spec1),
                    'spec2':_float_feature(spec2),
                    'spec_n1':_int64_feature([spec_n1]),
                    'spec_w1':_int64_feature([spec_w1]),
                    'spec_n2':_int64_feature([spec_n2]),
                    'spec_w2':_int64_feature([spec_w2]),
                    'output':_int64_feature(output)
                  }
                ))
                writer.write(example.SerializeToString())
            group_prog.close()
    prog.close()
    writer.close()
    const_file = os.path.join('./outputs',audio_group+'.constants')
    pickle.dump((n_spec,n_features,n_records),open(const_file,'wb'))

def padd_zeros(spgr, max_samples):
    return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)],
                      'constant')

def find_max_n(trf):
    max_n,n_records = 0,0
    max_n_it = tf.python_io.tf_record_iterator(path=trf)
    for string_record in max_n_it:
        example = tf.train.Example()
        example.ParseFromString(string_record)
        spec_n1 = example.features.feature['spec_n1'].int64_list.value[0]
        spec_n2 = example.features.feature['spec_n2'].int64_list.value[0]
        max_n = max([max_n,spec_n1,spec_n2])
        n_records+=1
    return (max_n,n_records)

def padd_zeros_siamese_tfrecords(audio_group='audio'):
    records_file  = os.path.join('./outputs',audio_group+'.tfrecords')
    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
    print('finding max_n...')
    max_n,n_records = find_max_n(records_file)
    p_spec1 = None
    print('reading tfrecords...')
    writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '_padded.tfrecords')
    for string_record in tqdm(record_iterator,desc='padding siamese record',total=n_records):
        example = tf.train.Example()
        example.ParseFromString(string_record)
        spec_n1 = example.features.feature['spec_n1'].int64_list.value[0]
        spec_n2 = example.features.feature['spec_n2'].int64_list.value[0]
        spec_w1 = example.features.feature['spec_w1'].int64_list.value[0]
        spec_w2 = example.features.feature['spec_w2'].int64_list.value[0]
        spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(spec_n1,spec_w1)
        spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(spec_n2,spec_w2)
        p_spec1,p_spec2 = padd_zeros(spec1,max_n),padd_zeros(spec2,max_n)
        output = example.features.feature['output'].int64_list.value
        w_example = tf.train.Example(features=tf.train.Features(
          feature={
            'spec1':_float_feature(p_spec1.reshape(-1)),
            'spec2':_float_feature(p_spec2.reshape(-1)),
            'output':_int64_feature(output)
          }
        ))
        writer.write(w_example.SerializeToString())
    const_file = os.path.join('./outputs',audio_group+'.constants')
    pickle.dump((max_n,p_spec1.shape[1],n_records),open(const_file,'wb'))
    writer.close()

def pickle_constants(audio_group='audio'):
    records_file  = os.path.join('./outputs',audio_group+'_padded.tfrecords')
    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
    print('finding max_n...')
    max_n,n_records = find_max_n(records_file)
    spec1 = 0
    print('finding spec_w1...')
    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
    for string_record in record_iterator:
        example = tf.train.Example()
        example.ParseFromString(string_record)
        spec1 = len(example.features.feature['spec1'].float_list.value)//max_n
        print('found spec_w1...')
        break
    const_file = os.path.join('./outputs',audio_group+'.constants')
    print(max_n,spec1,n_records)
    pickle.dump((max_n,spec1,n_records),open(const_file,'wb'))

def reservoir_sample(iterable, k):
    it = iter(iterable)
    if not (k > 0):
        raise ValueError("sample size must be positive")

    sample = list(itertools.islice(it, k)) # fill the reservoir
    random.shuffle(sample) # if number of items less then *k* then
                           #   return all items in random order.
    for i, item in enumerate(it, start=k+1):
        j = random.randrange(i) # random [0..i)
        if j < k:
            sample[j] = item # replace item with gradually decreasing probability
    return sample

def read_siamese_tfrecords_oneshot(audio_group='audio'):
    records_file  = os.path.join('./outputs',audio_group+'_padded.tfrecords')
    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
    input_pairs = []
    output_class = []
    const_file = os.path.join('./outputs',audio_group+'.constants')
    (n_spec,n_features,n_records) = pickle.load(open(const_file,'rb'))
    print('reading tfrecords...')
    samples = min([30000,n_records])
    input_data = np.zeros((samples,2,n_spec,n_features))
    output_data = np.zeros((samples,2))
    random_samples = enumerate(reservoir_sample(record_iterator,samples))
    for (i,string_record) in tqdm(random_samples,total=samples):
        # if i == samples:
        #     break
        example = tf.train.Example()
        example.ParseFromString(string_record)
        spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(n_spec,n_features)
        spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(n_spec,n_features)
        input_data[i] = np.asarray([spec1,spec2])
        output = example.features.feature['output'].int64_list.value
        output_data[i] = np.asarray(output)
    print('converting to nparray...')
    tr_pairs,te_pairs,tr_y,te_y = train_test_split(input_data,output_data,test_size=0.1)
    result = (tr_pairs,te_pairs,tr_y,te_y,n_spec,n_features)
    return result

def read_siamese_tfrecords(audio_group='audio'):
    audio_group='story_words_test'

    record_file = os.path.join('./outputs',audio_group+'_padded.tfrecords')
    const_file = os.path.join('./outputs',audio_group+'.constants')
    (n_spec,n_features) = pickle.load(open(const_file,'rb'))

    filename_queue = tf.train.string_input_producer([record_file])
    reader = tf.TFRecordReader()
    _, serialized_example = reader.read(filename_queue)
    features = tf.parse_single_example(serialized_example,
                    features={
                        'spec1': tf.FixedLenFeature([1,n_spec,n_features], tf.float32),
                        'spec2': tf.FixedLenFeature([1,n_spec,n_features], tf.float32),
                        'output':tf.FixedLenFeature([2], tf.int64)
                    })
    spec1 = features['spec1']
    spec1 = tf.cast(spec1, tf.float32) * (1. / 255)
    spec2 = features['spec2']
    spec2 = tf.cast(spec2, tf.float32) * (1. / 255)
    output = tf.cast(features['output'], tf.int32)
    return spec1,spec2, output,n_spec,n_features

def read_siamese_tfrecords_batch(audio_group='audio', batch_size=32):
    audio_group='story_words_test'
    record_file = os.path.join('./outputs',audio_group+'_padded.tfrecords')
    """ Return tensor to read from TFRecord """
    print('Creating graph for loading {} ...'.format(record_file))
    const_file = os.path.join('./outputs',audio_group+'.constants')
    (n_spec,n_features) = pickle.load(open(const_file,'rb'))
    records_file  = os.path.join('./outputs',audio_group+'.tfrecords')
    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
    n_records = len([i for i in record_iterator])
    batch_shape=[batch_size, n_spec, n_features]
    with tf.variable_scope("SiameseTFRecords"):
        record_input = data_flow_ops.RecordInput(record_file, batch_size=batch_size)
        records_op = record_input.get_yield_op()
        records_op = tf.split(records_op, batch_shape[0], 0)
        records_op = [tf.reshape(record, []) for record in records_op]
        specs1, specs2 = [],[]
        outputs = []
        for i, serialized_example in tqdm(enumerate(records_op)):
            with tf.variable_scope("parse_siamese_pairs", reuse=True):
                features = tf.parse_single_example(
                    serialized_example,
                    features={
                        'spec1': tf.FixedLenFeature([n_spec,n_features], tf.float32),
                        'spec2': tf.FixedLenFeature([n_spec,n_features], tf.float32),
                        'output':tf.FixedLenFeature([2], tf.int64)
                    })
                spec1 = features['spec1']
                spec1 = tf.cast(spec1, tf.float32) * (1. / 255)
                spec2 = features['spec2']
                output = tf.cast(spec2, tf.float32) * (1. / 255)
                output = tf.cast(features['output'], tf.float32)
                specs1.append(spec1)
                specs2.append(spec2)
                outputs.append(output)

        specs1 = tf.parallel_stack(specs1, 0)
        specs2 = tf.parallel_stack(specs2, 0)
        outputs = tf.parallel_stack(outputs, 0)
        specs1 = tf.cast(specs1, tf.float32)
        specs2 = tf.cast(specs2, tf.float32)

        specs1 = tf.reshape(specs1, shape=batch_shape)
        specs2 = tf.reshape(specs1, shape=batch_shape)
        specs1_shape = specs1.get_shape()
        specs2_shape = specs2.get_shape()
        outputs_shape = outputs.get_shape()
        copy_stage = data_flow_ops.StagingArea(
            [tf.float32, tf.float32, tf.float32],
            shapes=[specs1_shape, specs2_shape, outputs_shape])
        copy_stage_op = copy_stage.put(
            [specs1, specs2, outputs])
        staged_specs1, staged_specs2, staged_outputs = copy_stage.get()
        return specs1, spec2, outputs,n_spec,n_features,n_records

def audio_samples_word_count(audio_group='audio'):
    audio_group = 'story_all'
    audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv'
        , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
        , quoting=csv.QUOTE_NONE)
    audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
    audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists)
    audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index()
    return len(audio_samples.groupby(audio_samples['word']))

def fix_csv(audio_group='audio'):
    audio_csv_lines = open('./outputs/' + audio_group + '.csv','r').readlines()
    audio_csv_data = [i.strip().split(',') for i in audio_csv_lines]
    proper_rows = [i for i in audio_csv_data if len(i) == 7]
    with open('./outputs/' + audio_group + '-new.csv','w') as fixed_csv:
        fixed_csv_w = csv.writer(fixed_csv, quoting=csv.QUOTE_MINIMAL)
        fixed_csv_w.writerows(proper_rows)

def convert_old_audio():
    audio_samples = pd.read_csv( './outputs/audio.csv.old'
        , names=['word', 'voice', 'rate', 'variant', 'file'])
    audio_samples['phonemes'] = 'unknown'
    audio_samples['language'] = 'en-US'
    audio_samples.loc[audio_samples['variant'] == 'normal','variant'] = 'low'
    audio_samples.loc[audio_samples['variant'] == 'phoneme','variant'] = 'medium'
    audio_samples = audio_samples[['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']]
    audio_samples.to_csv('./outputs/audio_new.csv',index=False,header=False)

if __name__ == '__main__':
    # sunflower_pairs_data()
    # create_spectrogram_data()
    # create_spectrogram_data('story_words')
    # create_spectrogram_tfrecords('story_words')
    # create_spectrogram_tfrecords('story_words_test')
    # read_siamese_tfrecords('story_all')
    # read_siamese_tfrecords('story_words_test')
    pickle_constants('story_words_test')
    # create_spectrogram_tfrecords('audio')
    # padd_zeros_siamese_tfrecords('audio')
    # create_padded_spectrogram()
    # create_speech_pairs_data()
    # print(speech_model_data())