From 0a4d4fadebd82f754621c605974d0d59a611e1a1 Mon Sep 17 00:00:00 2001 From: Malar Kannan Date: Thu, 9 Nov 2017 15:00:17 +0530 Subject: [PATCH] implemented random sampling of data for oneshot loading --- speech_data.py | 217 ++++++++++++++++++++++++++++++++++++++++------ speech_siamese.py | 4 +- 2 files changed, 194 insertions(+), 27 deletions(-) diff --git a/speech_data.py b/speech_data.py index a9a87b4..26067c5 100644 --- a/speech_data.py +++ b/speech_data.py @@ -3,6 +3,7 @@ from pandas_parallel import apply_by_multiprocessing # import dask as dd # import dask.dataframe as ddf import tensorflow as tf +from tensorflow.python.ops import data_flow_ops import numpy as np from spectro_gen import generate_aiff_spectrogram from sklearn.model_selection import train_test_split @@ -11,6 +12,7 @@ import os import random import csv import gc +import pickle from tqdm import tqdm @@ -23,6 +25,16 @@ def siamese_pairs(rightGroup, wrongGroup): random.shuffle(rightRightPairs) return rightRightPairs[:32],rightWrongPairs[:32] + +def _float_feature(value): + return tf.train.Feature(float_list=tf.train.FloatList(value=value)) + +def _int64_feature(value): + return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) + +def _bytes_feature(value): + return tf.train.Feature(bytes_list=tf.train.BytesList(value=value)) + def create_spectrogram_tfrecords(audio_group='audio'): ''' http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/ @@ -35,14 +47,7 @@ def create_spectrogram_tfrecords(audio_group='audio'): audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists) audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index() - def _float_feature(value): - return tf.train.Feature(float_list=tf.train.FloatList(value=value)) - - def _int64_feature(value): - return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) - - def _bytes_feature(value): - return tf.train.Feature(bytes_list=tf.train.BytesList(value=value)) + n_records = n_spec = n_features = 0 writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '.tfrecords') prog = tqdm(audio_samples.groupby(audio_samples['word']),desc='Computing spectrogram') @@ -64,6 +69,11 @@ def create_spectrogram_tfrecords(audio_group='audio'): spec_n1,spec_n2 = spectro1.shape[0],spectro2.shape[0] spec_w1,spec_w2 = spectro1.shape[1],spectro2.shape[1] spec1,spec2 = spectro1.reshape(-1),spectro2.reshape(-1) + + n_spec = max([n_spec,spec_n1,spec_n2]) + n_features = spec_w1 + n_records+=1 + example = tf.train.Example(features=tf.train.Features( feature={ 'word': _bytes_feature([w.encode('utf-8')]), @@ -91,13 +101,15 @@ def create_spectrogram_tfrecords(audio_group='audio'): group_prog.close() prog.close() writer.close() + const_file = os.path.join('./outputs',audio_group+'.constants') + pickle.dump((n_spec,n_features,n_records),open(const_file,'wb')) def padd_zeros(spgr, max_samples): return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)], 'constant') def find_max_n(trf): - max_n = 0 + max_n,n_records = 0,0 max_n_it = tf.python_io.tf_record_iterator(path=trf) for string_record in max_n_it: example = tf.train.Example() @@ -105,19 +117,20 @@ def find_max_n(trf): spec_n1 = example.features.feature['spec_n1'].int64_list.value[0] spec_n2 = example.features.feature['spec_n2'].int64_list.value[0] max_n = max([max_n,spec_n1,spec_n2]) - return max_n + n_records+=1 + return (max_n,n_records) -def read_siamese_tfrecords(audio_group='audio'): +def padd_zeros_siamese_tfrecords(audio_group='audio'): records_file = os.path.join('./outputs',audio_group+'.tfrecords') record_iterator = tf.python_io.tf_record_iterator(path=records_file) - input_pairs = [] - output_class = [] - max_n = find_max_n(records_file) - spec_w1 = 0 - for string_record in record_iterator: + print('finding max_n...') + max_n,n_records = find_max_n(records_file) + p_spec1 = None + print('reading tfrecords...') + writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '_padded.tfrecords') + for string_record in tqdm(record_iterator,desc='padding siamese record',total=n_records): example = tf.train.Example() example.ParseFromString(string_record) - example.features.feature['spec2'].float_list.value[0] spec_n1 = example.features.feature['spec_n1'].int64_list.value[0] spec_n2 = example.features.feature['spec_n2'].int64_list.value[0] spec_w1 = example.features.feature['spec_w1'].int64_list.value[0] @@ -125,14 +138,155 @@ def read_siamese_tfrecords(audio_group='audio'): spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(spec_n1,spec_w1) spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(spec_n2,spec_w2) p_spec1,p_spec2 = padd_zeros(spec1,max_n),padd_zeros(spec2,max_n) - input_pairs.append(np.asarray([p_spec1,p_spec2])) output = example.features.feature['output'].int64_list.value - output_class.append(np.asarray(output)) - n_features = spec_w1 - input_data,output_data = np.asarray(input_pairs),np.asarray(output_class) - tr_pairs,te_pairs,tr_y,te_y = train_test_split(input_data,output_data) - n_step,n_features = int(max_n),int(spec_w1) - return (tr_pairs,te_pairs,tr_y,te_y,n_step,n_features) + w_example = tf.train.Example(features=tf.train.Features( + feature={ + 'spec1':_float_feature(p_spec1.reshape(-1)), + 'spec2':_float_feature(p_spec2.reshape(-1)), + 'output':_int64_feature(output) + } + )) + writer.write(w_example.SerializeToString()) + const_file = os.path.join('./outputs',audio_group+'.constants') + pickle.dump((max_n,p_spec1.shape[1],n_records),open(const_file,'wb')) + writer.close() + +def pickle_constants(audio_group='audio'): + records_file = os.path.join('./outputs',audio_group+'_padded.tfrecords') + record_iterator = tf.python_io.tf_record_iterator(path=records_file) + print('finding max_n...') + max_n,n_records = find_max_n(records_file) + spec1 = 0 + print('finding spec_w1...') + record_iterator = tf.python_io.tf_record_iterator(path=records_file) + for string_record in record_iterator: + example = tf.train.Example() + example.ParseFromString(string_record) + spec1 = len(example.features.feature['spec1'].float_list.value)//max_n + print('found spec_w1...') + break + const_file = os.path.join('./outputs',audio_group+'.constants') + print(max_n,spec1,n_records) + pickle.dump((max_n,spec1,n_records),open(const_file,'wb')) + +def reservoir_sample(iterable, k): + it = iter(iterable) + if not (k > 0): + raise ValueError("sample size must be positive") + + sample = list(itertools.islice(it, k)) # fill the reservoir + random.shuffle(sample) # if number of items less then *k* then + # return all items in random order. + for i, item in enumerate(it, start=k+1): + j = random.randrange(i) # random [0..i) + if j < k: + sample[j] = item # replace item with gradually decreasing probability + return sample + +def read_siamese_tfrecords_oneshot(audio_group='audio'): + records_file = os.path.join('./outputs',audio_group+'_padded.tfrecords') + record_iterator = tf.python_io.tf_record_iterator(path=records_file) + input_pairs = [] + output_class = [] + const_file = os.path.join('./outputs',audio_group+'.constants') + (n_spec,n_features,n_records) = pickle.load(open(const_file,'rb')) + print('reading tfrecords...') + samples = min([30000,n_records]) + input_data = np.zeros((samples,2,n_spec,n_features)) + output_data = np.zeros((samples,2)) + random_samples = enumerate(reservoir_sample(record_iterator,samples)) + for (i,string_record) in tqdm(random_samples,total=samples): + # if i == samples: + # break + example = tf.train.Example() + example.ParseFromString(string_record) + spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(n_spec,n_features) + spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(n_spec,n_features) + input_data[i] = np.asarray([spec1,spec2]) + output = example.features.feature['output'].int64_list.value + output_data[i] = np.asarray(output) + print('converting to nparray...') + tr_pairs,te_pairs,tr_y,te_y = train_test_split(input_data,output_data,test_size=0.1) + result = (tr_pairs,te_pairs,tr_y,te_y,n_spec,n_features) + return result + +def read_siamese_tfrecords(audio_group='audio'): + audio_group='story_words_test' + + record_file = os.path.join('./outputs',audio_group+'_padded.tfrecords') + const_file = os.path.join('./outputs',audio_group+'.constants') + (n_spec,n_features) = pickle.load(open(const_file,'rb')) + + filename_queue = tf.train.string_input_producer([record_file]) + reader = tf.TFRecordReader() + _, serialized_example = reader.read(filename_queue) + features = tf.parse_single_example(serialized_example, + features={ + 'spec1': tf.FixedLenFeature([1,n_spec,n_features], tf.float32), + 'spec2': tf.FixedLenFeature([1,n_spec,n_features], tf.float32), + 'output':tf.FixedLenFeature([2], tf.int64) + }) + spec1 = features['spec1'] + spec1 = tf.cast(spec1, tf.float32) * (1. / 255) + spec2 = features['spec2'] + spec2 = tf.cast(spec2, tf.float32) * (1. / 255) + output = tf.cast(features['output'], tf.int32) + return spec1,spec2, output,n_spec,n_features + +def read_siamese_tfrecords_batch(audio_group='audio', batch_size=32): + audio_group='story_words_test' + record_file = os.path.join('./outputs',audio_group+'_padded.tfrecords') + """ Return tensor to read from TFRecord """ + print('Creating graph for loading {} ...'.format(record_file)) + const_file = os.path.join('./outputs',audio_group+'.constants') + (n_spec,n_features) = pickle.load(open(const_file,'rb')) + records_file = os.path.join('./outputs',audio_group+'.tfrecords') + record_iterator = tf.python_io.tf_record_iterator(path=records_file) + n_records = len([i for i in record_iterator]) + batch_shape=[batch_size, n_spec, n_features] + with tf.variable_scope("SiameseTFRecords"): + record_input = data_flow_ops.RecordInput(record_file, batch_size=batch_size) + records_op = record_input.get_yield_op() + records_op = tf.split(records_op, batch_shape[0], 0) + records_op = [tf.reshape(record, []) for record in records_op] + specs1, specs2 = [],[] + outputs = [] + for i, serialized_example in tqdm(enumerate(records_op)): + with tf.variable_scope("parse_siamese_pairs", reuse=True): + features = tf.parse_single_example( + serialized_example, + features={ + 'spec1': tf.FixedLenFeature([n_spec,n_features], tf.float32), + 'spec2': tf.FixedLenFeature([n_spec,n_features], tf.float32), + 'output':tf.FixedLenFeature([2], tf.int64) + }) + spec1 = features['spec1'] + spec1 = tf.cast(spec1, tf.float32) * (1. / 255) + spec2 = features['spec2'] + output = tf.cast(spec2, tf.float32) * (1. / 255) + output = tf.cast(features['output'], tf.float32) + specs1.append(spec1) + specs2.append(spec2) + outputs.append(output) + + specs1 = tf.parallel_stack(specs1, 0) + specs2 = tf.parallel_stack(specs2, 0) + outputs = tf.parallel_stack(outputs, 0) + specs1 = tf.cast(specs1, tf.float32) + specs2 = tf.cast(specs2, tf.float32) + + specs1 = tf.reshape(specs1, shape=batch_shape) + specs2 = tf.reshape(specs1, shape=batch_shape) + specs1_shape = specs1.get_shape() + specs2_shape = specs2.get_shape() + outputs_shape = outputs.get_shape() + copy_stage = data_flow_ops.StagingArea( + [tf.float32, tf.float32, tf.float32], + shapes=[specs1_shape, specs2_shape, outputs_shape]) + copy_stage_op = copy_stage.put( + [specs1, specs2, outputs]) + staged_specs1, staged_specs2, staged_outputs = copy_stage.get() + return specs1, spec2, outputs,n_spec,n_features,n_records def audio_samples_word_count(audio_group='audio'): audio_group = 'story_all' @@ -152,14 +306,27 @@ def fix_csv(audio_group='audio'): fixed_csv_w = csv.writer(fixed_csv, quoting=csv.QUOTE_MINIMAL) fixed_csv_w.writerows(proper_rows) +def convert_old_audio(): + audio_samples = pd.read_csv( './outputs/audio.csv.old' + , names=['word', 'voice', 'rate', 'variant', 'file']) + audio_samples['phonemes'] = 'unknown' + audio_samples['language'] = 'en-US' + audio_samples.loc[audio_samples['variant'] == 'normal','variant'] = 'low' + audio_samples.loc[audio_samples['variant'] == 'phoneme','variant'] = 'medium' + audio_samples = audio_samples[['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']] + audio_samples.to_csv('./outputs/audio_new.csv',index=False,header=False) if __name__ == '__main__': # sunflower_pairs_data() # create_spectrogram_data() # create_spectrogram_data('story_words') - create_spectrogram_tfrecords('story_words') + # create_spectrogram_tfrecords('story_words') # create_spectrogram_tfrecords('story_words_test') # read_siamese_tfrecords('story_all') + # read_siamese_tfrecords('story_words_test') + pickle_constants('story_words_test') + # create_spectrogram_tfrecords('audio') + # padd_zeros_siamese_tfrecords('audio') # create_padded_spectrogram() # create_speech_pairs_data() # print(speech_model_data()) diff --git a/speech_siamese.py b/speech_siamese.py index 9ed9591..a2cdba3 100644 --- a/speech_siamese.py +++ b/speech_siamese.py @@ -2,7 +2,7 @@ from __future__ import absolute_import from __future__ import print_function import numpy as np # from speech_data import speech_model_data -from speech_data import read_siamese_tfrecords +from speech_data import read_siamese_tfrecords_oneshot from keras.models import Model,load_model from keras.layers import Input, Dense, Dropout, LSTM, Lambda, Concatenate from keras.losses import categorical_crossentropy @@ -82,7 +82,7 @@ def siamese_model(input_dim): def train_siamese(): # the data, shuffled and split between train and test sets # tr_pairs, te_pairs, tr_y_e, te_y_e = speech_model_data() - (tr_pairs,te_pairs,tr_y,te_y,n_step,n_features) = read_siamese_tfrecords('story_words_test') + (tr_pairs,te_pairs,tr_y,te_y,n_step,n_features) = read_siamese_tfrecords_oneshot() # tr_y = to_categorical(tr_y_e, num_classes=2) # te_y = to_categorical(te_y_e, num_classes=2) input_dim = (n_step, n_features)