implemented random sampling of data for oneshot loading

speech_data.py

@@ -3,6 +3,7 @@ from pandas_parallel import apply_by_multiprocessing
 # import dask as dd
 # import dask.dataframe as ddf
 import tensorflow as tf
+from tensorflow.python.ops import data_flow_ops
 import numpy as np
 from spectro_gen import generate_aiff_spectrogram
 from sklearn.model_selection import train_test_split
@@ -11,6 +12,7 @@ import os
 import random
 import csv
 import gc
+import pickle
 from tqdm import tqdm
 
 
@@ -23,6 +25,16 @@ def siamese_pairs(rightGroup, wrongGroup):
     random.shuffle(rightRightPairs)
     return rightRightPairs[:32],rightWrongPairs[:32]
 
+
+def _float_feature(value):
+  return tf.train.Feature(float_list=tf.train.FloatList(value=value))
+
+def _int64_feature(value):
+  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
+
+def _bytes_feature(value):
+  return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
+
 def create_spectrogram_tfrecords(audio_group='audio'):
     '''
     http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
@@ -35,14 +47,7 @@ def create_spectrogram_tfrecords(audio_group='audio'):
     audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists)
     audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index()
 
-    def _float_feature(value):
+    n_records = n_spec = n_features = 0
-      return tf.train.Feature(float_list=tf.train.FloatList(value=value))
-
-    def _int64_feature(value):
-      return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
-
-    def _bytes_feature(value):
-      return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
 
     writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '.tfrecords')
     prog = tqdm(audio_samples.groupby(audio_samples['word']),desc='Computing spectrogram')
@@ -64,6 +69,11 @@ def create_spectrogram_tfrecords(audio_group='audio'):
                 spec_n1,spec_n2 = spectro1.shape[0],spectro2.shape[0]
                 spec_w1,spec_w2 = spectro1.shape[1],spectro2.shape[1]
                 spec1,spec2 = spectro1.reshape(-1),spectro2.reshape(-1)
+
+                n_spec = max([n_spec,spec_n1,spec_n2])
+                n_features = spec_w1
+                n_records+=1
+
                 example = tf.train.Example(features=tf.train.Features(
                   feature={
                     'word': _bytes_feature([w.encode('utf-8')]),
@@ -91,13 +101,15 @@ def create_spectrogram_tfrecords(audio_group='audio'):
             group_prog.close()
     prog.close()
     writer.close()
+    const_file = os.path.join('./outputs',audio_group+'.constants')
+    pickle.dump((n_spec,n_features,n_records),open(const_file,'wb'))
 
 def padd_zeros(spgr, max_samples):
     return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)],
                       'constant')
 
 def find_max_n(trf):
-    max_n = 0
+    max_n,n_records = 0,0
     max_n_it = tf.python_io.tf_record_iterator(path=trf)
     for string_record in max_n_it:
         example = tf.train.Example()
@@ -105,19 +117,20 @@ def find_max_n(trf):
         spec_n1 = example.features.feature['spec_n1'].int64_list.value[0]
         spec_n2 = example.features.feature['spec_n2'].int64_list.value[0]
         max_n = max([max_n,spec_n1,spec_n2])
-    return max_n
+        n_records+=1
+    return (max_n,n_records)
 
-def read_siamese_tfrecords(audio_group='audio'):
+def padd_zeros_siamese_tfrecords(audio_group='audio'):
     records_file  = os.path.join('./outputs',audio_group+'.tfrecords')
     record_iterator = tf.python_io.tf_record_iterator(path=records_file)
-    input_pairs = []
+    print('finding max_n...')
-    output_class = []
+    max_n,n_records = find_max_n(records_file)
-    max_n = find_max_n(records_file)
+    p_spec1 = None
-    spec_w1 = 0
+    print('reading tfrecords...')
-    for string_record in record_iterator:
+    writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '_padded.tfrecords')
+    for string_record in tqdm(record_iterator,desc='padding siamese record',total=n_records):
         example = tf.train.Example()
         example.ParseFromString(string_record)
-        example.features.feature['spec2'].float_list.value[0]
         spec_n1 = example.features.feature['spec_n1'].int64_list.value[0]
         spec_n2 = example.features.feature['spec_n2'].int64_list.value[0]
         spec_w1 = example.features.feature['spec_w1'].int64_list.value[0]
@@ -125,14 +138,155 @@ def read_siamese_tfrecords(audio_group='audio'):
         spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(spec_n1,spec_w1)
         spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(spec_n2,spec_w2)
         p_spec1,p_spec2 = padd_zeros(spec1,max_n),padd_zeros(spec2,max_n)
-        input_pairs.append(np.asarray([p_spec1,p_spec2]))
         output = example.features.feature['output'].int64_list.value
-        output_class.append(np.asarray(output))
+        w_example = tf.train.Example(features=tf.train.Features(
-        n_features = spec_w1
+          feature={
-    input_data,output_data = np.asarray(input_pairs),np.asarray(output_class)
+            'spec1':_float_feature(p_spec1.reshape(-1)),
-    tr_pairs,te_pairs,tr_y,te_y = train_test_split(input_data,output_data)
+            'spec2':_float_feature(p_spec2.reshape(-1)),
-    n_step,n_features = int(max_n),int(spec_w1)
+            'output':_int64_feature(output)
-    return (tr_pairs,te_pairs,tr_y,te_y,n_step,n_features)
+          }
+        ))
+        writer.write(w_example.SerializeToString())
+    const_file = os.path.join('./outputs',audio_group+'.constants')
+    pickle.dump((max_n,p_spec1.shape[1],n_records),open(const_file,'wb'))
+    writer.close()
+
+def pickle_constants(audio_group='audio'):
+    records_file  = os.path.join('./outputs',audio_group+'_padded.tfrecords')
+    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
+    print('finding max_n...')
+    max_n,n_records = find_max_n(records_file)
+    spec1 = 0
+    print('finding spec_w1...')
+    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
+    for string_record in record_iterator:
+        example = tf.train.Example()
+        example.ParseFromString(string_record)
+        spec1 = len(example.features.feature['spec1'].float_list.value)//max_n
+        print('found spec_w1...')
+        break
+    const_file = os.path.join('./outputs',audio_group+'.constants')
+    print(max_n,spec1,n_records)
+    pickle.dump((max_n,spec1,n_records),open(const_file,'wb'))
+
+def reservoir_sample(iterable, k):
+    it = iter(iterable)
+    if not (k > 0):
+        raise ValueError("sample size must be positive")
+
+    sample = list(itertools.islice(it, k)) # fill the reservoir
+    random.shuffle(sample) # if number of items less then *k* then
+                           #   return all items in random order.
+    for i, item in enumerate(it, start=k+1):
+        j = random.randrange(i) # random [0..i)
+        if j < k:
+            sample[j] = item # replace item with gradually decreasing probability
+    return sample
+
+def read_siamese_tfrecords_oneshot(audio_group='audio'):
+    records_file  = os.path.join('./outputs',audio_group+'_padded.tfrecords')
+    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
+    input_pairs = []
+    output_class = []
+    const_file = os.path.join('./outputs',audio_group+'.constants')
+    (n_spec,n_features,n_records) = pickle.load(open(const_file,'rb'))
+    print('reading tfrecords...')
+    samples = min([30000,n_records])
+    input_data = np.zeros((samples,2,n_spec,n_features))
+    output_data = np.zeros((samples,2))
+    random_samples = enumerate(reservoir_sample(record_iterator,samples))
+    for (i,string_record) in tqdm(random_samples,total=samples):
+        # if i == samples:
+        #     break
+        example = tf.train.Example()
+        example.ParseFromString(string_record)
+        spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(n_spec,n_features)
+        spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(n_spec,n_features)
+        input_data[i] = np.asarray([spec1,spec2])
+        output = example.features.feature['output'].int64_list.value
+        output_data[i] = np.asarray(output)
+    print('converting to nparray...')
+    tr_pairs,te_pairs,tr_y,te_y = train_test_split(input_data,output_data,test_size=0.1)
+    result = (tr_pairs,te_pairs,tr_y,te_y,n_spec,n_features)
+    return result
+
+def read_siamese_tfrecords(audio_group='audio'):
+    audio_group='story_words_test'
+
+    record_file = os.path.join('./outputs',audio_group+'_padded.tfrecords')
+    const_file = os.path.join('./outputs',audio_group+'.constants')
+    (n_spec,n_features) = pickle.load(open(const_file,'rb'))
+
+    filename_queue = tf.train.string_input_producer([record_file])
+    reader = tf.TFRecordReader()
+    _, serialized_example = reader.read(filename_queue)
+    features = tf.parse_single_example(serialized_example,
+                    features={
+                        'spec1': tf.FixedLenFeature([1,n_spec,n_features], tf.float32),
+                        'spec2': tf.FixedLenFeature([1,n_spec,n_features], tf.float32),
+                        'output':tf.FixedLenFeature([2], tf.int64)
+                    })
+    spec1 = features['spec1']
+    spec1 = tf.cast(spec1, tf.float32) * (1. / 255)
+    spec2 = features['spec2']
+    spec2 = tf.cast(spec2, tf.float32) * (1. / 255)
+    output = tf.cast(features['output'], tf.int32)
+    return spec1,spec2, output,n_spec,n_features
+
+def read_siamese_tfrecords_batch(audio_group='audio', batch_size=32):
+    audio_group='story_words_test'
+    record_file = os.path.join('./outputs',audio_group+'_padded.tfrecords')
+    """ Return tensor to read from TFRecord """
+    print('Creating graph for loading {} ...'.format(record_file))
+    const_file = os.path.join('./outputs',audio_group+'.constants')
+    (n_spec,n_features) = pickle.load(open(const_file,'rb'))
+    records_file  = os.path.join('./outputs',audio_group+'.tfrecords')
+    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
+    n_records = len([i for i in record_iterator])
+    batch_shape=[batch_size, n_spec, n_features]
+    with tf.variable_scope("SiameseTFRecords"):
+        record_input = data_flow_ops.RecordInput(record_file, batch_size=batch_size)
+        records_op = record_input.get_yield_op()
+        records_op = tf.split(records_op, batch_shape[0], 0)
+        records_op = [tf.reshape(record, []) for record in records_op]
+        specs1, specs2 = [],[]
+        outputs = []
+        for i, serialized_example in tqdm(enumerate(records_op)):
+            with tf.variable_scope("parse_siamese_pairs", reuse=True):
+                features = tf.parse_single_example(
+                    serialized_example,
+                    features={
+                        'spec1': tf.FixedLenFeature([n_spec,n_features], tf.float32),
+                        'spec2': tf.FixedLenFeature([n_spec,n_features], tf.float32),
+                        'output':tf.FixedLenFeature([2], tf.int64)
+                    })
+                spec1 = features['spec1']
+                spec1 = tf.cast(spec1, tf.float32) * (1. / 255)
+                spec2 = features['spec2']
+                output = tf.cast(spec2, tf.float32) * (1. / 255)
+                output = tf.cast(features['output'], tf.float32)
+                specs1.append(spec1)
+                specs2.append(spec2)
+                outputs.append(output)
+
+        specs1 = tf.parallel_stack(specs1, 0)
+        specs2 = tf.parallel_stack(specs2, 0)
+        outputs = tf.parallel_stack(outputs, 0)
+        specs1 = tf.cast(specs1, tf.float32)
+        specs2 = tf.cast(specs2, tf.float32)
+
+        specs1 = tf.reshape(specs1, shape=batch_shape)
+        specs2 = tf.reshape(specs1, shape=batch_shape)
+        specs1_shape = specs1.get_shape()
+        specs2_shape = specs2.get_shape()
+        outputs_shape = outputs.get_shape()
+        copy_stage = data_flow_ops.StagingArea(
+            [tf.float32, tf.float32, tf.float32],
+            shapes=[specs1_shape, specs2_shape, outputs_shape])
+        copy_stage_op = copy_stage.put(
+            [specs1, specs2, outputs])
+        staged_specs1, staged_specs2, staged_outputs = copy_stage.get()
+        return specs1, spec2, outputs,n_spec,n_features,n_records
 
 def audio_samples_word_count(audio_group='audio'):
     audio_group = 'story_all'
@@ -152,14 +306,27 @@ def fix_csv(audio_group='audio'):
         fixed_csv_w = csv.writer(fixed_csv, quoting=csv.QUOTE_MINIMAL)
         fixed_csv_w.writerows(proper_rows)
 
+def convert_old_audio():
+    audio_samples = pd.read_csv( './outputs/audio.csv.old'
+        , names=['word', 'voice', 'rate', 'variant', 'file'])
+    audio_samples['phonemes'] = 'unknown'
+    audio_samples['language'] = 'en-US'
+    audio_samples.loc[audio_samples['variant'] == 'normal','variant'] = 'low'
+    audio_samples.loc[audio_samples['variant'] == 'phoneme','variant'] = 'medium'
+    audio_samples = audio_samples[['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']]
+    audio_samples.to_csv('./outputs/audio_new.csv',index=False,header=False)
 
 if __name__ == '__main__':
     # sunflower_pairs_data()
     # create_spectrogram_data()
     # create_spectrogram_data('story_words')
-    create_spectrogram_tfrecords('story_words')
+    # create_spectrogram_tfrecords('story_words')
     # create_spectrogram_tfrecords('story_words_test')
     # read_siamese_tfrecords('story_all')
+    # read_siamese_tfrecords('story_words_test')
+    pickle_constants('story_words_test')
+    # create_spectrogram_tfrecords('audio')
+    # padd_zeros_siamese_tfrecords('audio')
     # create_padded_spectrogram()
     # create_speech_pairs_data()
     # print(speech_model_data())

speech_siamese.py

@@ -2,7 +2,7 @@ from __future__ import absolute_import
 from __future__ import print_function
 import numpy as np
 # from speech_data import speech_model_data
-from speech_data import read_siamese_tfrecords
+from speech_data import read_siamese_tfrecords_oneshot
 from keras.models import Model,load_model
 from keras.layers import Input, Dense, Dropout, LSTM, Lambda, Concatenate
 from keras.losses import categorical_crossentropy
@@ -82,7 +82,7 @@ def siamese_model(input_dim):
 def train_siamese():
     # the data, shuffled and split between train and test sets
     # tr_pairs, te_pairs, tr_y_e, te_y_e = speech_model_data()
-    (tr_pairs,te_pairs,tr_y,te_y,n_step,n_features) = read_siamese_tfrecords('story_words_test')
+    (tr_pairs,te_pairs,tr_y,te_y,n_step,n_features) = read_siamese_tfrecords_oneshot()
     # tr_y = to_categorical(tr_y_e, num_classes=2)
     # te_y = to_categorical(te_y_e, num_classes=2)
     input_dim = (n_step, n_features)