import pandas as pd from pandas_parallel import apply_by_multiprocessing # import dask as dd # import dask.dataframe as ddf import tensorflow as tf import numpy as np from spectro_gen import generate_aiff_spectrogram from sklearn.model_selection import train_test_split import itertools import os import random import csv import gc # import progressbar from tqdm import tqdm # def prog_bar(title): # widgets = [title, progressbar.Counter(), ' [', progressbar.Bar(), '] - ', progressbar.ETA()] # return progressbar.ProgressBar(widgets=widgets) def siamese_pairs(rightGroup, wrongGroup): group1 = [r for (i, r) in rightGroup.iterrows()] group2 = [r for (i, r) in wrongGroup.iterrows()] rightWrongPairs = [(g1, g2) for g2 in group2 for g1 in group1] rightRightPairs = [i for i in itertools.combinations(group1, 2)] random.shuffle(rightWrongPairs) random.shuffle(rightRightPairs) # return (random.sample(same,10), random.sample(diff,10)) # return rightRightPairs[:10],rightWrongPairs[:10] return rightRightPairs[:32],rightWrongPairs[:32] # return rightRightPairs,rightWrongPairs def create_spectrogram_tfrecords(audio_group='audio'): ''' http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/ http://www.machinelearninguru.com/deep_learning/tensorflow/basics/tfrecord/tfrecord.html ''' audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv' , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file'] , quoting=csv.QUOTE_NONE) # audio_samples = audio_samples.loc[audio_samples['word'] == # 'sunflowers'].reset_index(drop=True) audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x) audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists) audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index() # audio_samples['rate_int'] = apply_by_multiprocessing(audio_samples['rate'], str.isdigit) # audio_samples = audio_samples[audio_samples['rate_int'] == True].reset_index().drop(['level_0'],axis=1) # audio_samples['rate'] = audio_samples['rate'].astype(int) def _float_feature(value): return tf.train.Feature(float_list=tf.train.FloatList(value=value)) def _int64_feature(value): return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) def _bytes_feature(value): return tf.train.Feature(bytes_list=tf.train.BytesList(value=value)) writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '.tfrecords') prog = tqdm(audio_samples.groupby(audio_samples['word']),desc='Computing spectrogram') for (w, word_group) in prog: prog.set_postfix(word=w) g = word_group.reset_index() g['spectrogram'] = apply_by_multiprocessing(g['file_path'],generate_aiff_spectrogram) sample_right = g.loc[g['variant'] == 'low'] sample_wrong = g.loc[g['variant'] == 'medium'] same, diff = siamese_pairs(sample_right, sample_wrong) groups = [([0,1],same),([1,0],diff)] for (output,group) in groups: group_prog = tqdm(group,desc='Writing Spectrogram') for sample1,sample2 in group_prog: group_prog.set_postfix(output=output ,var1=sample1['variant'] ,var2=sample2['variant']) spectro1,spectro2 = sample1['spectrogram'],sample2['spectrogram'] spec_n1,spec_n2 = spectro1.shape[0],spectro2.shape[0] spec_w1,spec_w2 = spectro1.shape[1],spectro2.shape[1] spec1,spec2 = spectro1.reshape(-1),spectro2.reshape(-1) example = tf.train.Example(features=tf.train.Features( feature={ 'word': _bytes_feature([w.encode('utf-8')]), 'phoneme1': _bytes_feature([sample1['phonemes'].encode('utf-8')]), 'phoneme2': _bytes_feature([sample2['phonemes'].encode('utf-8')]), 'voice1': _bytes_feature([sample1['voice'].encode('utf-8')]), 'voice2': _bytes_feature([sample2['voice'].encode('utf-8')]), 'language': _bytes_feature([sample1['language'].encode('utf-8')]), 'rate1':_int64_feature([sample1['rate']]), 'rate2':_int64_feature([sample2['rate']]), 'variant1': _bytes_feature([sample1['variant'].encode('utf-8')]), 'variant2': _bytes_feature([sample2['variant'].encode('utf-8')]), 'file1': _bytes_feature([sample1['file'].encode('utf-8')]), 'file2': _bytes_feature([sample2['file'].encode('utf-8')]), 'spec1':_float_feature(spec1), 'spec2':_float_feature(spec2), 'spec_n1':_int64_feature([spec_n1]), 'spec_w1':_int64_feature([spec_w1]), 'spec_n2':_int64_feature([spec_n2]), 'spec_w2':_int64_feature([spec_w2]), 'output':_int64_feature(output) } )) writer.write(example.SerializeToString()) prog.close() writer.close() def padd_zeros(spgr, max_samples): return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)], 'constant') def find_max_n(trf): max_n = 0 max_n_it = tf.python_io.tf_record_iterator(path=trf) for string_record in max_n_it: example = tf.train.Example() example.ParseFromString(string_record) spec_n1 = example.features.feature['spec_n1'].int64_list.value[0] spec_n2 = example.features.feature['spec_n2'].int64_list.value[0] max_n = max([max_n,spec_n1,spec_n2]) return max_n def read_siamese_tfrecords(audio_group='audio'): records_file = os.path.join('./outputs',audio_group+'.tfrecords') record_iterator = tf.python_io.tf_record_iterator(path=records_file) # input1,input2 = [],[] input_pairs = [] output_class = [] max_n = find_max_n(records_file) spec_w1 = 0 for string_record in record_iterator: example = tf.train.Example() example.ParseFromString(string_record) # word = example.features.feature['word'].bytes_list.value[0] # input_words.append(word) example.features.feature['spec2'].float_list.value[0] spec_n1 = example.features.feature['spec_n1'].int64_list.value[0] spec_n2 = example.features.feature['spec_n2'].int64_list.value[0] spec_w1 = example.features.feature['spec_w1'].int64_list.value[0] spec_w2 = example.features.feature['spec_w2'].int64_list.value[0] spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(spec_n1,spec_w1) spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(spec_n2,spec_w2) p_spec1,p_spec2 = padd_zeros(spec1,max_n),padd_zeros(spec2,max_n) # input1.append(spec1) # input2.append(spec2) input_pairs.append(np.asarray([p_spec1,p_spec2])) # input_pairs.append([spec1,spec2]) output = example.features.feature['output'].int64_list.value output_class.append(np.asarray(output)) n_features = spec_w1 # if len(input_pairs) > 50: # break input_data,output_data = np.asarray(input_pairs),np.asarray(output_class) import pdb; pdb.set_trace() # tr_x1,te_x1,tr_x2,te_x2,tr_y,te_y = train_test_split(input1,input2,output_class) tr_pairs,te_pairs,tr_y,te_y = train_test_split(input_data,output_data) # return (tr_x1,te_x1,tr_x2,te_x2,tr_y,te_y) n_step,n_features = int(max_n),int(spec_w1) return (tr_pairs,te_pairs,tr_y,te_y,n_step,n_features) def audio_samples_word_count(audio_group='audio'): audio_group = 'story_all' audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv' , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file'] , quoting=csv.QUOTE_NONE) # audio_samples = audio_samples.loc[audio_samples['word'] == # 'sunflowers'].reset_index(drop=True) audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x) audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists) audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index() return len(audio_samples.groupby(audio_samples['word'])) def fix_csv(audio_group='audio'): audio_csv_lines = open('./outputs/' + audio_group + '.csv','r').readlines() audio_csv_data = [i.strip().split(',') for i in audio_csv_lines] # audio_samples = pd.read_csv( './outputs/story_words.csv' # , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file'] # , quoting=csv.QUOTE_NONE) # voice_set = set(audio_samples['voice'].unique().tolist()) # to_be_fixed = [i for i in audio_csv_data if len(i) > 7] # def unite_words(entries): # entries = to_be_fixed[0] # word_entries = next(((entries[:i],entries[i:]) for (i,e) in enumerate(entries) if e in voice_set),'') # word_entries[1] # return # to_be_fixed[0] # entries = [unite_words for e in to_be_fixed] # [i for i in entries if len(i) % 2 != 0] proper_rows = [i for i in audio_csv_data if len(i) == 7] with open('./outputs/' + audio_group + '-new.csv','w') as fixed_csv: fixed_csv_w = csv.writer(fixed_csv, quoting=csv.QUOTE_MINIMAL) fixed_csv_w.writerows(proper_rows) if __name__ == '__main__': # sunflower_pairs_data() # create_spectrogram_data() # create_spectrogram_data('story_words') create_spectrogram_tfrecords('story_words') # create_spectrogram_tfrecords('story_words_test') # read_siamese_tfrecords('story_all') # create_padded_spectrogram() # create_speech_pairs_data() # print(speech_model_data())