199 lines
8.7 KiB
Python
199 lines
8.7 KiB
Python
import pandas as pd
|
|
from pandas_parallel import apply_by_multiprocessing
|
|
# import dask as dd
|
|
# import dask.dataframe as ddf
|
|
import tensorflow as tf
|
|
import numpy as np
|
|
from spectro_gen import generate_aiff_spectrogram
|
|
from sklearn.model_selection import train_test_split
|
|
import itertools
|
|
import os
|
|
import random
|
|
import csv
|
|
import gc
|
|
|
|
def get_siamese_pairs(groupF1, groupF2):
|
|
group1 = [r for (i, r) in groupF1.iterrows()]
|
|
group2 = [r for (i, r) in groupF2.iterrows()]
|
|
diff = [(g1, g2) for g2 in group2 for g1 in group1]
|
|
same = [i for i in itertools.combinations(group1, 2)
|
|
] + [i for i in itertools.combinations(group2, 2)]
|
|
random.shuffle(same)
|
|
random.shuffle(diff)
|
|
# return (random.sample(same,10), random.sample(diff,10))
|
|
return same[:10],diff[:10]
|
|
|
|
|
|
def append_zeros(spgr, max_samples):
|
|
return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)],
|
|
'median')
|
|
|
|
def padd_zeros(spgr, max_samples):
|
|
return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)],
|
|
'constant')
|
|
|
|
def to_onehot(a,class_count=2):
|
|
# >>> a = np.array([1, 0, 3])
|
|
a_row_n = a.shape[0]
|
|
b = np.zeros((a_row_n, class_count))
|
|
b[np.arange(a_row_n), a] = 1
|
|
return b
|
|
|
|
def create_pair(l, r, max_samples):
|
|
l_sample = padd_zeros(l, max_samples)
|
|
r_sample = padd_zeros(r, max_samples)
|
|
return np.asarray([l_sample, r_sample])
|
|
|
|
|
|
def create_test_pair(l, r, max_samples):
|
|
l_sample = append_zeros(l, max_samples)
|
|
r_sample = append_zeros(r, max_samples)
|
|
return np.asarray([[l_sample, r_sample]])
|
|
|
|
|
|
def create_X(sp, max_samples):
|
|
return create_pair(sp[0]['spectrogram'], sp[1]['spectrogram'], max_samples)
|
|
|
|
|
|
# def get_word_pairs_data(word, max_samples):
|
|
# audio_samples = pd.read_csv(
|
|
# './outputs/audio.csv',
|
|
# names=['word', 'voice', 'rate', 'variant', 'file'])
|
|
# audio_samples = audio_samples.loc[audio_samples['word'] ==
|
|
# word].reset_index(drop=True)
|
|
# audio_samples.loc[:, 'spectrogram'] = audio_samples.loc[:, 'file'].apply(
|
|
# lambda x: 'outputs/audio/' + x).apply(generate_aiff_spectrogram)
|
|
# max_samples = audio_samples['spectrogram'].apply(
|
|
# lambda x: x.shape[0]).max()
|
|
# same_data, diff_data = [], []
|
|
# for (w, g) in audio_samples.groupby(audio_samples['word']):
|
|
# sample_norm = g.loc[audio_samples['variant'] == 'normal']
|
|
# sample_phon = g.loc[audio_samples['variant'] == 'phoneme']
|
|
# same, diff = get_siamese_pairs(sample_norm, sample_phon)
|
|
# same_data.extend([create_X(s, max_samples) for s in same])
|
|
# diff_data.extend([create_X(d, max_samples) for d in diff])
|
|
# Y = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))])
|
|
# X = np.asarray(same_data + diff_data)
|
|
# # tr_pairs, te_pairs, tr_y, te_y = train_test_split(X, Y, test_size=0.1)
|
|
# return (X, Y)
|
|
|
|
|
|
def create_spectrogram_data(audio_group='audio'):
|
|
audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv'
|
|
, names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
|
|
, quoting=csv.QUOTE_NONE)
|
|
# audio_samples = audio_samples.loc[audio_samples['word'] ==
|
|
# 'sunflowers'].reset_index(drop=True)
|
|
audio_samples['file_paths'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
|
|
audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_paths'], os.path.exists)
|
|
audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index()
|
|
audio_samples['spectrogram'] = apply_by_multiprocessing(audio_samples['file_paths'],generate_aiff_spectrogram)#.apply(
|
|
audio_samples['window_count'] = audio_samples.loc[:,'spectrogram'].apply(lambda x: x.shape[0])
|
|
audio_samples.to_pickle('outputs/{}-spectrogram.pkl'.format(audio_group))
|
|
|
|
def create_spectrogram_tfrecords(audio_group='audio'):
|
|
# audio_group = 'story_words'
|
|
audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv'
|
|
, names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
|
|
, quoting=csv.QUOTE_NONE)
|
|
# audio_samples = audio_samples.loc[audio_samples['word'] ==
|
|
# 'sunflowers'].reset_index(drop=True)
|
|
audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
|
|
audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists)
|
|
audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index()
|
|
# audio_samples['spectrogram'] = apply_by_multiprocessing(audio_samples['file_paths'],generate_aiff_spectrogram)#.apply(
|
|
# audio_samples['window_count'] = audio_samples.loc[:,'spectrogram'].apply(lambda x: x.shape[0])
|
|
# audio_samples.to_pickle('outputs/{}-spectrogram.pkl'.format(audio_group))
|
|
|
|
def _float_feature(value):
|
|
return tf.train.Feature(float_list=tf.train.FloatList(value=value))
|
|
|
|
def _int64_feature(value):
|
|
return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
|
|
|
|
def _bytes_feature(value):
|
|
return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
|
|
|
|
writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '.tfrecords')
|
|
audio_samples = audio_samples[:100]
|
|
for (i,sample) in audio_samples.iterrows():
|
|
spectrogram = generate_aiff_spectrogram(sample['file_path'])
|
|
spec_n = spectrogram.shape[0]
|
|
spec_w = spectrogram.shape[1]
|
|
spec = spectrogram.reshape(-1)
|
|
example = tf.train.Example(features=tf.train.Features(
|
|
feature={
|
|
'word': _bytes_feature([sample['word'].encode('utf-8')]),
|
|
'phoneme': _bytes_feature([sample['phonemes'].encode('utf-8')]),
|
|
'voice': _bytes_feature([sample['voice'].encode('utf-8')]),
|
|
'language': _bytes_feature([sample['language'].encode('utf-8')]),
|
|
'rate':_int64_feature([sample['rate']]),
|
|
'variant': _bytes_feature([sample['variant'].encode('utf-8')]),
|
|
'file': _bytes_feature([sample['file'].encode('utf-8')]),
|
|
'spec':_float_feature(spec),
|
|
'spec_n':_int64_feature([spec_n]),
|
|
'spec_w':_int64_feature([spec_w])
|
|
}
|
|
))
|
|
writer.write(example.SerializeToString())
|
|
writer.close()
|
|
|
|
def create_tagged_data(audio_samples):
|
|
same_data, diff_data = [], []
|
|
for (w, g) in audio_samples.groupby(audio_samples['word']):
|
|
# sample_norm = g.loc[audio_samples['variant'] == 'low']
|
|
# sample_phon = g.loc[audio_samples['variant'] == 'medium']
|
|
sample_norm = g.loc[audio_samples['variant'] == 'normal']
|
|
sample_phon = g.loc[audio_samples['variant'] == 'phoneme']
|
|
same, diff = get_siamese_pairs(sample_norm, sample_phon)
|
|
same_data.extend([create_X(s) for s in same])
|
|
diff_data.extend([create_X(d) for d in diff])
|
|
print('creating all speech pairs')
|
|
Y_f = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))])
|
|
Y = to_onehot(Y_f.astype(np.int8))
|
|
print('casting as array speech pairs')
|
|
X = np.asarray(same_data + diff_data)
|
|
return X,Y
|
|
|
|
def create_speech_pairs_data(audio_group='audio'):
|
|
audio_samples = pd.read_pickle('outputs/{}-spectrogram.pkl'.format(audio_group))
|
|
# sample_size = audio_samples['spectrogram'][0].shape[1]
|
|
tr_audio_samples,te_audio_samples = train_test_split(audio_samples, test_size=0.1)
|
|
def save_samples_for(sample_name,samples):
|
|
print('generating {} siamese speech pairs'.format(sample_name))
|
|
X,Y = create_tagged_data(samples)
|
|
print('shuffling array speech pairs')
|
|
rng_state = np.random.get_state()
|
|
np.random.shuffle(X)
|
|
np.random.set_state(rng_state)
|
|
np.random.shuffle(Y)
|
|
print('pickling X/Y')
|
|
np.save('outputs/{}-train-X.npy'.format(audio_group), X)
|
|
np.save('outputs/{}-train-Y.npy'.format(audio_group), Y)
|
|
save_samples_for('train',tr_audio_samples)
|
|
save_samples_for('test',te_audio_samples)
|
|
|
|
def speech_data(audio_group='audio'):
|
|
X = np.load('outputs/{}-X.npy'.format(audio_group)) / 255.0
|
|
Y = np.load('outputs/{}-Y.npy'.format(audio_group))
|
|
return (X,Y)
|
|
|
|
def speech_model_data():
|
|
tr_pairs = np.load('outputs/tr_pairs.npy') / 255.0
|
|
te_pairs = np.load('outputs/te_pairs.npy') / 255.0
|
|
tr_pairs[tr_pairs < 0] = 0
|
|
te_pairs[te_pairs < 0] = 0
|
|
tr_y = np.load('outputs/tr_y.npy')
|
|
te_y = np.load('outputs/te_y.npy')
|
|
return tr_pairs, te_pairs, tr_y, te_y
|
|
|
|
|
|
if __name__ == '__main__':
|
|
# sunflower_pairs_data()
|
|
# create_spectrogram_data()
|
|
# create_spectrogram_data('story_words')
|
|
create_spectrogram_tfrecords('story_words')
|
|
# create_padded_spectrogram()
|
|
# create_speech_pairs_data()
|
|
# print(speech_model_data())
|