updated README to include testing

2. updated to run on complete data

.gitignore

@@ -143,3 +143,5 @@ inputs/audio*
 logs/*
 models/*
 *.pkl
+temp/*
+trained/*

CLI.md

@@ -0,0 +1,2 @@
+### Convert audio files
+$ `for f in *.mp3; do ffmpeg -i "$f" "${f%.mp3}.aiff"; done`

README.md

@@ -0,0 +1,23 @@
+### Setup
+`. env/bin/activate` to activate the virtualenv.
+
+### Data Generation
+* update `OUTPUT_NAME` in *speech_samplegen.py* to create the dataset folder with the name
+* `python speech_samplegen.py` generates variants of audio samples
+
+### Data Preprocessing
+* `python speech_data.py` creates the training-testing data from the generated samples.
+* run `fix_csv(OUTPUT_NAME)` once to create the fixed index of the dataset generated
+* run `generate_sppas_trans(OUTPUT_NAME)` once to create the SPPAS transcription(wav+txt) data
+* run `$ (SPPAS_DIR)/bin/annotation.py -l eng -e csv --ipus --tok --phon --align --align -w ./outputs/OUTPUT_NAME/` once to create the phoneme alignment csv files for all variants.
+* `create_seg_phonpair_tfrecords(OUTPUT_NAME)` creates the tfrecords files
+ with the phoneme level pairs of right/wrong stresses
+
+### Training
+* `python speech_model.py` trains the model with the training data generated.
+* `train_siamese(OUTPUT_NAME)` trains the siamese model with the generated dataset.
+
+### Testing
+* `python speech_test.py` tests the trained model with the test dataset
+* `evaluate_siamese(TEST_RECORD_FILE,audio_group=OUTPUT_NAME,weights = WEIGHTS_FILE_NAME)`
+  the TEST_RECORD_FILE will be under outputs directory and WEIGHTS_FILE_NAME will be under the models directory, pick the most recent weights file.

generate_similar.py

@@ -124,13 +124,18 @@ def parse_apple_phonemes(ph_str):
     return []
 
 
-def similar_phoneme(ph_str):
+def similar_phoneme_word(ph_str):
     phons = parse_apple_phonemes(ph_str)
     vowels = [i for i in phons if i.vowel]
     random.choice(vowels).adjust_stress()
     return ''.join([str(i) for i in phons])
 
+def similar_phoneme_phrase(ph_str):
+    return ' '.join([similar_phoneme_word(w) for w in ph_str.split()])
 
 def similar_word(word_str):
     similar = pronouncing.rhymes(word_str)
     return random.choice(similar) if len(similar) > 0 else word_str
+
+def similar_phrase(ph_str):
+    return ' '.join([similar_word(w) for w in ph_str.split()])

pandas_parallel.py

@@ -1,25 +0,0 @@
-import multiprocessing
-import pandas as pd
-import numpy as np
-
-
-
-def _apply_df(args):
-    df, func, num, kwargs = args
-    return num, df.apply(func, **kwargs)
-
-def apply_by_multiprocessing(df,func,**kwargs):
-  cores = multiprocessing.cpu_count()
-  workers=kwargs.pop('workers') if 'workers' in kwargs else cores
-  pool = multiprocessing.Pool(processes=workers)
-  result = pool.map(_apply_df, [(d, func, i, kwargs) for i,d in enumerate(np.array_split(df, workers))])
-  pool.close()
-  result=sorted(result,key=lambda x:x[0])
-  return pd.concat([i[1] for i in result])
-
-def square(x):
-    return x**x
-
-if __name__ == '__main__':
-    df = pd.DataFrame({'a':range(10), 'b':range(10)})
-    apply_by_multiprocessing(df, square, axis=1, workers=4)

record_mic_speech.py

@@ -1,42 +0,0 @@
-import pyaudio
-import numpy as np
-# from matplotlib import pyplot as plt
-from spectro_gen import plot_stft, generate_spec_frec
-
-
-def record_spectrogram(n_sec, plot=False, playback=False):
-    SAMPLE_RATE = 22050
-    N_CHANNELS = 2
-    N_SEC = n_sec
-    CHUNKSIZE = int(SAMPLE_RATE * N_SEC / N_CHANNELS)  # fixed chunk size
-    # show_record_prompt()
-    input('Press [Enter] to start recording sample... ')
-    p_inp = pyaudio.PyAudio()
-    stream = p_inp.open(
-        format=pyaudio.paFloat32,
-        channels=N_CHANNELS,
-        rate=SAMPLE_RATE,
-        input=True,
-        frames_per_buffer=CHUNKSIZE)
-    data = stream.read(CHUNKSIZE)
-    numpydata = np.frombuffer(data, dtype=np.float32)
-    multi_channel = np.abs(np.reshape(numpydata, (-1, 2))).mean(axis=1)
-    one_channel = np.asarray([multi_channel, -1 * multi_channel]).T.reshape(-1)
-    mean_channel_data = one_channel.tobytes()
-    stream.stop_stream()
-    stream.close()
-    p_inp.terminate()
-    if plot:
-        plot_stft(one_channel, SAMPLE_RATE)
-    if playback:
-        p_oup = pyaudio.PyAudio()
-        stream = p_oup.open(
-            format=pyaudio.paFloat32,
-            channels=2,
-            rate=SAMPLE_RATE,
-            output=True)
-        stream.write(mean_channel_data)
-        stream.close()
-        p_oup.terminate()
-    ims, _ = generate_spec_frec(one_channel, SAMPLE_RATE)
-    return ims

requirements-linux.txt

@@ -0,0 +1,80 @@
+bleach==1.5.0
+click==6.7
+cloudpickle==0.4.1
+cycler==0.10.0
+dask==0.15.4
+decorator==4.1.2
+distributed==1.19.3
+entrypoints==0.2.3
+enum34==1.1.6
+futures==3.1.1
+graphviz==0.8.1
+h5py==2.7.1
+HeapDict==1.0.0
+html5lib==0.9999999
+ipykernel==4.6.1
+ipython==6.2.1
+ipython-genutils==0.2.0
+ipywidgets==7.0.3
+jedi==0.11.0
+Jinja2==2.9.6
+jsonschema==2.6.0
+jupyter==1.0.0
+jupyter-client==5.1.0
+jupyter-console==5.2.0
+jupyter-core==4.3.0
+Keras==2.0.8
+locket==0.2.0
+Markdown==2.6.9
+MarkupSafe==1.0
+matplotlib==2.1.0
+mistune==0.7.4
+msgpack-python==0.4.8
+nbconvert==5.3.1
+nbformat==4.4.0
+notebook==5.2.0
+numexpr==2.6.4
+numpy==1.13.3
+pandas==0.20.3
+pandocfilters==1.4.2
+parso==0.1.0
+partd==0.3.8
+pexpect==4.2.1
+pickleshare==0.7.4
+praat-parselmouth==0.2.0
+progressbar2==3.34.3
+prompt-toolkit==1.0.15
+protobuf==3.5.0
+psutil==5.4.0
+ptyprocess==0.5.2
+PyAudio==0.2.11
+pydot==1.2.3
+Pygments==2.2.0
+pyparsing==2.2.0
+pysndfile==1.0.0
+python-dateutil==2.6.1
+python-utils==2.2.0
+pytz==2017.2
+PyYAML==3.12
+pyzmq==16.0.2
+qtconsole==4.3.1
+scikit-learn==0.19.0
+scipy==0.19.1
+seaborn==0.8.1
+simplegeneric==0.8.1
+six==1.11.0
+sortedcontainers==1.5.7
+tables==3.4.2
+tblib==1.3.2
+tensorflow==1.3.0
+tensorflow-tensorboard==0.4.0rc3
+terminado==0.6
+testpath==0.3.1
+toolz==0.8.2
+tornado==4.5.2
+tqdm==4.19.4
+traitlets==4.3.2
+wcwidth==0.1.7
+Werkzeug==0.12.2
+widgetsnbextension==3.0.6
+zict==0.1.3

segment_data.py

@@ -0,0 +1,265 @@
+import random
+import math
+import pickle
+from functools import reduce
+from tqdm import tqdm
+from sklearn.model_selection import train_test_split
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+import shutil
+
+from speech_pitch import *
+from speech_tools import reservoir_sample,padd_zeros
+
+# import importlib
+# import speech_tools
+# importlib.reload(speech_tools)
+# %matplotlib inline
+
+SPEC_MAX_FREQUENCY = 8000
+SPEC_WINDOW_SIZE = 0.03
+
+def fix_csv(collection_name = 'test'):
+    seg_data = pd.read_csv('./outputs/segments/'+collection_name+'/index.csv',names=['phrase','filename'
+                ,'start_phoneme','end_phoneme','start_time','end_time'])
+    seg_data.to_csv('./outputs/segments/'+collection_name+'/index.fixed.csv')
+
+def pick_random_phrases(collection_name='test'):
+    collection_name = 'test'
+    seg_data = pd.read_csv('./outputs/'+collection_name+'.fixed.csv',index_col=0)
+    phrase_groups = random.sample([i for i in seg_data.groupby(['phrase'])],10)
+    result = []
+    for ph,g in phrase_groups:
+        result.append(ph)
+    pd.DataFrame(result,columns=['phrase']).to_csv('./outputs/'+collection_name+'.random.csv')
+
+# pick_random_phrases()
+
+def plot_random_phrases(collection_name = 'test'):
+    # collection_name = 'test'
+    rand_words = pd.read_csv('./outputs/'+collection_name+'.random.csv',index_col=0)
+    rand_w_list = rand_words['phrase'].tolist()
+    seg_data = pd.read_csv('./outputs/'+collection_name+'.fixed.csv',index_col=0)
+    result = (seg_data['phrase'] == rand_w_list[0])
+    for i in rand_w_list[1:]:
+         result |= (seg_data['phrase'] == i)
+    phrase_groups = [i for i in seg_data[result].groupby(['phrase'])]
+    self_files = ['a_wrong_turn-low1.aiff','great_pin-low1.aiff'
+    ,'he_set_off_at_once_to_find_the_beast-low1.aiff'
+    ,'hound-low1.aiff','noises-low1.aiff','po_burped-low1.aiff'
+    ,'she_loves_the_roses-low1.aiff','the_busy_spider-low1.aiff'
+    ,'the_rain_helped-low1.aiff','to_go_to_the_doctor-low1.aiff']
+    co_files = map(lambda x: './inputs/self/'+x,self_files)
+
+    for ((ph,g),s_f) in zip(phrase_groups,co_files):
+    # ph,g = phrase_groups[0]
+        file_path = './outputs/test/'+g.iloc[0]['filename']
+        phrase_sample = pm_snd(file_path)
+        self_sample = pm_snd(s_f)
+        player,closer = play_sound()
+        # rows = [i for i in g.iterrows()]
+        # random.shuffle(rows)
+        print(ph)
+        phon_stops = []
+        for (i,phon) in g.iterrows():
+            end_t = phon['end_time']/1000
+            phon_ch = phon['start_phoneme']
+            phon_stops.append((end_t,phon_ch))
+        plot_sample_pitch(phrase_sample,phons = phon_stops)
+        plot_sample_pitch(self_sample)
+        # player(phrase_sample)
+        # input()
+    # for (i,phon) in g.iterrows():
+    #     # phon = g.iloc[1]
+    #     start_t = phon['start_time']/1000
+    #     end_t = phon['end_time']/1000
+    #     phon_ch = phon['start_phoneme']
+    #     phon_sample = phrase_sample.extract_part(from_time=start_t,to_time=end_t)
+    #     if phon_sample.n_samples*phon_sample.sampling_period < 6.4/100:
+    #         continue
+    #     # if phon_ch[0] not in 'AEIOU':
+    #     #     continue
+    #     # phon_sample
+    #     # player(phon_sample)
+    #     # plot_sample_intensity(phon_sample)
+    #     print(phon_ch)
+    #     plot_sample_pitch(phon_sample)
+    # closer()
+
+def plot_segments(collection_name = 'story_test_segments'):
+    collection_name = 'story_test_segments'
+    seg_data = pd.read_csv('./outputs/'+collection_name+'.fixed.csv',index_col=0)
+    phrase_groups = [i for i in seg_data.groupby(['phrase'])]
+    for (ph,g) in phrase_groups:
+        # ph,g = phrase_groups[0]
+        file_path = './outputs/'+collection_name+'/'+g.iloc[0]['filename']
+        phrase_sample = pm_snd(file_path)
+        # player,closer = play_sound()
+        print(ph)
+        phon_stops = []
+        for (i,phon) in g.iterrows():
+            end_t = phon['end_time']/1000
+            phon_ch = phon['start_phoneme']
+            phon_stops.append((end_t,phon_ch))
+        phrase_spec = phrase_sample.to_spectrogram(window_length=0.03, maximum_frequency=8000)
+        sg_db = 10 * np.log10(phrase_spec.values)
+        result = np.zeros(sg_db.shape[0],dtype=np.int64)
+        ph_bounds = [t[0] for t in phon_stops[1:]]
+        b_frames = np.asarray([spec_frame(phrase_spec,b) for b in ph_bounds])
+        result[b_frames] = 1
+    # print(audio)
+
+def generate_spec(aiff_file):
+    phrase_sample = pm_snd(aiff_file)
+    phrase_spec = phrase_sample.to_spectrogram(window_length=SPEC_WINDOW_SIZE, maximum_frequency=SPEC_MAX_FREQUENCY)
+    sshow_abs = np.abs(phrase_spec.values + np.finfo(phrase_spec.values.dtype).eps)
+    sg_db = 10 * np.log10(sshow_abs)
+    sg_db[sg_db < 0] = 0
+    return sg_db,phrase_spec
+
+
+def spec_frame(spec,b):
+    return int(round(spec.frame_number_to_time(b)))
+
+def _float_feature(value):
+  return tf.train.Feature(float_list=tf.train.FloatList(value=value))
+
+def _int64_feature(value):
+  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
+
+def _bytes_feature(value):
+  return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
+
+def create_segments_tfrecords(collection_name='story_test_segments',sample_count=0,train_test_ratio=0.1):
+    audio_samples = pd.read_csv( './outputs/segments/' + collection_name + '/index.fixed.csv',index_col=0)
+    audio_samples['file_path'] = audio_samples.loc[:, 'filename'].apply(lambda x: 'outputs/segments/' + collection_name + '/samples/' + x)
+    n_records,n_spec,n_features = 0,0,0
+
+    def write_samples(wg,sample_name):
+        phrase_groups = tqdm(wg,desc='Computing segmentation')
+        record_file = './outputs/segments/{}/{}.tfrecords'.format(collection_name,sample_name)
+        writer = tf.python_io.TFRecordWriter(record_file)
+        for (ph,g) in phrase_groups:
+            fname = g.iloc[0]['filename']
+            sg_db,phrase_spec = generate_spec(g.iloc[0]['file_path'])
+            phon_stops = []
+            phrase_groups.set_postfix(phrase=ph)
+            spec_n,spec_w = sg_db.shape
+            spec = sg_db.reshape(-1)
+            for (i,phon) in g.iterrows():
+                end_t = phon['end_time']/1000
+                phon_ch = phon['start_phoneme']
+                phon_stops.append((end_t,phon_ch))
+            result = np.zeros(spec_n,dtype=np.int64)
+            ph_bounds = [t[0] for t in phon_stops]
+            f_bounds = [spec_frame(phrase_spec,b) for b in ph_bounds]
+            valid_bounds = [i for i in f_bounds if 0 < i < spec_n]
+            b_frames = np.asarray(valid_bounds)
+            if len(b_frames) > 0:
+                result[b_frames] = 1
+            nonlocal n_records,n_spec,n_features
+            n_spec = max([n_spec,spec_n])
+            n_features = spec_w
+            n_records+=1
+            example = tf.train.Example(features=tf.train.Features(
+              feature={
+                'phrase': _bytes_feature([ph.encode('utf-8')]),
+                'file': _bytes_feature([fname.encode('utf-8')]),
+                'spec':_float_feature(spec),
+                'spec_n':_int64_feature([spec_n]),
+                'spec_w':_int64_feature([spec_w]),
+                'output':_int64_feature(result)
+              }
+            ))
+            writer.write(example.SerializeToString())
+        phrase_groups.close()
+        writer.close()
+
+    word_groups = [i for i in audio_samples.groupby('phrase')]
+    wg_sampled = reservoir_sample(word_groups,sample_count) if sample_count > 0 else word_groups
+    # write_samples(word_groups,'all')
+    tr_audio_samples,te_audio_samples = train_test_split(wg_sampled,test_size=train_test_ratio)
+    write_samples(tr_audio_samples,'train')
+    write_samples(te_audio_samples,'test')
+    const_file = './outputs/segments/'+collection_name+'/constants.pkl'
+    pickle.dump((n_spec,n_features,n_records),open(const_file,'wb'))
+
+def record_generator_count(records_file):
+    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
+    count,spec_n = 0,0
+    for i in record_iterator:
+        count+=1
+    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
+    return record_iterator,count
+
+def read_segments_tfrecords_generator(collection_name='audio',batch_size=32,test_size=0):
+    # collection_name = 'story_test'
+    records_file  = './outputs/segments/'+collection_name+'/train.tfrecords'
+    const_file = './outputs/segments/'+collection_name+'/constants.pkl'
+    (n_spec,n_features,n_records) = pickle.load(open(const_file,'rb'))
+    def copy_read_consts(dest_dir):
+        shutil.copy2(const_file,dest_dir+'/constants.pkl')
+        return (n_spec,n_features,n_records)
+    # @threadsafe_iter
+    def record_generator():
+        print('reading tfrecords({}-train)...'.format(collection_name))
+        input_data = []
+        output_data = []
+        while True:
+            record_iterator,records_count = record_generator_count(records_file)
+            for (i,string_record) in enumerate(record_iterator):
+                # (i,string_record) = next(enumerate(record_iterator))
+                example = tf.train.Example()
+                example.ParseFromString(string_record)
+                spec_n = example.features.feature['spec_n'].int64_list.value[0]
+                spec_w = example.features.feature['spec_w'].int64_list.value[0]
+                spec = np.array(example.features.feature['spec'].float_list.value).reshape(spec_n,spec_w)
+                p_spec = padd_zeros(spec,n_spec)
+                input_data.append(p_spec)
+                output = np.asarray(example.features.feature['output'].int64_list.value)
+                p_output = np.pad(output,(0,n_spec-output.shape[0]),'constant')
+                output_data.append(p_output)
+                if len(input_data) == batch_size or i == n_records-1:
+                    input_arr = np.asarray(input_data)
+                    output_arr = np.asarray(output_data)
+                    input_arr.shape,output_arr.shape
+                    yield (input_arr,output_arr)
+                    input_data = []
+                    output_data = []
+
+    # Read test in one-shot
+    print('reading tfrecords({}-test)...'.format(collection_name))
+    te_records_file  = './outputs/segments/'+collection_name+'/test.tfrecords'
+    te_re_iterator,te_n_records = record_generator_count(te_records_file)
+    # test_size = 10
+    test_size = min([test_size,te_n_records]) if test_size > 0 else te_n_records
+    input_data = np.zeros((test_size,n_spec,n_features))
+    output_data = np.zeros((test_size,n_spec))
+    random_samples = enumerate(reservoir_sample(te_re_iterator,test_size))
+    for (i,string_record) in tqdm(random_samples,total=test_size):
+        # (i,string_record) = next(random_samples)
+        example = tf.train.Example()
+        example.ParseFromString(string_record)
+        spec_n = example.features.feature['spec_n'].int64_list.value[0]
+        spec_w = example.features.feature['spec_w'].int64_list.value[0]
+        spec = np.array(example.features.feature['spec'].float_list.value).reshape(spec_n,spec_w)
+        p_spec = padd_zeros(spec,n_spec)
+        input_data[i] = p_spec
+        output = np.asarray(example.features.feature['output'].int64_list.value)
+        p_output = np.pad(output,(0,n_spec-output.shape[0]),'constant')
+        output_data[i] = p_output
+
+    return record_generator,input_data,output_data,copy_read_consts
+
+if __name__ == '__main__':
+    # plot_random_phrases()
+    # fix_csv('story_test_segments')
+    # plot_segments('story_test_segments')
+    # fix_csv('story_words')
+    # pass
+    create_segments_tfrecords('story_words.30', sample_count=36,train_test_ratio=0.1)
+    # record_generator,input_data,output_data,copy_read_consts = read_segments_tfrecords_generator('story_test')
+    # tr_gen = record_generator()
+    # for i in tr_gen:
+    #     print(i[0].shape,i[1].shape)

segment_model.py

@@ -0,0 +1,144 @@
+from __future__ import absolute_import
+from __future__ import print_function
+import numpy as np
+from keras.models import Model,load_model,model_from_yaml
+from keras.layers import Input,Concatenate,Lambda, Reshape, Dropout
+from keras.layers import Dense,Conv2D, LSTM, Bidirectional, GRU
+from keras.layers import BatchNormalization,Activation
+from keras.losses import categorical_crossentropy
+from keras.utils import to_categorical
+from keras.optimizers import RMSprop,Adadelta,Adagrad,Adam,Nadam
+from keras.callbacks import TensorBoard, ModelCheckpoint
+from keras import backend as K
+from keras.utils import plot_model
+from speech_tools import create_dir,step_count
+from segment_data import read_segments_tfrecords_generator
+
+# import importlib
+# import segment_data
+# import speech_tools
+# importlib.reload(segment_data)
+# importlib.reload(speech_tools)
+
+
+# TODO implement ctc losses
+# https://github.com/fchollet/keras/blob/master/examples/image_ocr.py
+def accuracy(y_true, y_pred):
+    '''Compute classification accuracy with a fixed threshold on distances.
+    '''
+    return K.mean(K.equal(y_true, K.cast(y_pred > 0.5, y_true.dtype)))
+
+def ctc_lambda_func(args):
+    y_pred, labels, input_length, label_length = args
+    # the 2 is critical here since the first couple outputs of the RNN
+    # tend to be garbage:
+    y_pred = y_pred[:, 2:, :]
+    return K.ctc_batch_cost(labels, y_pred, input_length, label_length)
+
+def segment_model(input_dim):
+    inp = Input(shape=input_dim)
+    cnv1 = Conv2D(filters=32, kernel_size=(5,9))(inp)
+    cnv2 = Conv2D(filters=1, kernel_size=(5,9))(cnv1)
+    dr_cnv2 = Dropout(rate=0.95)(cnv2)
+    cn_rnn_dim = (dr_cnv2.shape[1].value,dr_cnv2.shape[2].value)
+    r_dr_cnv2 = Reshape(target_shape=cn_rnn_dim)(dr_cnv2)
+    b_gr1 = Bidirectional(GRU(512, return_sequences=True),merge_mode='sum')(r_dr_cnv2)
+    b_gr2 = Bidirectional(GRU(512, return_sequences=True),merge_mode='sum')(b_gr1)
+    b_gr3 = Bidirectional(GRU(512, return_sequences=True),merge_mode='sum')(b_gr2)
+    oup = Dense(2, activation='softmax')(b_gr3)
+    return Model(inp, oup)
+
+def simple_segment_model(input_dim):
+    inp = Input(shape=input_dim)
+    b_gr1 = Bidirectional(LSTM(32, return_sequences=True))(inp)
+    b_gr1 = Bidirectional(LSTM(16, return_sequences=True),merge_mode='sum')(b_gr1)
+    b_gr1 = LSTM(1, return_sequences=True,activation='softmax')(b_gr1)
+    # b_gr1 = LSTM(4, return_sequences=True)(b_gr1)
+    # b_gr1 = LSTM(2, return_sequences=True)(b_gr1)
+    # bn_b_gr1 = BatchNormalization(momentum=0.98)(b_gr1)
+    # b_gr2 = GRU(64, return_sequences=True)(b_gr1)
+    # bn_b_gr2 = BatchNormalization(momentum=0.98)(b_gr2)
+    # d1 = Dense(32)(b_gr2)
+    # bn_d1 = BatchNormalization(momentum=0.98)(d1)
+    # bn_da1 = Activation('relu')(bn_d1)
+    # d2 = Dense(8)(bn_da1)
+    # bn_d2 = BatchNormalization(momentum=0.98)(d2)
+    # bn_da2 = Activation('relu')(bn_d2)
+    # d3 = Dense(1)(b_gr1)
+    # # bn_d3 = BatchNormalization(momentum=0.98)(d3)
+    # bn_da3 = Activation('softmax')(d3)
+    oup = Reshape(target_shape=(input_dim[0],))(b_gr1)
+    return Model(inp, oup)
+
+def write_model_arch(mod,mod_file):
+    model_f = open(mod_file,'w')
+    model_f.write(mod.to_yaml())
+    model_f.close()
+
+def load_model_arch(mod_file):
+    model_f = open(mod_file,'r')
+    mod = model_from_yaml(model_f.read())
+    model_f.close()
+    return mod
+
+def train_segment(collection_name = 'test',resume_weights='',initial_epoch=0):
+    # collection_name = 'story_test'
+    batch_size = 128
+    # batch_size  = 4
+    model_dir = './models/segment/'+collection_name
+    create_dir(model_dir)
+    log_dir = './logs/segment/'+collection_name
+    create_dir(log_dir)
+    tr_gen_fn,te_x,te_y,copy_read_consts = read_segments_tfrecords_generator(collection_name,batch_size,2*batch_size)
+    tr_gen = tr_gen_fn()
+    n_step,n_features,n_records = copy_read_consts(model_dir)
+    input_dim = (n_step, n_features)
+    model = simple_segment_model(input_dim)
+    # model.output_shape,model.input_shape
+    plot_model(model,show_shapes=True, to_file=model_dir+'/model.png')
+    # loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')([y_pred, labels, input_length, label_length])
+    tb_cb = TensorBoard(
+        log_dir=log_dir,
+        histogram_freq=1,
+        batch_size=32,
+        write_graph=True,
+        write_grads=True,
+        write_images=True,
+        embeddings_freq=0,
+        embeddings_layer_names=None,
+        embeddings_metadata=None)
+    cp_file_fmt = model_dir+'/speech_segment_model-{epoch:02d}-epoch-{val_loss:0.2f}\
+-acc.h5'
+
+    cp_cb = ModelCheckpoint(
+        cp_file_fmt,
+        monitor='val_loss',
+        verbose=0,
+        save_best_only=False,
+        save_weights_only=True,
+        mode='auto',
+        period=1)
+    # train
+    opt = RMSprop()
+    model.compile(loss=categorical_crossentropy, optimizer=opt, metrics=[accuracy])
+    write_model_arch(model,model_dir+'/speech_segment_model_arch.yaml')
+    epoch_n_steps = step_count(n_records,batch_size)
+    if resume_weights != '':
+        model.load_weights(resume_weights)
+    model.fit_generator(tr_gen
+                        , epochs=10000
+                        , steps_per_epoch=epoch_n_steps
+                        , validation_data=(te_x, te_y)
+                        , max_queue_size=32
+                        , callbacks=[tb_cb, cp_cb],initial_epoch=initial_epoch)
+    model.save(model_dir+'/speech_segment_model-final.h5')
+
+    # y_pred = model.predict([te_pairs[:, 0], te_pairs[:, 1]])
+    # te_acc = compute_accuracy(te_y, y_pred)
+    # print('* Accuracy on test set: %0.2f%%' % (100 * te_acc))
+
+
+
+if __name__ == '__main__':
+    # pass
+    train_segment('story_words')#,'./models/segment/story_phrases.1000/speech_segment_model-final.h5',1001)

speech_data.py

@@ -1,220 +1,394 @@
 import pandas as pd
-from pandas_parallel import apply_by_multiprocessing
+from speech_tools import *
+from speech_pitch import *
 # import dask as dd
 # import dask.dataframe as ddf
 import tensorflow as tf
+from tensorflow.python.ops import data_flow_ops
 import numpy as np
-from spectro_gen import generate_aiff_spectrogram
+from speech_spectrum import generate_aiff_spectrogram,generate_sample_spectrogram
+from speech_similar import segmentable_phoneme
 from sklearn.model_selection import train_test_split
-import itertools
-import os
+import os,shutil
 import random
 import csv
 import gc
+import pickle
+import itertools
+from tqdm import tqdm
 
-def get_siamese_pairs(groupF1, groupF2):
-    group1 = [r for (i, r) in groupF1.iterrows()]
-    group2 = [r for (i, r) in groupF2.iterrows()]
-    diff = [(g1, g2) for g2 in group2 for g1 in group1]
-    same = [i for i in itertools.combinations(group1, 2)
-         ] + [i for i in itertools.combinations(group2, 2)]
-    random.shuffle(same)
-    random.shuffle(diff)
-    # return (random.sample(same,10), random.sample(diff,10))
-    return same[:10],diff[:10]
 
 def siamese_pairs(rightGroup, wrongGroup):
     group1 = [r for (i, r) in rightGroup.iterrows()]
     group2 = [r for (i, r) in wrongGroup.iterrows()]
-    rightWrongPairs = [(g1, g2) for g2 in group2 for g1 in group1]
-    rightRightPairs = [i for i in itertools.combinations(group1, 2)]
-    random.shuffle(rightWrongPairs)
-    random.shuffle(rightRightPairs)
-    # return (random.sample(same,10), random.sample(diff,10))
-    return rightRightPairs[:10],rightWrongPairs[:10]
+    rightWrongPairs = [(g1, g2) for g2 in group2 for g1 in group1]#+[(g2, g1) for g2 in group2 for g1 in group1]
+    rightRightPairs = [i for i in itertools.combinations(group1, 2)]#+[i for i in itertools.combinations(group2, 2)]
+    def filter_criteria(s1,s2):
+        same = s1['variant'] == s2['variant']
+        phon_same = s1['phonemes'] == s2['phonemes']
+        voice_diff = s1['voice'] != s2['voice']
+        if not same and phon_same:
+            return False
+        # if same and not voice_diff:
+        #     return False
+        return True
+    validRWPairs = [i for i in rightWrongPairs if filter_criteria(*i)]
+    validRRPairs = [i for i in rightRightPairs if filter_criteria(*i)]
+    random.shuffle(validRWPairs)
+    random.shuffle(validRRPairs)
+    # return rightRightPairs[:10],rightWrongPairs[:10]
+    return validRRPairs[:32],validRWPairs[:32]
 
-def append_zeros(spgr, max_samples):
-    return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)],
-                      'median')
-
-def padd_zeros(spgr, max_samples):
-    return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)],
-                      'constant')
-
-def to_onehot(a,class_count=2):
-    # >>> a = np.array([1, 0, 3])
-    a_row_n = a.shape[0]
-    b = np.zeros((a_row_n, class_count))
-    b[np.arange(a_row_n), a] = 1
-    return b
-
-def create_pair(l, r, max_samples):
-    l_sample = padd_zeros(l, max_samples)
-    r_sample = padd_zeros(r, max_samples)
-    return np.asarray([l_sample, r_sample])
+def seg_siamese_pairs(rightGroup, wrongGroup):
+    group1 = [r for (i, r) in rightGroup.iterrows()]
+    group2 = [r for (i, r) in wrongGroup.iterrows()]
+    rightWrongPairs = [(g1, g2) for g2 in group2 for g1 in group1]#+[(g2, g1) for g2 in group2 for g1 in group1]
+    rightRightPairs = [i for i in itertools.combinations(group1, 2)]#+[i for i in itertools.combinations(group2, 2)]
+    def filter_criteria(s1,s2):
+        same = s1['variant'] == s2['variant']
+        phon_same = s1['phonemes'] == s2['phonemes']
+        voice_diff = s1['voice'] != s2['voice']
+        if not same and phon_same:
+            return False
+        # if same and not voice_diff:
+        #     return False
+        return True
+    validRWPairs = [i for i in rightWrongPairs if filter_criteria(*i)]
+    validRRPairs = [i for i in rightRightPairs if filter_criteria(*i)]
+    random.shuffle(validRWPairs)
+    random.shuffle(validRRPairs)
+    rrPhonePairs = []
+    rwPhonePairs = []
+    def compute_seg_spec(s1,s2):
+        phon_count = len(s1['parsed_phoneme'])
+        seg1_count = len(s1['segments'].index)
+        seg2_count = len(s2['segments'].index)
+        if phon_count == seg1_count and seg2_count == phon_count:
+            s1nd,s2nd = pm_snd(s1['file_path']),pm_snd(s2['file_path'])
+            segs1 = [tuple(x) for x in s1['segments'][['start','end']].values]
+            segs2 = [tuple(x) for x in s2['segments'][['start','end']].values]
+            s1_cp = pd.Series(s1)
+            s2_cp = pd.Series(s2)
+            pp12 = zip(s1['parsed_phoneme'],s2['parsed_phoneme'],segs1,segs2)
+            for (p1,p2,(s1s,s1e),(s2s,s2e)) in pp12:
+                spc1 = generate_sample_spectrogram(s1nd.extract_part(s1s,s1e).values)
+                spc2 = generate_sample_spectrogram(s2nd.extract_part(s2s,s2e).values)
+                s1_cp['spectrogram'] = spc1
+                s2_cp['spectrogram'] = spc2
+                # import pdb; pdb.set_trace()
+                if repr(p1) == repr(p2):
+                    rrPhonePairs.append((s1_cp,s2_cp))
+                else:
+                    rwPhonePairs.append((s1_cp,s2_cp))
+    for (s1,s2) in validRRPairs:
+        compute_seg_spec(s1,s2)
+    for (s1,s2) in validRWPairs:
+        compute_seg_spec(s1,s2)
+    return rrPhonePairs[:32],rwPhonePairs[:32]
+    # return rightRightPairs[:10],rightWrongPairs[:10]
+    # return
+    # validRRPairs[:8],validRWPairs[:8]
 
 
-def create_test_pair(l, r, max_samples):
-    l_sample = append_zeros(l, max_samples)
-    r_sample = append_zeros(r, max_samples)
-    return np.asarray([[l_sample, r_sample]])
 
 
-def create_X(sp, max_samples):
-    return create_pair(sp[0]['spectrogram'], sp[1]['spectrogram'], max_samples)
+def _float_feature(value):
+  return tf.train.Feature(float_list=tf.train.FloatList(value=value))
 
+def _int64_feature(value):
+  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
 
-# def get_word_pairs_data(word, max_samples):
-#     audio_samples = pd.read_csv(
-#         './outputs/audio.csv',
-#         names=['word', 'voice', 'rate', 'variant', 'file'])
-#     audio_samples = audio_samples.loc[audio_samples['word'] ==
-#                                       word].reset_index(drop=True)
-#     audio_samples.loc[:, 'spectrogram'] = audio_samples.loc[:, 'file'].apply(
-#         lambda x: 'outputs/audio/' + x).apply(generate_aiff_spectrogram)
-#     max_samples = audio_samples['spectrogram'].apply(
-#         lambda x: x.shape[0]).max()
-#     same_data, diff_data = [], []
-#     for (w, g) in audio_samples.groupby(audio_samples['word']):
-#         sample_norm = g.loc[audio_samples['variant'] == 'normal']
-#         sample_phon = g.loc[audio_samples['variant'] == 'phoneme']
-#         same, diff = get_siamese_pairs(sample_norm, sample_phon)
-#         same_data.extend([create_X(s, max_samples) for s in same])
-#         diff_data.extend([create_X(d, max_samples) for d in diff])
-#     Y = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))])
-#     X = np.asarray(same_data + diff_data)
-#     # tr_pairs, te_pairs, tr_y, te_y = train_test_split(X, Y, test_size=0.1)
-#     return (X, Y)
+def _bytes_feature(value):
+  return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
 
+def create_spectrogram_tfrecords(audio_group='audio',sample_count=0,train_test_ratio=0.1):
+    '''
+    http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
+    http://www.machinelearninguru.com/deep_learning/tensorflow/basics/tfrecord/tfrecord.html
+    '''
+    audio_samples = pd.read_csv( './outputs/' + audio_group + '.fixed.csv',index_col=0)
+    audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
+    n_records,n_spec,n_features = 0,0,0
 
-def create_spectrogram_data(audio_group='audio'):
-    audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv'
-        , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
-        , quoting=csv.QUOTE_NONE)
-    # audio_samples = audio_samples.loc[audio_samples['word'] ==
-    #                                   'sunflowers'].reset_index(drop=True)
-    audio_samples['file_paths'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
-    audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_paths'], os.path.exists)
-    audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index()
-    audio_samples['spectrogram'] = apply_by_multiprocessing(audio_samples['file_paths'],generate_aiff_spectrogram)#.apply(
-    audio_samples['window_count'] = audio_samples.loc[:,'spectrogram'].apply(lambda x: x.shape[0])
-    audio_samples.to_pickle('outputs/{}-spectrogram.pkl'.format(audio_group))
+    def write_samples(wg,sample_name):
+        word_group_prog = tqdm(wg,desc='Computing spectrogram')
+        record_file = './outputs/{}.{}.tfrecords'.format(audio_group,sample_name)
+        writer = tf.python_io.TFRecordWriter(record_file)
+        for (w, word_group) in word_group_prog:
+            word_group_prog.set_postfix(word=w,sample_name=sample_name)
+            g = word_group.reset_index()
+            # g['spectrogram'] = apply_by_multiprocessing(g['file_path'],pitch_array)
+            g['spectrogram'] = apply_by_multiprocessing(g['file_path'],generate_aiff_spectrogram)
+            # g['spectrogram'] = apply_by_multiprocessing(g['file_path'],compute_mfcc)
+            sample_right = g.loc[g['variant'] == 'low']
+            sample_wrong = g.loc[g['variant'] == 'medium']
+            same, diff = siamese_pairs(sample_right, sample_wrong)
+            groups = [([0,1],same),([1,0],diff)]
+            for (output,group) in groups:
+                group_prog = tqdm(group,desc='Writing Spectrogram')
+                for sample1,sample2 in group_prog:
+                    group_prog.set_postfix(output=output
+                        ,var1=sample1['variant']
+                        ,var2=sample2['variant'])
+                    spectro1,spectro2 = sample1['spectrogram'],sample2['spectrogram']
+                    spec_n1,spec_n2 = spectro1.shape[0],spectro2.shape[0]
+                    spec_w1,spec_w2 = spectro1.shape[1],spectro2.shape[1]
+                    spec1,spec2 = spectro1.reshape(-1),spectro2.reshape(-1)
+                    nonlocal n_spec,n_records,n_features
+                    n_spec = max([n_spec,spec_n1,spec_n2])
+                    n_features = spec_w1
+                    n_records+=1
+                    example = tf.train.Example(features=tf.train.Features(
+                      feature={
+                        'word': _bytes_feature([w.encode('utf-8')]),
+                        'phoneme1': _bytes_feature([sample1['phonemes'].encode('utf-8')]),
+                        'phoneme2': _bytes_feature([sample2['phonemes'].encode('utf-8')]),
+                        'voice1': _bytes_feature([sample1['voice'].encode('utf-8')]),
+                        'voice2': _bytes_feature([sample2['voice'].encode('utf-8')]),
+                        'language': _bytes_feature([sample1['language'].encode('utf-8')]),
+                        'rate1':_int64_feature([sample1['rate']]),
+                        'rate2':_int64_feature([sample2['rate']]),
+                        'variant1': _bytes_feature([sample1['variant'].encode('utf-8')]),
+                        'variant2': _bytes_feature([sample2['variant'].encode('utf-8')]),
+                        'file1': _bytes_feature([sample1['file'].encode('utf-8')]),
+                        'file2': _bytes_feature([sample2['file'].encode('utf-8')]),
+                        'spec1':_float_feature(spec1),
+                        'spec2':_float_feature(spec2),
+                        'spec_n1':_int64_feature([spec_n1]),
+                        'spec_w1':_int64_feature([spec_w1]),
+                        'spec_n2':_int64_feature([spec_n2]),
+                        'spec_w2':_int64_feature([spec_w2]),
+                        'output':_int64_feature(output)
+                      }
+                    ))
+                    writer.write(example.SerializeToString())
+                group_prog.close()
+        word_group_prog.close()
+        writer.close()
 
-def create_spectrogram_tfrecords(audio_group='audio'):
-    audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv'
-        , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']
-        , quoting=csv.QUOTE_NONE)
-    # audio_samples = audio_samples.loc[audio_samples['word'] ==
-    #                                   'sunflowers'].reset_index(drop=True)
+    word_groups = [i for i in audio_samples.groupby('word')]
+    wg_sampled = reservoir_sample(word_groups,sample_count) if sample_count > 0 else word_groups
+    tr_audio_samples,te_audio_samples = train_test_split(wg_sampled,test_size=train_test_ratio)
+    write_samples(tr_audio_samples,'train')
+    write_samples(te_audio_samples,'test')
+    const_file = os.path.join('./outputs',audio_group+'.constants')
+    pickle.dump((n_spec,n_features,n_records),open(const_file,'wb'))
+
+def read_siamese_tfrecords_generator(audio_group='audio',batch_size=32,test_size=0):
+    records_file  = os.path.join('./outputs',audio_group+'.train.tfrecords')
+    input_pairs = []
+    output_class = []
+    const_file = os.path.join('./outputs',audio_group+'.constants')
+    (n_spec,n_features,n_records) = pickle.load(open(const_file,'rb'))
+
+    def copy_read_consts(dest_dir):
+        shutil.copy2(const_file,dest_dir+'/constants.pkl')
+        return (n_spec,n_features,n_records)
+    # @threadsafe_iter
+    def record_generator():
+        print('reading tfrecords({}-train)...'.format(audio_group))
+        input_data = []
+        output_data = []
+        while True:
+            record_iterator,records_count = record_generator_count(records_file)
+            #tqdm(enumerate(record_iterator),total=records_count)
+            #enumerate(record_iterator)
+            for (i,string_record) in enumerate(record_iterator):
+                example = tf.train.Example()
+                example.ParseFromString(string_record)
+                spec_n1 = example.features.feature['spec_n1'].int64_list.value[0]
+                spec_n2 = example.features.feature['spec_n2'].int64_list.value[0]
+                spec_w1 = example.features.feature['spec_w1'].int64_list.value[0]
+                spec_w2 = example.features.feature['spec_w2'].int64_list.value[0]
+                spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(spec_n1,spec_w1)
+                spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(spec_n2,spec_w2)
+                p_spec1,p_spec2 = padd_zeros(spec1,n_spec),padd_zeros(spec2,n_spec)
+                input_data.append(np.asarray([p_spec1,p_spec2]))
+                output = example.features.feature['output'].int64_list.value
+                output_data.append(np.asarray(output))
+                if len(input_data) == batch_size or i == n_records-1:
+                    input_arr = np.asarray(input_data)
+                    output_arr = np.asarray(output_data)
+                    yield ([input_arr[:, 0], input_arr[:, 1]],output_arr)
+                    input_data = []
+                    output_data = []
+
+    # Read test in one-shot
+    print('reading tfrecords({}-test)...'.format(audio_group))
+    te_records_file  = os.path.join('./outputs',audio_group+'.test.tfrecords')
+    te_re_iterator,te_n_records = record_generator_count(te_records_file)
+    test_size = min([test_size,te_n_records]) if test_size > 0 else te_n_records
+    input_data = np.zeros((test_size,2,n_spec,n_features))
+    output_data = np.zeros((test_size,2))
+    random_samples = enumerate(reservoir_sample(te_re_iterator,test_size))
+    for (i,string_record) in tqdm(random_samples,total=test_size):
+        example = tf.train.Example()
+        example.ParseFromString(string_record)
+        spec_n1 = example.features.feature['spec_n1'].int64_list.value[0]
+        spec_n2 = example.features.feature['spec_n2'].int64_list.value[0]
+        spec_w1 = example.features.feature['spec_w1'].int64_list.value[0]
+        spec_w2 = example.features.feature['spec_w2'].int64_list.value[0]
+        spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(spec_n1,spec_w1)
+        spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(spec_n2,spec_w2)
+        p_spec1,p_spec2 = padd_zeros(spec1,n_spec),padd_zeros(spec2,n_spec)
+        input_data[i] = np.asarray([p_spec1,p_spec2])
+        output = example.features.feature['output'].int64_list.value
+        output_data[i] = np.asarray(output)
+
+    return record_generator,input_data,output_data,copy_read_consts
+
+def audio_samples_word_count(audio_group='audio'):
+    audio_samples = pd.read_csv( './outputs/' + audio_group + '.csv')
+    return len(audio_samples.groupby(audio_samples['word']))
+
+def record_generator_count(records_file):
+    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
+    count,spec_n = 0,0
+    for i in record_iterator:
+    #    example = tf.train.Example()
+    #    example.ParseFromString(i)
+    #    spec_n1 = example.features.feature['spec_n1'].int64_list.value[0]
+    #    spec_n2 = example.features.feature['spec_n2'].int64_list.value[0]
+    #    spec_n = max([spec_n,spec_n1,spec_n2])
+        count+=1
+    record_iterator = tf.python_io.tf_record_iterator(path=records_file)
+    return record_iterator,count #,spec_n
+
+def fix_csv(audio_group='audio'):
+    audio_csv_lines = open('./outputs/' + audio_group + '.csv','r').readlines()
+    audio_csv_data = [i.strip().split(',') for i in audio_csv_lines]
+    proper_rows = [i for i in audio_csv_data if len(i) == 7]
+    with open('./outputs/' + audio_group + '.fixed.csv','w') as fixed_csv:
+        fixed_csv_w = csv.writer(fixed_csv, quoting=csv.QUOTE_MINIMAL)
+        fixed_csv_w.writerows(proper_rows)
+    audio_samples = pd.read_csv( './outputs/' + audio_group + '.fixed.csv'
+        , names=['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file'])
     audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
     audio_samples['file_exists'] = apply_by_multiprocessing(audio_samples['file_path'], os.path.exists)
-    audio_samples = audio_samples[audio_samples['file_exists'] == True].reset_index()
+    audio_samples = audio_samples[audio_samples['file_exists'] == True]
+    audio_samples = audio_samples.drop(['file_path','file_exists'],axis=1).reset_index(drop=True)
+    audio_samples.to_csv('./outputs/' + audio_group + '.fixed.csv')
 
-    def _float_feature(value):
-      return tf.train.Feature(float_list=tf.train.FloatList(value=value))
+def convert_old_audio():
+    audio_samples = pd.read_csv( './outputs/audio.csv.old'
+        , names=['word', 'voice', 'rate', 'variant', 'file'])
+    audio_samples['phonemes'] = 'unknown'
+    audio_samples['language'] = 'en-US'
+    audio_samples.loc[audio_samples['variant'] == 'normal','variant'] = 'low'
+    audio_samples.loc[audio_samples['variant'] == 'phoneme','variant'] = 'medium'
+    audio_samples = audio_samples[['word','phonemes', 'voice', 'language', 'rate', 'variant', 'file']]
+    audio_samples.to_csv('./outputs/audio_new.csv',index=False,header=False)
 
-    def _int64_feature(value):
-      return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
+def generate_sppas_trans(audio_group='story_words.all'):
+    # audio_group='story_words.all'
+    audio_samples = pd.read_csv( './outputs/' + audio_group + '.fixed.csv',index_col=0)
+    audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
+    # audio_samples = audio_samples.head(5)
+    rows = tqdm(audio_samples.iterrows(),total = len(audio_samples.index)
+                , desc='Transcribing Words ')
+    for (i,row) in rows:
+    # len(audio_samples.iterrows())
+    # (i,row) = next(audio_samples.iterrows())
+        rows.set_postfix(word=row['word'])
+        transribe_audio_text(row['file_path'],row['word'])
+    rows.close()
 
-    def _bytes_feature(value):
-      return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
+def create_seg_phonpair_tfrecords(audio_group='story_words.all',sample_count=0,train_test_ratio=0.1):
+    audio_samples = pd.read_csv( './outputs/' + audio_group + '.fixed.csv',index_col=0)
+    audio_samples['file_path'] = audio_samples.loc[:, 'file'].apply(lambda x: 'outputs/' + audio_group + '/' + x)
+    audio_samples = audio_samples[(audio_samples['variant'] == 'low') | (audio_samples['variant'] == 'medium')]
+    audio_samples['parsed_phoneme'] = apply_by_multiprocessing(audio_samples['phonemes'],segmentable_phoneme)
+    # audio_samples['sound'] = apply_by_multiprocessing(audio_samples['file_path'],pm_snd)
+    # read_seg_file(audio_samples.iloc[0]['file_path'])
+    audio_samples['segments'] = apply_by_multiprocessing(audio_samples['file_path'],read_seg_file)
+    n_records,n_spec,n_features = 0,0,0
 
-    writer = tf.python_io.TFRecordWriter('./outputs/' + audio_group + '.tfrecords')
-    # audio_samples = audio_samples[:100]
-    for (w, word_group) in audio_samples.groupby(audio_samples['word']):
-        g = word_group.reset_index()
-        g['spectrogram'] = apply_by_multiprocessing(g['file_path'],generate_aiff_spectrogram)
-        sample_right = g.loc[audio_samples['variant'] == 'low']
-        sample_wrong = g.loc[audio_samples['variant'] == 'medium']
-        same, diff = siamese_pairs(sample_right, sample_wrong)
-        groups = [([0,1],same),([1,0],diff)]
-        for (output,group) in groups:
-            for sample1,sample2 in group:
-                spectro1,spectro2 = sample1['spectrogram'],sample2['spectrogram']
-                spec_n1,spec_n2 = spectro1.shape[0],spectro2.shape[0]
-                spec_w1,spec_w2 = spectro1.shape[1],spectro2.shape[1]
-                spec1,spec2 = spectro1.reshape(-1),spectro2.reshape(-1)
-                example = tf.train.Example(features=tf.train.Features(
-                  feature={
-                    'word': _bytes_feature([w.encode('utf-8')]),
-                    'phoneme1': _bytes_feature([sample1['phonemes'].encode('utf-8')]),
-                    'phoneme2': _bytes_feature([sample2['phonemes'].encode('utf-8')]),
-                    'voice1': _bytes_feature([sample1['voice'].encode('utf-8')]),
-                    'voice2': _bytes_feature([sample2['voice'].encode('utf-8')]),
-                    'language': _bytes_feature([sample1['language'].encode('utf-8')]),
-                    'rate1':_int64_feature([sample1['rate']]),
-                    'rate2':_int64_feature([sample2['rate']]),
-                    'variant1': _bytes_feature([sample1['variant'].encode('utf-8')]),
-                    'variant2': _bytes_feature([sample2['variant'].encode('utf-8')]),
-                    'file1': _bytes_feature([sample1['file'].encode('utf-8')]),
-                    'file2': _bytes_feature([sample2['file'].encode('utf-8')]),
-                    'spec1':_float_feature(spec1),
-                    'spec2':_float_feature(spec2),
-                    'spec_n1':_int64_feature([spec_n1]),
-                    'spec_w1':_int64_feature([spec_w1]),
-                    'spec_n2':_int64_feature([spec_n2]),
-                    'spec_w2':_int64_feature([spec_w2]),
-                    'output':_int64_feature(output)
-                  }
-                ))
-                writer.write(example.SerializeToString())
-    writer.close()
-
-def create_tagged_data(audio_samples):
-    same_data, diff_data = [], []
-    for (w, g) in audio_samples.groupby(audio_samples['word']):
-        # sample_norm = g.loc[audio_samples['variant'] == 'low']
-        # sample_phon = g.loc[audio_samples['variant'] == 'medium']
-        sample_norm = g.loc[audio_samples['variant'] == 'normal']
-        sample_phon = g.loc[audio_samples['variant'] == 'phoneme']
-        same, diff = get_siamese_pairs(sample_norm, sample_phon)
-        same_data.extend([create_X(s) for s in same])
-        diff_data.extend([create_X(d) for d in diff])
-    print('creating all speech pairs')
-    Y_f = np.hstack([np.ones(len(same_data)), np.zeros(len(diff_data))])
-    Y = to_onehot(Y_f.astype(np.int8))
-    print('casting as array speech pairs')
-    X = np.asarray(same_data + diff_data)
-    return X,Y
-
-def create_speech_pairs_data(audio_group='audio'):
-    audio_samples = pd.read_pickle('outputs/{}-spectrogram.pkl'.format(audio_group))
-    # sample_size = audio_samples['spectrogram'][0].shape[1]
-    tr_audio_samples,te_audio_samples = train_test_split(audio_samples, test_size=0.1)
-    def save_samples_for(sample_name,samples):
-        print('generating {} siamese speech pairs'.format(sample_name))
-        X,Y = create_tagged_data(samples)
-        print('shuffling array speech pairs')
-        rng_state = np.random.get_state()
-        np.random.shuffle(X)
-        np.random.set_state(rng_state)
-        np.random.shuffle(Y)
-        print('pickling X/Y')
-        np.save('outputs/{}-train-X.npy'.format(audio_group), X)
-        np.save('outputs/{}-train-Y.npy'.format(audio_group), Y)
-    save_samples_for('train',tr_audio_samples)
-    save_samples_for('test',te_audio_samples)
-
-def speech_data(audio_group='audio'):
-    X = np.load('outputs/{}-X.npy'.format(audio_group)) / 255.0
-    Y = np.load('outputs/{}-Y.npy'.format(audio_group))
-    return (X,Y)
-
-def speech_model_data():
-    tr_pairs = np.load('outputs/tr_pairs.npy') / 255.0
-    te_pairs = np.load('outputs/te_pairs.npy') / 255.0
-    tr_pairs[tr_pairs < 0] = 0
-    te_pairs[te_pairs < 0] = 0
-    tr_y = np.load('outputs/tr_y.npy')
-    te_y = np.load('outputs/te_y.npy')
-    return tr_pairs, te_pairs, tr_y, te_y
+    def write_samples(wg,sample_name):
+        word_group_prog = tqdm(wg,desc='Computing PhonPair spectrogram')
+        record_file = './outputs/{}.{}.tfrecords'.format(audio_group,sample_name)
+        writer = tf.python_io.TFRecordWriter(record_file)
+        for (w, word_group) in word_group_prog:
+            word_group_prog.set_postfix(word=w,sample_name=sample_name)
+            g = word_group.reset_index()
+            # g['spectrogram'] = apply_by_multiprocessing(g['file_path'],pitch_array)
+            # g['spectrogram'] = apply_by_multiprocessing(g['file_path'],generate_aiff_spectrogram)
+            # g['spectrogram'] = apply_by_multiprocessing(g['file_path'],compute_mfcc)
+            sample_right = g.loc[g['variant'] == 'low']
+            sample_wrong = g.loc[g['variant'] == 'medium']
+            same, diff = seg_siamese_pairs(sample_right, sample_wrong)
+            groups = [([0,1],same),([1,0],diff)]
+            for (output,group) in groups:
+                group_prog = tqdm(group,desc='Writing Spectrogram')
+                for sample1,sample2 in group_prog:
+                    group_prog.set_postfix(output=output
+                        ,var1=sample1['variant']
+                        ,var2=sample2['variant'])
+                    spectro1,spectro2 = sample1['spectrogram'],sample2['spectrogram']
+                    spec_n1,spec_n2 = spectro1.shape[0],spectro2.shape[0]
+                    spec_w1,spec_w2 = spectro1.shape[1],spectro2.shape[1]
+                    spec1,spec2 = spectro1.reshape(-1),spectro2.reshape(-1)
+                    nonlocal n_spec,n_records,n_features
+                    n_spec = max([n_spec,spec_n1,spec_n2])
+                    n_features = spec_w1
+                    n_records+=1
+                    example = tf.train.Example(features=tf.train.Features(
+                      feature={
+                        'word': _bytes_feature([w.encode('utf-8')]),
+                        'phoneme1': _bytes_feature([sample1['phonemes'].encode('utf-8')]),
+                        'phoneme2': _bytes_feature([sample2['phonemes'].encode('utf-8')]),
+                        'voice1': _bytes_feature([sample1['voice'].encode('utf-8')]),
+                        'voice2': _bytes_feature([sample2['voice'].encode('utf-8')]),
+                        'language': _bytes_feature([sample1['language'].encode('utf-8')]),
+                        'rate1':_int64_feature([sample1['rate']]),
+                        'rate2':_int64_feature([sample2['rate']]),
+                        'variant1': _bytes_feature([sample1['variant'].encode('utf-8')]),
+                        'variant2': _bytes_feature([sample2['variant'].encode('utf-8')]),
+                        'file1': _bytes_feature([sample1['file'].encode('utf-8')]),
+                        'file2': _bytes_feature([sample2['file'].encode('utf-8')]),
+                        'spec1':_float_feature(spec1),
+                        'spec2':_float_feature(spec2),
+                        'spec_n1':_int64_feature([spec_n1]),
+                        'spec_w1':_int64_feature([spec_w1]),
+                        'spec_n2':_int64_feature([spec_n2]),
+                        'spec_w2':_int64_feature([spec_w2]),
+                        'output':_int64_feature(output)
+                      }
+                    ))
+                    writer.write(example.SerializeToString())
+                group_prog.close()
+        word_group_prog.close()
+        writer.close()
 
+    word_groups = [i for i in audio_samples.groupby('word')]
+    wg_sampled = reservoir_sample(word_groups,sample_count) if sample_count > 0 else word_groups
+    tr_audio_samples,te_audio_samples = train_test_split(wg_sampled,test_size=train_test_ratio)
+    write_samples(tr_audio_samples,'train')
+    write_samples(te_audio_samples,'test')
+    const_file = os.path.join('./outputs',audio_group+'.constants')
+    pickle.dump((n_spec,n_features,n_records),open(const_file,'wb'))
 
 if __name__ == '__main__':
     # sunflower_pairs_data()
     # create_spectrogram_data()
     # create_spectrogram_data('story_words')
-    create_spectrogram_tfrecords('story_words')
+    # create_spectrogram_tfrecords('story_words')
+    # create_spectrogram_tfrecords('story_words_test')
+    # read_siamese_tfrecords('story_all')
+    # read_siamese_tfrecords('story_words_test')
+    # padd_zeros_siamese_tfrecords('story_words')
+    # fix_csv('story_words')
+    # pickle_constants('story_words')
+    # create_spectrogram_tfrecords('audio',sample_count=100)
+    # create_spectrogram_tfrecords('story_all',sample_count=25)
+    # fix_csv('story_words_test')
+    # fix_csv('test_5_words')
+    # generate_sppas_trans('test_5_words')
+    create_seg_phonpair_tfrecords('test_5_words')
+    # create_spectrogram_tfrecords('story_words.all',sample_count=0,train_test_ratio=0.1)
+    #record_generator_count()
+    # create_spectrogram_tfrecords('audio',sample_count=50)
+    # read_siamese_tfrecords_generator('audio')
+    # padd_zeros_siamese_tfrecords('audio')
     # create_padded_spectrogram()
     # create_speech_pairs_data()
     # print(speech_model_data())

speech_model.py

@@ -0,0 +1,134 @@
+from __future__ import absolute_import
+from __future__ import print_function
+import numpy as np
+from speech_data import read_siamese_tfrecords_generator
+from keras.models import Model,load_model,model_from_yaml
+from keras.layers import Input,Concatenate,Lambda, BatchNormalization, Dropout
+from keras.layers import Dense, LSTM, Bidirectional, GRU
+from keras.losses import categorical_crossentropy
+from keras.utils import to_categorical
+from keras.optimizers import RMSprop
+from keras.callbacks import TensorBoard, ModelCheckpoint
+from keras import backend as K
+from keras.utils import plot_model
+from speech_tools import create_dir,step_count
+
+
+def create_base_rnn_network(input_dim):
+    '''Base network to be shared (eq. to feature extraction).
+    '''
+    inp = Input(shape=input_dim)
+    # ls0 = LSTM(512, return_sequences=True)(inp)
+    ls1 = LSTM(128, return_sequences=True)(inp)
+    bn_ls1 = BatchNormalization(momentum=0.98)(ls1)
+    ls2 = LSTM(64, return_sequences=True)(bn_ls1)
+    bn_ls2 = BatchNormalization(momentum=0.98)(ls2)
+    # ls3 = LSTM(32, return_sequences=True)(ls2)
+    ls4 = LSTM(32)(bn_ls2)
+    # d1 = Dense(128, activation='relu')(ls4)
+    #d2 = Dense(64, activation='relu')(ls2)
+    return Model(inp, ls4)
+
+
+def compute_accuracy(y_true, y_pred):
+    '''Compute classification accuracy with a fixed threshold on distances.
+    '''
+    pred = y_pred.ravel() > 0.5
+    return np.mean(pred == y_true)
+
+
+def accuracy(y_true, y_pred):
+    '''Compute classification accuracy with a fixed threshold on distances.
+    '''
+    return K.mean(K.equal(y_true, K.cast(y_pred > 0.5, y_true.dtype)))
+
+def dense_classifier(processed):
+    conc_proc = Concatenate()(processed)
+    d1 = Dense(64, activation='relu')(conc_proc)
+    # dr1 = Dropout(0.1)(d1)
+    bn_d1 = BatchNormalization(momentum=0.98)(d1)
+    # d2 = Dense(128, activation='relu')(d1)
+    d3 = Dense(8, activation='relu')(bn_d1)
+    bn_d3 = BatchNormalization(momentum=0.98)(d3)
+    # dr2 = Dropout(0.1)(d2)
+    return Dense(2, activation='softmax')(bn_d3)
+
+def siamese_model(input_dim):
+    base_network = create_base_rnn_network(input_dim)
+    input_a = Input(shape=input_dim)
+    input_b = Input(shape=input_dim)
+    processed_a = base_network(input_a)
+    processed_b = base_network(input_b)
+    final_output = dense_classifier([processed_a,processed_b])
+    model = Model([input_a, input_b], final_output)
+    return model,base_network
+
+def write_model_arch(mod,mod_file):
+    model_f = open(mod_file,'w')
+    model_f.write(mod.to_yaml())
+    model_f.close()
+
+def load_model_arch(mod_file):
+    model_f = open(mod_file,'r')
+    mod = model_from_yaml(model_f.read())
+    model_f.close()
+    return mod
+
+def train_siamese(audio_group = 'audio',resume_weights='',initial_epoch=0):
+    batch_size = 128
+    model_dir = './models/'+audio_group
+    create_dir(model_dir)
+    log_dir = './logs/'+audio_group
+    create_dir(log_dir)
+    tr_gen_fn,te_pairs,te_y,copy_read_consts = read_siamese_tfrecords_generator(audio_group,batch_size=batch_size,test_size=batch_size)
+    n_step,n_features,n_records = copy_read_consts(model_dir)
+    tr_gen = tr_gen_fn()
+    input_dim = (n_step, n_features)
+
+    model,base_model = siamese_model(input_dim)
+    plot_model(model,show_shapes=True, to_file=model_dir+'/model.png')
+    plot_model(base_model,show_shapes=True, to_file=model_dir+'/base_model.png')
+    tb_cb = TensorBoard(
+        log_dir=log_dir,
+        histogram_freq=1,
+        batch_size=32,
+        write_graph=True,
+        write_grads=True,
+        write_images=True,
+        embeddings_freq=0,
+        embeddings_layer_names=None,
+        embeddings_metadata=None)
+    cp_file_fmt = model_dir+'/siamese_speech_model-{epoch:02d}-epoch-{val_loss:0.2f}\
+-acc.h5'
+
+    cp_cb = ModelCheckpoint(
+        cp_file_fmt,
+        monitor='acc',
+        verbose=0,
+        save_best_only=True,
+        save_weights_only=True,
+        mode='auto',
+        period=1)
+    # train
+    rms = RMSprop()
+    model.compile(loss=categorical_crossentropy, optimizer=rms, metrics=[accuracy])
+    write_model_arch(model,model_dir+'/siamese_speech_model_arch.yaml')
+    epoch_n_steps = step_count(n_records,batch_size)
+    if resume_weights != '':
+        model.load_weights(resume_weights)
+    model.fit_generator(tr_gen
+                        , epochs=10000
+                        , steps_per_epoch=epoch_n_steps
+                        , validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y)
+                        , max_queue_size=8
+                        , callbacks=[tb_cb, cp_cb],initial_epoch=initial_epoch)
+    model.save(model_dir+'/siamese_speech_model-final.h5')
+
+    # y_pred = model.predict([te_pairs[:, 0], te_pairs[:, 1]])
+    # te_acc = compute_accuracy(te_y, y_pred)
+    # print('* Accuracy on test set: %0.2f%%' % (100 * te_acc))
+
+
+
+if __name__ == '__main__':
+    train_siamese('test_5_words')

speech_pitch.py

@@ -0,0 +1,158 @@
+import parselmouth as pm
+from pysndfile import sndio as snd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pyaudio as pa
+sns.set() # Use seaborn's default style to make graphs more pretty
+
+
+def pm_snd(sample_file):
+    # sample_file = 'inputs/self-apple/apple-low1.aiff'
+    samples, samplerate, _  = snd.read(sample_file)
+    return pm.Sound(values=samples,sampling_frequency=samplerate)
+
+def pitch_array(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
+    sample_sound = pm_snd(sample_file)
+    sample_pitch = sample_sound.to_pitch()
+    return sample_pitch.to_matrix().as_array()
+
+def intensity_array(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
+    sample_sound = pm_snd(sample_file)
+    sample_intensity = sample_sound.to_mfcc()
+    sample_intensity.as_array().shape
+    return sample_pitch.to_matrix().as_array()
+
+def compute_mfcc(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
+    sample_sound = pm_snd(sample_file)
+    sample_mfcc = sample_sound.to_mfcc()
+    # sample_mfcc.to_array().shape
+    return sample_mfcc.to_array()
+
+def compute_formants(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
+    # sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'
+    sample_sound = pm_snd(sample_file)
+    sample_formant = sample_sound.to_formant_burg()
+    # sample_formant.x_bins()
+    return sample_formant.x_bins()
+
+def draw_spectrogram(spectrogram, dynamic_range=70):
+    X, Y = spectrogram.x_grid(), spectrogram.y_grid()
+    sg_db = 10 * np.log10(spectrogram.values.T)
+    plt.pcolormesh(X, Y, sg_db, vmin=sg_db.max() - dynamic_range, cmap='afmhot')
+    plt.ylim([spectrogram.ymin, spectrogram.ymax])
+    plt.xlabel("time [s]")
+    plt.ylabel("frequency [Hz]")
+
+def draw_intensity(intensity):
+    plt.plot(intensity.xs(), intensity.values, linewidth=3, color='w')
+    plt.plot(intensity.xs(), intensity.values, linewidth=1)
+    plt.grid(False)
+    plt.ylim(0)
+    plt.ylabel("intensity [dB]")
+
+def draw_pitch(pitch):
+    # Extract selected pitch contour, and
+    # replace unvoiced samples by NaN to not plot
+    pitch_values = pitch.to_matrix().values
+    pitch_values[pitch_values==0] = np.nan
+    plt.plot(pitch.xs(), pitch_values, linewidth=3, color='w')
+    plt.plot(pitch.xs(), pitch_values, linewidth=1)
+    plt.grid(False)
+    plt.ylim(0, pitch.ceiling)
+    plt.ylabel("pitch [Hz]")
+
+def draw_formants(formant):
+    # Extract selected pitch contour, and
+    # replace unvoiced samples by NaN to not plot
+    formant_values = formant.to_matrix().values
+    pitch_values[pitch_values==0] = np.nan
+    plt.plot(pitch.xs(), pitch_values, linewidth=3, color='w')
+    plt.plot(pitch.xs(), pitch_values, linewidth=1)
+    plt.grid(False)
+    plt.ylim(0, pitch.ceiling)
+    plt.ylabel("Formants [val]")
+
+
+def plot_sample_raw(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
+    # %matplotlib inline
+    # sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff
+    snd_d = pm_snd(sample_file)
+    plt.figure()
+    plt.plot(snd_d.xs(), snd_d.values)
+    plt.xlim([snd_d.xmin, snd_d.xmax])
+    plt.xlabel("time [s]")
+    plt.ylabel("amplitude")
+    plt.show()
+
+def plot_file_intensity(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
+    snd_d = pm_snd(sample_file)
+    plot_sample_intensity(snd_d)
+
+def plot_sample_intensity(snd_d):
+    intensity = snd_d.to_intensity()
+    spectrogram = snd_d.to_spectrogram()
+    plt.figure()
+    draw_spectrogram(spectrogram)
+    plt.twinx()
+    draw_intensity(intensity)
+    plt.xlim([snd_d.xmin, snd_d.xmax])
+    plt.show()
+
+def plot_file_pitch(sample_file='outputs/audio/sunflowers-Victoria-180-normal-870.aiff'):
+    snd_d = pm_snd(sample_file)
+    plot_sample_pitch(snd_d)
+
+def plot_sample_pitch(snd_d,phons = []):
+    pitch = snd_d.to_pitch()
+    spectrogram = snd_d.to_spectrogram(window_length=0.03, maximum_frequency=8000)
+    plt.figure()
+    draw_spectrogram(spectrogram)
+    plt.twinx()
+    draw_pitch(pitch)
+    for (p,c) in phons:
+        plt.axvline(x=p)
+        plt.text(p,-1,c)
+    plt.xlim([snd_d.xmin, snd_d.xmax])
+    plt.show()
+
+def play_sound(samplerate=22050):
+    #snd_sample = pm_snd('outputs/test/a_warm_smile_and_a_good_heart-1917.aiff')
+    p_oup = pa.PyAudio()
+    stream = p_oup.open(
+        format=pa.paFloat32,
+        channels=2,
+        rate=samplerate,
+        output=True)
+    def sample_player(snd_sample=None):
+        samples = snd_sample.as_array()[:,0]
+
+        one_channel = np.asarray([samples, samples]).T.reshape(-1)
+        audio_data = one_channel.astype(np.float32).tobytes()
+        stream.write(audio_data)
+    def close_player():
+        stream.close()
+        p_oup.terminate()
+    return sample_player,close_player
+    # snd_part = snd_d.extract_part(from_time=0.9, preserve_times=True)
+    # plt.figure()
+    # plt.plot(snd_part.xs(), snd_part.values, linewidth=0.5)
+    # plt.xlim([snd_part.xmin, snd_part.xmax])
+    # plt.xlabel("time [s]")
+    # plt.ylabel("amplitude")
+    # plt.show()
+
+
+if __name__ == '__main__':
+    plot_file_pitch('outputs/audio/sunflowers-Victoria-180-normal-870.aiff')
+    plot_file_pitch('outputs/test/a_warm_smile_and_a_good_heart-1917.aiff')
+    play_sound(pm_snd('outputs/test/a_warm_smile_and_a_good_heart-1917.aiff'))
+    plot_file_pitch('outputs/test/a_wrong_turn-3763.aiff')
+    play_sound(pm_snd('outputs/test/a_wrong_turn-3763.aiff'))
+    plot_file_pitch('inputs/self/a_wrong_turn-low1.aiff')
+    play_sound(pm_snd('inputs/self/a_wrong_turn-low1.aiff'))
+    plot_file_pitch('inputs/self/a_wrong_turn-low2.aiff')
+    play_sound(pm_snd('inputs/self/a_wrong_turn-low2.aiff'))
+    plot_file_pitch('inputs/self/apple-low1.aiff')
+    plot_file_pitch('inputs/self/apple-low2.aiff')
+    plot_file_pitch('inputs/self/apple-medium1.aiff')

speech_samplegen.py

@@ -3,37 +3,27 @@ from AppKit import NSSpeechSynthesizer, NSSpeechInputModeProperty
 from AppKit import NSSpeechModePhoneme
 from Foundation import NSURL
 import json
+import csv
 import random
 import os
 import re
 import subprocess
-import progressbar
+import time
+from tqdm import tqdm
 
-from generate_similar import similar_phoneme,similar_word
+from generate_similar import similar_phoneme_phrase,similar_phrase
+from speech_tools import hms_string,create_dir,format_filename,reservoir_sample
 
-OUTPUT_NAME = 'rand_edu'
+OUTPUT_NAME = 'test_5_words'
 dest_dir = os.path.abspath('.') + '/outputs/' + OUTPUT_NAME + '/'
 dest_file = './outputs/' + OUTPUT_NAME + '.csv'
 
 
-def prog_bar(title):
-    widgets = [title, progressbar.Counter(), 'th entry - ', progressbar.FormatLabel(
-        ''), ' [', progressbar.Bar(), '] - ', progressbar.ETA()]
-    prog = progressbar.ProgressBar(widgets=widgets)
-
-    def update_prog(current):
-        widgets[3] = progressbar.FormatLabel(current)
-        prog.update()
-    return (update_prog, prog)
-
-
-def create_dir(direc):
-    if not os.path.exists(direc):
-        os.makedirs(direc)
-
-
 def dest_filename(w, v, r, t):
-    return '{}-{}-{}-{}-{}.aiff'.format(w, v, r, t, str(random.randint(0, 10000)))
+    rand_no = str(random.randint(0, 10000))
+    fname = '{}-{}-{}-{}-{}.aiff'.format(w, v, r, t, rand_no)
+    sanitized = format_filename(fname)
+    return sanitized
 
 
 def dest_path(v, r, n):
@@ -44,7 +34,7 @@ def dest_path(v, r, n):
 def cli_gen_audio(speech_cmd, rate, voice, out_path):
     subprocess.call(
         ['say', '-v', voice, '-r',
-         str(rate), '-o', out_path, speech_cmd])
+         str(rate), '-o', out_path, "'"+speech_cmd+"'"])
 
 
 class SynthFile(object):
@@ -75,6 +65,11 @@ class SynthFile(object):
 
         return ','.join([str(c) for c in cols])+'\n'
 
+    def get_values(self):
+        cols = [self.word, self.phoneme, self.voice,
+                self.voice_lang, self.rate, self.variant,
+                self.filename]
+        return [str(c) for c in cols]
 
 class SynthVariant(object):
     """docstring for SynthVariant."""
@@ -104,28 +99,28 @@ class SynthVariant(object):
         orig_phon = self.synth.phonemesFromText_('water')
         return orig_phon != ''
 
-    def generate_audio(self, word, variant):
+    def generate_audio(self, text, variant):
         orig_phon, phoneme, phon_cmd = self.synth.phonemesFromText_(
-            word), '', word
+            text), '', text
         if variant == 'low':
             # self.synth.startSpeakingString_toURL_(word,d_url)
             phoneme = orig_phon
         elif variant == 'medium':
-            phoneme = similar_phoneme(orig_phon)
+            phoneme = similar_phoneme_phrase(orig_phon)
             phon_cmd = '[[inpt PHON]] ' + phoneme
         elif variant == 'high':
-            phoneme = similar_word(word)
+            phoneme = similar_phrase(text)
             phon_cmd = phoneme
         # elif variant == 'long':
         # if phon != '':
         # self.phone_synth.startSpeakingString_toURL_(phon,d_url)
         # else:
         #     self.synth.startSpeakingString_toURL_(word,d_url)
-        fname = dest_filename(word, self.name, self.rate, variant)
+        fname = dest_filename(text, self.name, self.rate, variant)
         d_path, r_path = dest_path(self.name, self.rate, fname)
         # d_url = NSURL.fileURLWithPath_(d_path)
         cli_gen_audio(phon_cmd, self.rate, self.name, d_path)
-        return SynthFile(word, phoneme, r_path, self.name, self.lang, self.rate, variant)
+        return SynthFile(text, phoneme, r_path, self.name, self.lang, self.rate, variant)
 
     def create_synth_dirs(self):
         if self.phoneme_capable:
@@ -157,7 +152,7 @@ class SynthVariant(object):
 
 def synth_generator():
     us_voices_ids = SynthVariant.voices_for_lang('en-US')
-    voice_rates = [150, 180, 210, 250]
+    voice_rates = [150, 180, 210]#, 250]
     voice_synths = []
     create_dir(dest_dir)
     for vp in us_voices_ids:
@@ -170,54 +165,49 @@ def synth_generator():
                 print('Discarding phoneme incapable ', s)
 
     def synth_for_words(words, writer):
+        start_time = time.time()
         prog_title = "Synthesizing {} words : ".format(len(words))
         for s in voice_synths:
             s.create_synth_dirs()
             for v in ['low', 'medium', 'high']:
-                (update, prog) = prog_bar(prog_title)
-                for w in prog(words):
-                    update('"{}" with {} variant ({})'.format(w, s, v))
+                prog = tqdm(words)
+                prog.set_postfix(variant=v,voice=s.name,rate=s.rate)
+                for w in tqdm(words):
+                    prog.set_description('Synthesizing text:"{}"'.format(w))
                     synthed = s.generate_audio(w, v)
                     writer(synthed)
+                prog.close()
+        end_time = time.time()
+        time_str = hms_string(end_time - start_time)
+        print("It took {} to synthsize all variants.".format(time_str))
     return synth_for_words
 
-
-def write_synths(synth_list, fname, csv=False):
+def synth_logger(fname, csv_mode=False):
     f = open(fname, 'w')
-    if csv:
-        for s in synth_list:
-            f.write(s.get_csv())
-    else:
-        json.dump([s.get_json() for s in synth_list], f)
-    f.close()
-
-
-def synth_logger(fname, csv=False):
-    f = open(fname, 'w')
-
+    s_csv_w = csv.writer(f, quoting=csv.QUOTE_MINIMAL)
     def csv_writer(s):
-        f.write(s.get_csv())
+        s_csv_w.writerow(s.get_values())
     synth_list = []
 
     def json_writer(s):
         synth_list.append(s)
 
     def close_file():
-        if csv:
+        if csv_mode:
             f.close()
         else:
             json.dump([s.get_json() for s in synth_list], f)
             f.close()
-    if csv:
+    if csv_mode:
         return csv_writer, close_file
     else:
         return json_writer, close_file
 
 def generate_audio_for_text_list(text_list):
-    (writer, closer) = synth_logger(dest_file, csv=True)
-    synth_for_words = synth_generator()
+    (writer, closer) = synth_logger(dest_file, csv_mode=True)
+    synth_for_texts = synth_generator()
     try:
-        synth_for_words(text_list, writer)
+        synth_for_texts(text_list, writer)
     except:
         import traceback
         import sys
@@ -226,22 +216,37 @@ def generate_audio_for_text_list(text_list):
     closer()
 
 def generate_audio_for_stories():
+    '''
+    Generates the audio sample variants for the list of words in the stories
+    '''
     # story_file = './inputs/all_stories_hs.json'
     story_file = './inputs/all_stories.json'
     stories_data = json.load(open(story_file))
-    # word_list = [t[0] for i in stories_data.values() for t in i]
-    word_list = [i for g in stories_data.values() for i in g]
-    (writer, closer) = synth_logger(dest_file, csv=True)
-    synth_for_words = synth_generator()
-    try:
-        synth_for_words(word_list, writer)
-    except:
-        import traceback
-        import sys
-        traceback.print_exc(file=sys.stdout)
-        pass
-    closer()
+    # text_list_dup = [t[0] for i in stories_data.values() for t in i]
+    text_list_dup = [t for i in stories_data.values() for t in i]
+    text_list = sorted(list(set(text_list_dup)))
+    generate_audio_for_text_list(text_list)
+
+def generate_test_audio_for_stories(sample_count=0):
+    '''
+    Picks a list of words from the wordlist that are not in story words
+    and generates the variants
+    '''
+    story_file = './inputs/all_stories_hs.json'
+    # story_file = './inputs/all_stories.json'
+    stories_data = json.load(open(story_file))
+    text_list_dup = [t[0] for i in stories_data.values() for t in i]
+    text_list = sorted(list(set(text_list_dup)))
+    # text_list = [i.replace('-','') for g in stories_data.values() for i in g]
+    word_list = [i.strip('\n_') for i in open('./inputs/wordlist.txt','r').readlines()]
+    text_set = set(text_list)
+    new_word_list = [i for i in word_list if i not in text_set and len(i) > 4]
+    # test_words = new_word_list[:int(len(text_list)/5+1)]
+    test_words = reservoir_sample(new_word_list,sample_count) if sample_count > 0 else new_word_list
+    generate_audio_for_text_list(test_words)
+
 
 if __name__ == '__main__':
-    generate_audio_for_text_list(['random','education'])
+    generate_test_audio_for_stories(5)
+    # generate_audio_for_text_list(['I want to go home','education'])
     # generate_audio_for_stories()

speech_segmentgen.py

@@ -0,0 +1,237 @@
+import objc
+from AppKit import *
+from Foundation import NSURL
+from PyObjCTools import AppHelper
+from time import time
+import os
+import sys
+import random
+import json
+import csv
+import subprocess
+from tqdm import tqdm
+from speech_tools import create_dir,format_filename
+
+apple_phonemes = [
+    '%', '@', 'AE', 'EY', 'AO', 'AX', 'IY', 'EH', 'IH', 'AY', 'IX', 'AA', 'UW',
+    'UH', 'UX', 'OW', 'AW', 'OY', 'b', 'C', 'd', 'D', 'f', 'g', 'h', 'J', 'k',
+    'l', 'm', 'n', 'N', 'p', 'r', 's', 'S', 't', 'T', 'v', 'w', 'y', 'z', 'Z'
+]
+
+OUTPUT_NAME = 'story_test_segments'
+
+dest_dir = os.path.abspath('.') + '/outputs/' + OUTPUT_NAME + '/'
+csv_dest_file = os.path.abspath('.') + '/outputs/' + OUTPUT_NAME + '.csv'
+create_dir(dest_dir)
+
+
+def cli_gen_audio(speech_cmd, out_path):
+    subprocess.call(
+        ['say', '-o', out_path, "'" + speech_cmd + "'"])
+
+
+class SpeechDelegate (NSObject):
+    def speechSynthesizer_willSpeakWord_ofString_(self, sender, word, text):
+        '''Called automatically when the application has launched'''
+        # print("Speaking word {} in sentence {}".format(word,text))
+        self.wordWillSpeak()
+
+    def speechSynthesizer_willSpeakPhoneme_(self, sender, phoneme):
+        phon_ch = apple_phonemes[phoneme]
+        self.phonemeWillSpeak(phon_ch)
+
+    def speechSynthesizer_didFinishSpeaking_(self, synth, didFinishSpeaking):
+        if didFinishSpeaking:
+            self.completeCB()
+
+    def setC_W_Ph_(self, completed, word, phoneme):
+        self.completeCB = completed
+        self.wordWillSpeak = word
+        self.phonemeWillSpeak = phoneme
+
+# del SpeechDelegate
+
+
+class Delegate (NSObject):
+    def applicationDidFinishLaunching_(self, aNotification):
+        '''Called automatically when the application has launched'''
+        print("App Launched!")
+        # phrases = story_texts()#random.sample(story_texts(), 100)  #
+        # phrases = test_texts(30)
+        phrases = story_words()
+        # print(phrases)
+        generate_audio(phrases)
+
+
+class PhonemeTiming(object):
+    """docstring for PhonemeTiming."""
+
+    def __init__(self, phon, start):
+        super(PhonemeTiming, self).__init__()
+        self.phoneme = phon
+        self.start = start
+        self.fraction = 0
+        self.duration = None
+        self.end = None
+
+    def is_audible(self):
+        return self.phoneme not in ['%', '~']
+
+    def tune(self):
+        if self.is_audible():
+            dur_ms = int(self.duration * 1000)
+            return '{} {{D {}}}'.format(self.phoneme, dur_ms)
+        else:
+            return '~'
+
+    def __repr__(self):
+        return '[{}]({:0.4f})'.format(self.phoneme, self.fraction)
+
+    @staticmethod
+    def to_tune(phone_ts):
+        tune_list = ['[[inpt TUNE]]']
+        for ph in phone_ts:
+            tune_list.append(ph.tune())
+        tune_list.append('[[inpt TEXT]]')
+        return '\n'.join(tune_list)
+
+
+class SegData(object):
+    """docstring for SegData."""
+
+    def __init__(self, text, filename):
+        super(SegData, self).__init__()
+        self.text = text
+        self.tune = ''
+        self.filename = filename
+        self.segments = []
+
+    def csv_rows(self):
+        result = []
+        s_tim = self.segments[0].start
+        for i in range(len(self.segments) - 1):
+            cs = self.segments[i]
+            # if cs.is_audible():
+            ns = self.segments[i + 1]
+            row = [self.text, self.filename, cs.phoneme, ns.phoneme,
+                   (cs.start - s_tim) * 1000, (cs.end - s_tim) * 1000]
+            result.append(row)
+        return result
+
+
+class SynthesizerQueue(object):
+    """docstring for SynthesizerQueue."""
+
+    def __init__(self):
+        super(SynthesizerQueue, self).__init__()
+        self.synth = NSSpeechSynthesizer.alloc().init()
+        self.didComplete = None
+        q_delg = SpeechDelegate.alloc().init()
+        self.synth.setDelegate_(q_delg)
+
+        def synth_complete():
+            end_time = time()
+            for i in range(len(self.phoneme_timing)):
+                if i == len(self.phoneme_timing) - 1:
+                    self.phoneme_timing[i].duration = end_time - \
+                        self.phoneme_timing[i].start
+                    self.phoneme_timing[i].end = end_time
+                else:
+                    self.phoneme_timing[i].duration = self.phoneme_timing[i +
+                                                                          1].start - self.phoneme_timing[i].start
+                    self.phoneme_timing[i].end = self.phoneme_timing[i + 1].start
+
+            total_time = sum(
+                [i.duration for i in self.phoneme_timing if i.is_audible()])
+            for ph in self.phoneme_timing:
+                if ph.is_audible():
+                    ph.fraction = ph.duration / total_time
+            if self.didComplete:
+                self.data.segments = self.phoneme_timing
+                self.data.tune = PhonemeTiming.to_tune(self.phoneme_timing)
+                self.didComplete(self.data)
+
+        def will_speak_phoneme(phon):
+            phtm = PhonemeTiming(phon, time())
+            self.phoneme_timing.append(phtm)
+
+        def will_speak_word():
+            pass
+            # coz it comes after the first phoneme of the word is started
+            # phtm = PhonemeTiming('~', time())
+            # self.phoneme_timing.append(phtm)
+
+        q_delg.setC_W_Ph_(synth_complete, will_speak_word, will_speak_phoneme)
+
+    def queueTask(self, text):
+        rand_no = str(random.randint(0, 10000))
+        fname = '{}-{}.aiff'.format(text, rand_no)
+        sanitized = format_filename(fname)
+        dest_file = dest_dir + sanitized
+        cli_gen_audio(text, dest_file)
+        self.phoneme_timing = []
+        self.data = SegData(text, sanitized)
+        self.synth.startSpeakingString_(text)
+
+
+def story_texts():
+    story_file = './inputs/all_stories.json'
+    stories_data = json.load(open(story_file))
+    text_list_dup = [t for i in stories_data.values() for t in i]
+    text_list = sorted(list(set(text_list_dup)))
+    return text_list
+
+def story_words():
+    story_file = './inputs/all_stories_hs.json'
+    stories_data = json.load(open(story_file))
+    text_list_dup = [t[0] for i in stories_data.values() for t in i]
+    text_list = sorted(list(set(text_list_dup)))
+    return text_list
+
+def test_texts(count=10):
+    word_list = [i.strip('\n_') for i in open('./inputs/wordlist.txt','r').readlines()]
+    text_list = sorted(random.sample(list(set(word_list)),count))
+    return text_list
+
+def generate_audio(phrases):
+    synthQ = SynthesizerQueue()
+    f = open(csv_dest_file, 'w')
+    s_csv_w = csv.writer(f, quoting=csv.QUOTE_MINIMAL)
+    i = 0
+    p = tqdm(total=len(phrases))
+
+    def nextTask(seg_data=None):
+        nonlocal i
+        if i < len(phrases):
+            p.set_postfix(phrase=phrases[i])
+            p.update()
+            synthQ.queueTask(phrases[i])
+            i += 1
+        else:
+            p.close()
+            f.close()
+            dg = NSApplication.sharedApplication().delegate
+            print('App terminated.')
+            NSApp().terminate_(dg)
+        if seg_data:
+            s_csv_w.writerows(seg_data.csv_rows())
+    synthQ.didComplete = nextTask
+    nextTask()
+
+
+def main():
+
+    # Create a new application instance ...
+    a = NSApplication.sharedApplication()
+    # ... and create its delgate.  Note the use of the
+    # Objective C constructors below, because Delegate
+    # is a subcalss of an Objective C class, NSObject
+    delegate = Delegate.alloc().init()
+    # Tell the application which delegate object to use.
+    a.setDelegate_(delegate)
+
+    AppHelper.runEventLoop()
+
+
+if __name__ == '__main__':
+    main()

speech_siamese.py

@@ -1,135 +0,0 @@
-from __future__ import absolute_import
-from __future__ import print_function
-import numpy as np
-from speech_data import speech_model_data
-from keras.models import Model,load_model
-from keras.layers import Input, Dense, Dropout, LSTM, Lambda, Concatenate
-from keras.losses import categorical_crossentropy
-# from keras.losses import binary_crossentropy
-from keras.utils import to_categorical
-# from keras.utils.np_utils import to_categorical
-from keras.optimizers import RMSprop
-from keras.callbacks import TensorBoard, ModelCheckpoint
-from keras import backend as K
-
-
-def euclidean_distance(vects):
-    x, y = vects
-    return K.sqrt(
-        K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
-
-
-def eucl_dist_output_shape(shapes):
-    shape1, shape2 = shapes
-    return (shape1[0], 1)
-
-
-def contrastive_loss(y_true, y_pred):
-    '''Contrastive loss from Hadsell-et-al.'06
-    http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
-    '''
-    return K.mean(y_true * K.square(y_pred) +
-                  (1 - y_true) * K.square(K.maximum(1 - y_pred, 0)))
-
-def create_base_rnn_network(input_dim):
-    '''Base network to be shared (eq. to feature extraction).
-    '''
-    inp = Input(shape=input_dim)
-    ls1 = LSTM(256, return_sequences=True)(inp)
-    ls2 = LSTM(128, return_sequences=True)(ls1)
-    # ls3 = LSTM(32, return_sequences=True)(ls2)
-    ls4 = LSTM(64)(ls2)
-    return Model(inp, ls4)
-
-
-def compute_accuracy(y_true, y_pred):
-    '''Compute classification accuracy with a fixed threshold on distances.
-    '''
-    pred = y_pred.ravel() < 0.5
-    return np.mean(pred == y_true)
-
-
-def accuracy(y_true, y_pred):
-    '''Compute classification accuracy with a fixed threshold on distances.
-    '''
-    return K.mean(K.equal(y_true, K.cast(y_pred > 0.5, y_true.dtype)))
-
-def dense_classifier(processed):
-    conc_proc = Concatenate()(processed)
-    d1 = Dense(16, activation='relu')(conc_proc)
-    # dr1 = Dropout(0.1)(d1)
-    d2 = Dense(8, activation='relu')(d1)
-    # dr2 = Dropout(0.1)(d2)
-    return Dense(2, activation='softmax')(d2)
-
-def siamese_model(input_dim):
-    # input_dim = (15, 1654)
-    base_network = create_base_rnn_network(input_dim)
-    input_a = Input(shape=input_dim)
-    input_b = Input(shape=input_dim)
-    processed_a = base_network(input_a)
-    processed_b = base_network(input_b)
-    final_output = dense_classifier([processed_a,processed_b])
-    model = Model([input_a, input_b], final_output)
-    # distance = Lambda(
-    #     euclidean_distance,
-    #     output_shape=eucl_dist_output_shape)([processed_a, processed_b])
-    # model = Model([input_a, input_b], distance)
-    return model
-
-
-def train_siamese():
-    # the data, shuffled and split between train and test sets
-    tr_pairs, te_pairs, tr_y_e, te_y_e = speech_model_data()
-    tr_y = to_categorical(tr_y_e, num_classes=2)
-    te_y = to_categorical(te_y_e, num_classes=2)
-    input_dim = (tr_pairs.shape[2], tr_pairs.shape[3])
-
-    model = siamese_model(input_dim)
-
-    tb_cb = TensorBoard(
-        log_dir='./logs/siamese_logs',
-        histogram_freq=1,
-        batch_size=32,
-        write_graph=True,
-        write_grads=True,
-        write_images=True,
-        embeddings_freq=0,
-        embeddings_layer_names=None,
-        embeddings_metadata=None)
-    cp_file_fmt = './models/siamese_speech_model-{epoch:02d}-epoch-{val_loss:0.2f}\
--acc.h5'
-
-    cp_cb = ModelCheckpoint(
-        cp_file_fmt,
-        monitor='val_loss',
-        verbose=0,
-        save_best_only=False,
-        save_weights_only=False,
-        mode='auto',
-        period=1)
-    # train
-    rms = RMSprop(lr=0.001)
-    model.compile(loss=categorical_crossentropy, optimizer=rms, metrics=[accuracy])
-    model.fit(
-        [tr_pairs[:, 0], tr_pairs[:, 1]],
-        tr_y,
-        batch_size=128,
-        epochs=50,
-        validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y),
-        callbacks=[tb_cb, cp_cb])
-
-    model.save('./models/siamese_speech_model-final.h5')
-    # compute final accuracy on training and test sets
-    y_pred = model.predict([tr_pairs[:, 0], tr_pairs[:, 1]])
-    tr_acc = compute_accuracy(tr_y, y_pred)
-    y_pred = model.predict([te_pairs[:, 0], te_pairs[:, 1]])
-    te_acc = compute_accuracy(te_y, y_pred)
-
-    print('* Accuracy on training set: %0.2f%%' % (100 * tr_acc))
-    print('* Accuracy on test set: %0.2f%%' % (100 * te_acc))
-
-
-
-if __name__ == '__main__':
-    train_siamese()

speech_similar.py

@@ -0,0 +1,143 @@
+import pandas as pd
+import pronouncing
+import re
+import numpy as np
+import random
+
+# mapping = {
+#     s.split()[0]: s.split()[1]
+#     for s in """
+# AA AA
+# AE AE
+# AH UX
+# AO AO
+# AW AW
+# AY AY
+# B  b
+# CH C
+# D  d
+# DH D
+# EH EH
+# ER UXr
+# EY EY
+# F  f
+# G  g
+# HH h
+# IH IH
+# IY IY
+# JH J
+# K  k
+# L  l
+# M  m
+# N  n
+# NG N
+# OW OW
+# OY OY
+# P  p
+# R  r
+# S  s
+# SH S
+# T  t
+# TH T
+# UH UH
+# UW UW
+# V  v
+# W  w
+# Y  y
+# Z  z
+# ZH Z
+# """.strip().split('\n')
+# }
+
+# sim_mat = pd.read_csv('./similarity.csv', header=0, index_col=0)
+#
+#
+# def convert_ph(ph):
+#     stress_level = re.search("(\w+)([0-9])", ph)
+#     if stress_level:
+#         return stress_level.group(2) + mapping[stress_level.group(1)]
+#     else:
+#         return mapping[ph]
+#
+#
+# def sim_mat_to_apple_table(smt):
+#     colnames = [convert_ph(ph) for ph in smt.index.tolist()]
+#     smt = pd.DataFrame(np.nan_to_num(smt.values))
+#     fsmt = (smt.T + smt)
+#     np.fill_diagonal(fsmt.values, 100.0)
+#     asmt = pd.DataFrame.copy(fsmt)
+#     asmt.columns = colnames
+#     asmt.index = colnames
+#     apple_sim_table = asmt.stack().reset_index()
+#     apple_sim_table.columns = ['q', 'r', 's']
+#     return apple_sim_table
+#
+#
+# apple_sim_table = sim_mat_to_apple_table(sim_mat)
+#
+#
+# def top_match(ph):
+#     selected = apple_sim_table[(apple_sim_table.q == ph)
+#                                & (apple_sim_table.s < 100) &
+#                                (apple_sim_table.s >= 70)]
+#     tm = ph
+#     if len(selected) > 0:
+#         tm = pd.DataFrame.sort_values(selected, 's', ascending=False).iloc[0].r
+#     return tm
+
+
+apple_phonemes = [
+    '%', '@', 'AE', 'EY', 'AO', 'AX', 'IY', 'EH', 'IH', 'AY', 'IX', 'AA', 'UW',
+    'UH', 'UX', 'OW', 'AW', 'OY', 'b', 'C', 'd', 'D', 'f', 'g', 'h', 'J', 'k',
+    'l', 'm', 'n', 'N', 'p', 'r', 's', 'S', 't', 'T', 'v', 'w', 'y', 'z', 'Z'
+]
+
+class ApplePhoneme(object):
+    """docstring for ApplePhoneme."""
+
+    def __init__(self, phone, stress, vowel=False):
+        super(ApplePhoneme, self).__init__()
+        self.phone = phone
+        self.stress = stress
+        self.vowel = vowel
+
+    def __str__(self):
+        return (str(self.stress) if (self.vowel and self.stress>0) else '') + self.phone
+
+    def __repr__(self):
+        return "'{}'".format(str(self))
+
+    def adjust_stress(self):
+        self.stress = random.choice([i for i in range(3) if i != self.stress])
+
+
+def parse_apple_phonemes(ph_str):
+    for i in range(len(ph_str)):
+        pref, rest = ph_str[:i + 1], ph_str[i + 1:]
+        if pref in apple_phonemes:
+            vowel = pref[0] in 'AEIOU'
+            return [ApplePhoneme(pref, 0, vowel)] + parse_apple_phonemes(rest)
+        elif pref[0].isdigit() and pref[1:] in apple_phonemes:
+            return [ApplePhoneme(pref[1:], int(pref[0]) , True)] + parse_apple_phonemes(rest)
+        elif not pref.isalnum():
+            return [ApplePhoneme(pref, -1, False)] + parse_apple_phonemes(rest)
+    return []
+
+def segmentable_phoneme(ph_str):
+    return [p for p in parse_apple_phonemes(ph_str) if p.stress >=0]
+
+def similar_phoneme_word(ph_str):
+    phons = parse_apple_phonemes(ph_str)
+    vowels = [i for i in phons if i.vowel]
+    random.choice(vowels).adjust_stress()
+    return ''.join([str(i) for i in phons])
+
+def similar_phoneme_phrase(ph_str):
+    return ' '.join([similar_phoneme_word(w) for w in ph_str.split()])
+
+def similar_word(word_str):
+    similar = pronouncing.rhymes(word_str)
+    return random.choice(similar) if len(similar) > 0 else word_str
+
+def similar_phrase(ph_str):
+    return ' '.join([similar_word(w) for w in ph_str.split()])

speech_spectrum.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-
 """ This work is licensed under a Creative Commons Attribution 3.0 Unported
     License.
     Frank Zalkow, 2012-2013
@@ -13,6 +12,9 @@ from pysndfile import sndio as snd
 from numpy.lib import stride_tricks
 """ short time fourier transform of audio signal """
 
+STFT_WINDOWS_MSEC = 20
+STFT_WINDOW_OVERLAP = 1.0 / 3
+
 
 def stft(sig, frameSize, overlapFac=0.5, window=np.hanning):
     win = window(frameSize)
@@ -20,9 +22,7 @@ def stft(sig, frameSize, overlapFac=0.5, window=np.hanning):
 
     # zeros at beginning (thus center of 1st window should be for sample nr. 0)
     # sig = (sig*255).astype(np.uint8)
-    # import pdb;pdb.set_trace()
     count = int(np.floor(frameSize / 2.0))
-    # import pdb;pdb.set_trace()
     samples = np.append(np.zeros(count), sig)
     # cols for windowing
     cols = int(np.ceil((len(samples) - frameSize) / float(hopSize)) + 1)
@@ -34,7 +34,6 @@ def stft(sig, frameSize, overlapFac=0.5, window=np.hanning):
         shape=(cols, frameSize),
         strides=(samples.strides[0] * hopSize, samples.strides[0])).copy()
     frames *= win
-
     return np.fft.rfft(frames)
 
 
@@ -47,7 +46,6 @@ def logscale_spec(spec, sr=44100, factor=20.):
     scale = np.linspace(0, 1, freqbins)**factor
     scale *= (freqbins - 1) / max(scale)
     scale = np.unique(np.round(scale)).astype(np.uint32)
-    # import pdb;pdb.set_trace()
     # create spectrogram with new freq bins
     newspec = np.complex128(np.zeros([timebins, len(scale)]))
     for i in range(0, len(scale)):
@@ -55,7 +53,6 @@ def logscale_spec(spec, sr=44100, factor=20.):
             newspec[:, i] = np.sum(spec[:, scale[i]:], axis=1)
         else:
             newspec[:, i] = np.sum(spec[:, scale[i]:scale[i + 1]], axis=1)
-
     # list center freq of bins
     allfreqs = np.abs(np.fft.fftfreq(freqbins * 2, 1. / sr)[:freqbins + 1])
     freqs = []
@@ -64,7 +61,6 @@ def logscale_spec(spec, sr=44100, factor=20.):
             freqs += [np.mean(allfreqs[scale[i]:])]
         else:
             freqs += [np.mean(allfreqs[scale[i]:scale[i + 1]])]
-
     return newspec, freqs
 
 
@@ -74,13 +70,18 @@ def logscale_spec(spec, sr=44100, factor=20.):
 def generate_spec_frec(samples, samplerate):
     # samplerate, samples = wav.read(audiopath)
     # s = stft(samples, binsize)
-    s = stft(samples, samplerate * 150 // 1000, 1.0 / 3)
-
+    s = stft(samples, samplerate * STFT_WINDOWS_MSEC // 1000,
+             STFT_WINDOW_OVERLAP)
     sshow, freq = logscale_spec(s, factor=1.0, sr=samplerate)
-    ims = 20. * np.log10(np.abs(sshow) / 10e-6)
-    ims[ims<0] = 0
+    # add epison so that log10 doesn't break
+    sshow_abs = np.abs(sshow + np.finfo(sshow.dtype).eps)
+    ims = 20. * np.log10(sshow_abs / 10e-6)
+    ims[ims < 0] = 0  #np.finfo(sshow.dtype).eps
     return ims, freq
 
+def generate_sample_spectrogram(samples):
+    ims, _ = generate_spec_frec(samples, 22050)
+    return ims
 
 def generate_aiff_spectrogram(audiopath):
     samples, samplerate, _ = snd.read(audiopath)
@@ -88,7 +89,11 @@ def generate_aiff_spectrogram(audiopath):
     return ims
 
 
-def plot_stft(samples, samplerate, binsize=2**10, plotpath=None, colormap="jet"):
+def plot_stft(samples,
+              samplerate,
+              binsize=2**10,
+              plotpath=None,
+              colormap="jet"):
     (ims, freq) = generate_spec_frec(samples, samplerate)
     timebins, freqbins = np.shape(ims)
     plt.figure(figsize=(15, 7.5))
@@ -126,8 +131,10 @@ def plot_aiff_stft(audiopath, binsize=2**10, plotpath=None, colormap="jet"):
 
 
 def play_sunflower():
-    sample_r = snd.get_info('./outputs/audio/sunflowers-Alex-150-normal-589.aiff')[0]
-    snd_data_f64 = snd.read('./outputs/audio/sunflowers-Alex-150-normal-589.aiff')[0]
+    sample_r = snd.get_info(
+        './outputs/audio/sunflowers-Alex-150-normal-589.aiff')[0]
+    snd_data_f64 = snd.read(
+        './outputs/audio/sunflowers-Alex-150-normal-589.aiff')[0]
     snd_data_f32 = snd_data_f64.astype(np.float32)
     print(snd_data_f32.shape)
     snd_data = snd_data_f32.tobytes()
@@ -141,8 +148,13 @@ def play_sunflower():
 
 
 if __name__ == '__main__':
-    play_sunflower()
-    # plot_aiff_stft('./outputs/sunflowers-Alex-150-normal-589.aiff')
+    # play_sunflower()
+    plot_aiff_stft(
+        './outputs/story_words/Agnes/150/chicken-Agnes-150-low-1077.aiff')
+    plot_aiff_stft(
+        './outputs/story_words/Agnes/150/chicken-Agnes-150-medium-1762.aiff')
+    # spec = generate_aiff_spectrogram('./outputs/story_words/Agnes/150/chicken-Agnes-150-low-1077.aiff')
+    # print(spec.shape)
     # plot_aiff_stft('./outputs/sunflowers-Alex-180-normal-4763.aiff')
     # plot_aiff_stft('./outputs/sunflowers-Victoria-180-normal-870.aiff')
     # plot_aiff_stft('./outputs/sunflowers-Fred-180-phoneme-9733.aiff')

speech_test.py

@@ -0,0 +1,202 @@
+from speech_model import load_model_arch
+from speech_tools import record_spectrogram, file_player, padd_zeros, pair_for_word
+from speech_data import record_generator_count
+# from importlib import reload
+# import speech_data
+# reload(speech_data)
+import numpy as np
+import pandas as pd
+import os
+import pickle
+import tensorflow as tf
+import csv
+from tqdm import tqdm
+from speech_data import padd_zeros
+import seaborn as sns
+
+def predict_recording_with(m,sample_size=15):
+    spec1 = record_spectrogram(n_sec=1.4)
+    spec2 = record_spectrogram(n_sec=1.4)
+    inp = create_test_pair(spec1,spec2,sample_size)
+    return m.predict([inp[:, 0], inp[:, 1]])
+
+def predict_tts_sample(sample_word = 'able',audio_group='story_words',weights = 'siamese_speech_model-153-epoch-0.55-acc.h5'):
+    # sample_word = 'able';audio_group='story_words';weights = 'siamese_speech_model-153-epoch-0.55-acc.h5'
+    const_file = './models/'+audio_group+'/constants.pkl'
+    arch_file='./models/'+audio_group+'/siamese_speech_model_arch.yaml'
+    weight_file='./models/'+audio_group+'/'+weights
+    (sample_size,n_features,n_records) = pickle.load(open(const_file,'rb'))
+    model = load_model_arch(arch_file)
+    model.load_weights(weight_file)
+    spec1,spec2 = pair_for_word(sample_word)
+    p_spec1 = padd_zeros(spec1,sample_size)
+    p_spec2 = padd_zeros(spec2,sample_size)
+    inp = np.array([[p_spec1,p_spec2]])
+    result = model.predict([inp[:, 0], inp[:, 1]])[0]
+    res_str = 'same' if result[0] < result[1] else 'diff'
+    return res_str
+
+def test_with(audio_group):
+    X,Y = speech_data(audio_group)
+    print(np.argmax(model.predict([X[:, 0], X[:, 1]]),axis=1))
+    print(Y.astype(np.int8))
+
+def evaluate_siamese(records_file,audio_group='audio',weights = 'siamese_speech_model-final.h5'):
+    # audio_group='audio';model_file = 'siamese_speech_model-305-epoch-0.20-acc.h5'
+    # records_file  = os.path.join('./outputs',eval_group+'.train.tfrecords')
+    const_file = os.path.join('./models/'+audio_group+'/','constants.pkl')
+    arch_file='./models/'+audio_group+'/siamese_speech_model_arch.yaml'
+    weight_file='./models/'+audio_group+'/'+weights
+    (n_spec,n_features,n_records) = pickle.load(open(const_file,'rb'))
+    print('evaluating {}...'.format(records_file))
+    model = load_model_arch(arch_file)
+    # model = siamese_model((n_spec, n_features))
+    model.load_weights(weight_file)
+    record_iterator,records_count = record_generator_count(records_file)
+    total,same_success,diff_success,skipped,same_failed,diff_failed = 0,0,0,0,0,0
+    all_results = []
+    for (i,string_record) in tqdm(enumerate(record_iterator),total=records_count):
+        total+=1
+        example = tf.train.Example()
+        example.ParseFromString(string_record)
+        spec_n1 = example.features.feature['spec_n1'].int64_list.value[0]
+        spec_n2 = example.features.feature['spec_n2'].int64_list.value[0]
+        if n_spec < spec_n1 or n_spec < spec_n2:
+            skipped+=1
+            continue
+        spec_w1 = example.features.feature['spec_w1'].int64_list.value[0]
+        spec_w2 = example.features.feature['spec_w2'].int64_list.value[0]
+        spec1 = np.array(example.features.feature['spec1'].float_list.value).reshape(spec_n1,spec_w1)
+        spec2 = np.array(example.features.feature['spec2'].float_list.value).reshape(spec_n2,spec_w2)
+        word = example.features.feature['word'].bytes_list.value[0].decode()
+        phoneme1 = example.features.feature['phoneme1'].bytes_list.value[0].decode()
+        phoneme2 = example.features.feature['phoneme2'].bytes_list.value[0].decode()
+        voice1 = example.features.feature['voice1'].bytes_list.value[0].decode()
+        voice2 = example.features.feature['voice2'].bytes_list.value[0].decode()
+        language = example.features.feature['language'].bytes_list.value[0].decode()
+        rate1 = example.features.feature['rate1'].int64_list.value[0]
+        rate2 = example.features.feature['rate2'].int64_list.value[0]
+        variant1 = example.features.feature['variant1'].bytes_list.value[0].decode()
+        variant2 = example.features.feature['variant2'].bytes_list.value[0].decode()
+        file1 = example.features.feature['file1'].bytes_list.value[0].decode()
+        file2 = example.features.feature['file2'].bytes_list.value[0].decode()
+
+        p_spec1,p_spec2 = padd_zeros(spec1,n_spec),padd_zeros(spec2,n_spec)
+        input_arr = np.asarray([[p_spec1,p_spec2]])
+        output_arr = np.asarray([example.features.feature['output'].int64_list.value])
+        y_pred = model.predict([input_arr[:, 0], input_arr[:, 1]])
+        predicted = np.asarray(y_pred[0]>0.5).astype(output_arr.dtype)
+        expected = output_arr[0]
+        status = np.all(predicted == expected)
+        result = {"phoneme1":phoneme1,"phoneme2":phoneme2,"voice1":voice1
+                 ,"voice2":voice2,"rate1":rate1,"rate2":rate2
+                 ,"variant1":variant1,"variant2":variant2,"file1":file1
+                 ,"file2":file2,"expected":expected[0],"predicted":y_pred[0][0]
+                 ,"success":status}
+        all_results.append(result)
+        if status:
+            if variant1 == variant2:
+                same_success+=1
+            else:
+                diff_success+=1
+            continue
+        else:
+            if variant1 == variant2:
+                same_failed+=1
+            else:
+                diff_failed+=1
+    print('total-{},same_success-{},diff_success-{},skipped-{},same_failed-{},diff_failed-{}'.format(total,same_success,diff_success,skipped,same_failed,diff_failed))
+    success = same_success+diff_success
+    failure = same_failed+diff_failed
+    print('accuracy-{:.3f}'.format(success*100/(success+failure)))
+    print('same_accuracy-{:.3f}'.format(same_success*100/(same_success+same_failed)))
+    print('diff_accuracy-{:.3f}'.format(diff_success*100/(diff_success+diff_failed)))
+    result_data = pd.DataFrame(all_results,columns=["phoneme1","phoneme2"
+    ,"voice1","voice2","rate1","rate2","variant1","variant2","file1","file2",
+    "expected","predicted","success"])
+    result_data.to_csv('./outputs/' + audio_group + '.results.csv')
+
+def inspect_tfrecord(records_file,audio_group='audio'):
+    record_iterator,records_count = record_generator_count(records_file)
+    all_results = []
+    for (i,string_record) in tqdm(enumerate(record_iterator),total=records_count):
+        # string_record = next(record_iterator)
+        example = tf.train.Example()
+        example.ParseFromString(string_record)
+        spec_n1 = example.features.feature['spec_n1'].int64_list.value[0]
+        spec_n2 = example.features.feature['spec_n2'].int64_list.value[0]
+        word = example.features.feature['word'].bytes_list.value[0].decode()
+        phoneme1 = example.features.feature['phoneme1'].bytes_list.value[0].decode()
+        phoneme2 = example.features.feature['phoneme2'].bytes_list.value[0].decode()
+        voice1 = example.features.feature['voice1'].bytes_list.value[0].decode()
+        voice2 = example.features.feature['voice2'].bytes_list.value[0].decode()
+        language = example.features.feature['language'].bytes_list.value[0].decode()
+        rate1 = example.features.feature['rate1'].int64_list.value[0]
+        rate2 = example.features.feature['rate2'].int64_list.value[0]
+        variant1 = example.features.feature['variant1'].bytes_list.value[0].decode()
+        variant2 = example.features.feature['variant2'].bytes_list.value[0].decode()
+        file1 = example.features.feature['file1'].bytes_list.value[0].decode()
+        file2 = example.features.feature['file2'].bytes_list.value[0].decode()
+        output_arr = np.asarray([example.features.feature['output'].int64_list.value])
+        expected = output_arr[0]
+        result = {"phoneme1":phoneme1,"phoneme2":phoneme2,"voice1":voice1
+                 ,"voice2":voice2,"rate1":rate1,"rate2":rate2,"spec_n1":spec_n1
+                 ,"spec_n2":spec_n2,"variant1":variant1,"variant2":variant2
+                 ,"file1":file1,"file2":file2,"expected":expected[0]}
+        all_results.append(result)
+    result_data = pd.DataFrame(all_results,columns=["phoneme1","phoneme2"
+    ,"voice1","voice2","rate1","rate2","spec_n1","spec_n2","variant1","variant2","file1","file2",
+    "expected"])
+    result_data.to_csv('./outputs/' + audio_group + '.pairs.csv')
+
+def play_results(audio_group='audio'):
+    result_data = pd.read_csv('./outputs/' + audio_group + '.results.csv')
+    play_file,close_player = file_player()
+    quit = False
+    for (i,r) in result_data.iterrows():
+        if quit:
+            break
+        keys = ["phoneme1","phoneme2","voice1","voice2","rate1","rate2","variant1","variant2"]
+        row_vals = [str(r[k]) for k in keys]
+        h_str = '\t'.join(keys)
+        row_str = '\t'.join(row_vals)
+        while True:
+            print(h_str)
+            print(row_str)
+            play_file('./outputs/'+audio_group+'/'+r['file1'],True)
+            play_file('./outputs/'+audio_group+'/'+r['file2'],True)
+            a = input("press 'r/q/[Enter]' to replay/quit/continue:\t")
+            if a == 'r':
+                continue
+            if a == 'q':
+                quit = True
+                break
+            else:
+                break
+    close_player()
+
+def visualize_results(audio_group='audio'):
+    # %matplotlib inline
+    audio_group = 'story_phrases'
+    source = pd.read_csv('./outputs/'+audio_group+'.pairs.csv',index_col=0)
+    source.groupby(['voice1','voice2']).size()
+    result = pd.read_csv('./outputs/' + audio_group + '.results.csv',index_col=0)
+    # result.groupby('success').size().plot(kind='bar')
+    result.describe(include=['object'])
+    failed = result[result['success'] == False]
+    same_failed = failed[failed['variant1'] == failed['variant2']]
+    diff_failed = failed[failed['variant1'] != failed['variant2']]
+    result.groupby(['voice1','voice2']).size()
+
+
+if __name__ == '__main__':
+    # evaluate_siamese('./outputs/story_words_test.train.tfrecords',audio_group='story_words.gpu',weights ='siamese_speech_model-58-epoch-0.00-acc.h5')
+    # evaluate_siamese('./outputs/story_words.test.tfrecords',audio_group='story_words',weights ='siamese_speech_model-675-epoch-0.00-acc.h5')
+    evaluate_siamese('./outputs/story_words.test.tfrecords',audio_group='story_words',weights ='siamese_speech_model-153-epoch-0.55-acc.h5')
+    # play_results('story_words')
+    #inspect_tfrecord('./outputs/story_phrases.test.tfrecords',audio_group='story_phrases')
+    # visualize_results('story_words.gpu')
+# test_with('rand_edu')
+# sunflower_data,sunflower_result = get_word_pairs_data('sweater',15)
+# print(np.argmax(model.predict([sunflower_data[:, 0], sunflower_data[:, 1]]),axis=1))
+# print(sunflower_result)

speech_testgen.py

@@ -0,0 +1,50 @@
+import voicerss_tts
+import json
+from speech_tools import format_filename
+
+def generate_voice(phrase):
+    voice = voicerss_tts.speech({
+        'key': '0ae89d82aa78460691c99a4ac8c0f9ec',
+        'hl': 'en-us',
+        'src': phrase,
+        'r': '0',
+        'c': 'mp3',
+        'f': '22khz_16bit_mono',
+        'ssml': 'false',
+        'b64': 'false'
+    })
+    if not voice['error']:
+        return voice[b'response']
+    return None
+
+
+def generate_test_audio_for_stories():
+    story_file = './inputs/all_stories_hs.json'
+    # story_file = './inputs/all_stories.json'
+    stories_data = json.load(open(story_file))
+    text_list_dup = [t[0] for i in stories_data.values() for t in i]
+    text_list = sorted(list(set(text_list_dup)))[:10]
+    for t in text_list:
+        v = generate_voice(t)
+        if v:
+            f_name = format_filename(t)
+            tf = open('inputs/voicerss/'+f_name+'.mp3','wb')
+            tf.write(v)
+            tf.close()
+
+# def generate_test_audio_for(records_file,audio_group='audio'):
+#     # audio_group='audio';model_file = 'siamese_speech_model-305-epoch-0.20-acc.h5'
+#     # records_file  = os.path.join('./outputs',eval_group+'.train.tfrecords')
+#     const_file = os.path.join('./models/'+audio_group+'/','constants.pkl')
+#     (n_spec,n_features,n_records) = pickle.load(open(const_file,'rb'))
+#     print('evaluating {}...'.format(records_file))
+#     record_iterator,records_count = record_generator_count(records_file)
+#     all_results = []
+#     for (i,string_record) in tqdm(enumerate(record_iterator),total=records_count):
+#         total+=1
+#         example = tf.train.Example()
+#         example.ParseFromString(string_record)
+#         word = example.features.feature['word'].bytes_list.value[0].decode()
+
+# audio = generate_voice('hello world')
+# audio

speech_tools.py

@@ -0,0 +1,204 @@
+import os
+import math
+import string
+import threading
+import itertools
+import random
+import multiprocessing
+import subprocess
+import pandas as pd
+import numpy as np
+import pyaudio
+from pysndfile import sndio as snd
+# from matplotlib import pyplot as plt
+from speech_spectrum import plot_stft, generate_spec_frec,generate_aiff_spectrogram
+
+SAMPLE_RATE = 22050
+N_CHANNELS = 2
+
+devnull = open(os.devnull, 'w')
+
+def step_count(n_records,batch_size):
+    return int(math.ceil(n_records*1.0/batch_size))
+
+def file_player():
+    p_oup = pyaudio.PyAudio()
+    def play_file(audiopath,plot=False):
+        print('playing',audiopath)
+        samples, samplerate, form = snd.read(audiopath)
+        stream = p_oup.open(
+            format=pyaudio.paFloat32,
+            channels=2,
+            rate=samplerate,
+            output=True)
+        one_channel = np.asarray([samples, samples]).T.reshape(-1)
+        audio_data = one_channel.astype(np.float32).tobytes()
+        stream.write(audio_data)
+        stream.close()
+        if plot:
+            plot_stft(samples, SAMPLE_RATE)
+    def close_player():
+        p_oup.terminate()
+    return play_file,close_player
+
+def reservoir_sample(iterable, k):
+    it = iter(iterable)
+    if not (k > 0):
+        raise ValueError("sample size must be positive")
+
+    sample = list(itertools.islice(it, k)) # fill the reservoir
+    random.shuffle(sample) # if number of items less then *k* then
+                           #   return all items in random order.
+    for i, item in enumerate(it, start=k+1):
+        j = random.randrange(i) # random [0..i)
+        if j < k:
+            sample[j] = item # replace item with gradually decreasing probability
+    return sample
+
+def padd_zeros(spgr, max_samples):
+    return np.lib.pad(spgr, [(0, max_samples - spgr.shape[0]), (0, 0)],
+                      'constant')
+
+def read_seg_file(aiff_name):
+    base_name = aiff_name.rsplit('.aiff',1)[0]
+    seg_file = base_name+'-palign.csv'
+    seg_data = pd.read_csv(seg_file,names=['action','start','end','phoneme'])
+    seg_data = seg_data[(seg_data['action'] == 'PhonAlign') & (seg_data['phoneme'] != '#')]
+    return seg_data
+
+def record_spectrogram(n_sec, plot=False, playback=False):
+    # show_record_prompt()
+    N_SEC = n_sec
+    CHUNKSIZE = int(SAMPLE_RATE * N_SEC / N_CHANNELS)  # fixed chunk size
+    input('Press [Enter] to start recording sample... ')
+    p_inp = pyaudio.PyAudio()
+    stream = p_inp.open(
+        format=pyaudio.paFloat32,
+        channels=N_CHANNELS,
+        rate=SAMPLE_RATE,
+        input=True,
+        frames_per_buffer=CHUNKSIZE)
+    data = stream.read(CHUNKSIZE)
+    numpydata = np.frombuffer(data, dtype=np.float32)
+    multi_channel = np.abs(np.reshape(numpydata, (-1, 2))).mean(axis=1)
+    one_channel = np.asarray([multi_channel, -1 * multi_channel]).T.reshape(-1)
+    mean_channel_data = one_channel.tobytes()
+    stream.stop_stream()
+    stream.close()
+    p_inp.terminate()
+    if plot:
+        plot_stft(one_channel, SAMPLE_RATE)
+    if playback:
+        p_oup = pyaudio.PyAudio()
+        stream = p_oup.open(
+            format=pyaudio.paFloat32,
+            channels=2,
+            rate=SAMPLE_RATE,
+            output=True)
+        stream.write(mean_channel_data)
+        stream.close()
+        p_oup.terminate()
+    ims, _ = generate_spec_frec(one_channel, SAMPLE_RATE)
+    return ims
+
+def pair_for_word(phrase='able'):
+    spec1 = generate_aiff_spectrogram('./inputs/pairs/good/'+phrase+'.aiff')
+    spec2 = generate_aiff_spectrogram('./inputs/pairs/test/'+phrase+'.aiff')
+    return spec1,spec2
+
+def transribe_audio_text(aiff_name,phrase):
+    base_name = aiff_name.rsplit('.aiff',1)[0]
+    wav_name = base_name+'.wav'
+    txt_name = base_name+'.txt'
+    params = ['ffmpeg', '-y', '-i',aiff_name,wav_name]
+    subprocess.call(params,stdout=devnull,stderr=devnull)
+    trcr_f = open(txt_name,'w')
+    trcr_f.write(phrase)
+    trcr_f.close()
+
+def _apply_df(args):
+    df, func, num, kwargs = args
+    return num, df.apply(func, **kwargs)
+
+def apply_by_multiprocessing(df,func,**kwargs):
+  cores = multiprocessing.cpu_count()
+  workers=kwargs.pop('workers') if 'workers' in kwargs else cores
+  pool = multiprocessing.Pool(processes=workers)
+  result = pool.map(_apply_df, [(d, func, i, kwargs) for i,d in enumerate(np.array_split(df, workers))])
+  pool.close()
+  result=sorted(result,key=lambda x:x[0])
+  return pd.concat([i[1] for i in result])
+
+def square(x):
+    return x**x
+
+# if __name__ == '__main__':
+#     df = pd.DataFrame({'a':range(10), 'b':range(10)})
+#     apply_by_multiprocessing(df, square, axis=1, workers=4)
+
+def hms_string(sec_elapsed):
+    h = int(sec_elapsed / (60 * 60))
+    m = int((sec_elapsed % (60 * 60)) / 60)
+    s = sec_elapsed % 60.
+    return "{}:{:>02}:{:>05.2f}".format(h, m, s)
+
+def rm_rf(d):
+    for path in (os.path.join(d,f) for f in os.listdir(d)):
+        if os.path.isdir(path):
+            rm_rf(path)
+        else:
+            os.unlink(path)
+    os.rmdir(d)
+
+def create_dir(direc):
+    if not os.path.exists(direc):
+        os.makedirs(direc)
+    else:
+        rm_rf(direc)
+        create_dir(direc)
+
+
+def format_filename(s):
+    """
+    Take a string and return a valid filename constructed from the string.
+    Uses a whitelist approach: any characters not present in valid_chars are
+    removed. Also spaces are replaced with underscores.
+
+    Note: this method may produce invalid filenames such as ``, `.` or `..`
+    When I use this method I prepend a date string like '2009_01_15_19_46_32_'
+    and append a file extension like '.txt', so I avoid the potential of using
+    an invalid filename.
+    """
+    valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
+    filename = ''.join(c for c in s if c in valid_chars)
+    filename = filename.replace(' ','_') # I don't like spaces in filenames.
+    return filename
+
+#################### Now make the data generator threadsafe ####################
+
+class threadsafe_iter:
+    """Takes an iterator/generator and makes it thread-safe by
+    serializing call to the `next` method of given iterator/generator.
+    """
+    def __init__(self, it):
+        self.it = it
+        self.lock = threading.Lock()
+
+    def __iter__(self):
+        return self
+
+    def __next__(self): # Py3
+        with self.lock:
+            return next(self.it)
+
+    def next(self):     # Py2
+        with self.lock:
+            return self.it.next()
+
+
+def threadsafe_generator(f):
+    """A decorator that takes a generator function and makes it thread-safe.
+    """
+    def g(*a, **kw):
+        return threadsafe_iter(f(*a, **kw))
+    return g

test_siamese.py

@@ -1,30 +0,0 @@
-from speech_siamese import siamese_model
-from record_mic_speech import record_spectrogram
-from importlib import reload
-# import speech_data
-# reload(speech_data)
-from speech_data import create_test_pair,get_word_pairs_data,speech_data
-import numpy as np
-
-model = siamese_model((15, 1654))
-model.load_weights('./models/siamese_speech_model-final.h5')
-
-def predict_recording_with(m,sample_size=15):
-    spec1 = record_spectrogram(n_sec=1.4)
-    spec2 = record_spectrogram(n_sec=1.4)
-    inp = create_test_pair(spec1,spec2,sample_size)
-    return m.predict([inp[:, 0], inp[:, 1]])
-
-# while(True):
-#     print(predict_recording_with(model))
-
-
-def test_with(audio_group):
-    X,Y = speech_data(audio_group)
-    print(np.argmax(model.predict([X[:, 0], X[:, 1]]),axis=1))
-    print(Y.astype(np.int8))
-
-test_with('rand_edu')
-# sunflower_data,sunflower_result = get_word_pairs_data('sweater',15)
-# print(np.argmax(model.predict([sunflower_data[:, 0], sunflower_data[:, 1]]),axis=1))
-# print(sunflower_result)

voicerss_tts.py

@@ -0,0 +1,52 @@
+import http.client, urllib.request, urllib.parse, urllib.error
+
+def speech(settings):
+	__validate(settings)
+	return __request(settings)
+
+def __validate(settings):
+	if not settings: raise RuntimeError('The settings are undefined')
+	if 'key' not in settings or not settings['key']: raise RuntimeError('The API key is undefined')
+	if 'src' not in settings or not settings['src']: raise RuntimeError('The text is undefined')
+	if 'hl' not in settings or not settings['hl']: raise RuntimeError('The language is undefined')
+
+def __request(settings):
+	result = {'error': None, 'response': None}
+
+	headers = {'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'}
+	params = urllib.parse.urlencode(__buildRequest(settings))
+
+	if 'ssl' in settings and settings['ssl']:
+		conn = http.client.HTTPSConnection('api.voicerss.org:443')
+	else:
+		conn = http.client.HTTPConnection('api.voicerss.org:80')
+
+	conn.request('POST', '/', params, headers)
+
+	response = conn.getresponse()
+	content = response.read()
+
+	if response.status != 200:
+		result[b'error'] = response.reason
+	elif content.find(b'ERROR') == 0:
+		result[b'error'] = content
+	else:
+		result[b'response'] = content
+
+	conn.close()
+
+	return result
+
+def __buildRequest(settings):
+	params = {'key': '', 'src': '', 'hl': '', 'r': '', 'c': '', 'f': '', 'ssml': '', 'b64': ''}
+
+	if 'key' in settings: params['key'] = settings['key']
+	if 'src' in settings: params['src'] = settings['src']
+	if 'hl' in settings: params['hl'] = settings['hl']
+	if 'r' in settings: params['r'] = settings['r']
+	if 'c' in settings: params['c'] = settings['c']
+	if 'f' in settings: params['f'] = settings['f']
+	if 'ssml' in settings: params['ssml'] = settings['ssml']
+	if 'b64' in settings: params['b64'] = settings['b64']
+
+	return params

voicerss_tts.py.bak

@@ -0,0 +1,52 @@
+import httplib, urllib
+
+def speech(settings):
+	__validate(settings)
+	return __request(settings)
+
+def __validate(settings):
+	if not settings: raise RuntimeError('The settings are undefined')
+	if 'key' not in settings or not settings['key']: raise RuntimeError('The API key is undefined')
+	if 'src' not in settings or not settings['src']: raise RuntimeError('The text is undefined')
+	if 'hl' not in settings or not settings['hl']: raise RuntimeError('The language is undefined')
+
+def __request(settings):
+	result = {'error': None, 'response': None}
+
+	headers = {'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'}
+	params = urllib.urlencode(__buildRequest(settings))
+	
+	if 'ssl' in settings and settings['ssl']:
+		conn = httplib.HTTPSConnection('api.voicerss.org:443')
+	else:
+		conn = httplib.HTTPConnection('api.voicerss.org:80')
+		
+	conn.request('POST', '/', params, headers)
+	
+	response = conn.getresponse()
+	content = response.read()
+	
+	if response.status != 200:
+		result['error'] = response.reason
+	elif content.find('ERROR') == 0:
+		result['error'] = content
+	else:
+		result['response'] = content
+		
+	conn.close()
+
+	return result
+
+def __buildRequest(settings):
+	params = {'key': '', 'src': '', 'hl': '', 'r': '', 'c': '', 'f': '', 'ssml': '', 'b64': ''}
+	
+	if 'key' in settings: params['key'] = settings['key']
+	if 'src' in settings: params['src'] = settings['src']
+	if 'hl' in settings: params['hl'] = settings['hl']
+	if 'r' in settings: params['r'] = settings['r']
+	if 'c' in settings: params['c'] = settings['c']
+	if 'f' in settings: params['f'] = settings['f']
+	if 'ssml' in settings: params['ssml'] = settings['ssml']
+	if 'b64' in settings: params['b64'] = settings['b64']
+	
+	return params