{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ubuntu/tacotron2/plotting_utils.py:2: UserWarning: \n", "This call to matplotlib.use() has no effect because the backend has already\n", "been chosen; matplotlib.use() must be called *before* pylab, matplotlib.pyplot,\n", "or matplotlib.backends is imported for the first time.\n", "\n", "The backend was *originally* set to 'module://ipykernel.pylab.backend_inline' by the following code:\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n", " \"__main__\", mod_spec)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/runpy.py\", line 85, in _run_code\n", " exec(code, run_globals)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/ipykernel/__main__.py\", line 3, in \n", " app.launch_new_instance()\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n", " app.start()\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 505, in start\n", " self.io_loop.start()\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/tornado/platform/asyncio.py\", line 148, in start\n", " self.asyncio_loop.run_forever()\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/asyncio/base_events.py\", line 438, in run_forever\n", " self._run_once()\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/asyncio/base_events.py\", line 1451, in _run_once\n", " handle._run()\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/asyncio/events.py\", line 145, in _run\n", " self._callback(*self._args)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/tornado/ioloop.py\", line 690, in \n", " lambda f: self._run_callback(functools.partial(callback, future))\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n", " ret = callback()\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/tornado/gen.py\", line 781, in inner\n", " self.run()\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/tornado/gen.py\", line 742, in run\n", " yielded = self.gen.send(value)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 357, in process_one\n", " yield gen.maybe_future(dispatch(*args))\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n", " yielded = next(result)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 267, in dispatch_shell\n", " yield gen.maybe_future(handler(stream, idents, msg))\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n", " yielded = next(result)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 534, in execute_request\n", " user_expressions, allow_stdin,\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n", " yielded = next(result)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 294, in do_execute\n", " res = shell.run_cell(code, store_history=store_history, silent=silent)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 536, in run_cell\n", " return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2848, in run_cell\n", " raw_cell, store_history, silent, shell_futures)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2874, in _run_cell\n", " return runner(coro)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/IPython/core/async_helpers.py\", line 67, in _pseudo_sync_runner\n", " coro.send(None)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3049, in run_cell_async\n", " interactivity=interactivity, compiler=compiler, result=result)\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3214, in run_ast_nodes\n", " if (yield from self.run_code(code, result)):\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3296, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 3, in \n", " import matplotlib.pylab as plt\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/matplotlib/pylab.py\", line 257, in \n", " from matplotlib import cbook, mlab, pyplot as plt\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/matplotlib/pyplot.py\", line 69, in \n", " from matplotlib.backends import pylab_setup\n", " File \"/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/matplotlib/backends/__init__.py\", line 14, in \n", " line for line in traceback.format_stack()\n", "\n", "\n", " matplotlib.use(\"Agg\")\n" ] } ], "source": [ "import matplotlib\n", "#%matplotlib inline\n", "import matplotlib.pylab as plt\n", "\n", "import IPython.display as ipd\n", "\n", "import sys\n", "sys.path.append('waveglow/')\n", "import numpy as np\n", "import torch\n", "\n", "from hparams import create_hparams\n", "from model import Tacotron2\n", "from layers import TacotronSTFT, STFT\n", "from audio_processing import griffin_lim\n", "from train import load_model\n", "from text import text_to_sequence\n", "from denoiser import Denoiser" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import os" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", "For more information, please see:\n", " * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n", " * https://github.com/tensorflow/addons\n", "If you depend on functionality not listed there, please file an issue.\n", "\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/torch/serialization.py:454: SourceChangeWarning: source code of class 'glow.WaveGlow' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n", " warnings.warn(msg, SourceChangeWarning)\n", "/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/torch/serialization.py:454: SourceChangeWarning: source code of class 'torch.nn.modules.conv.ConvTranspose1d' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n", " warnings.warn(msg, SourceChangeWarning)\n", "/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/torch/serialization.py:454: SourceChangeWarning: source code of class 'glow.WN' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n", " warnings.warn(msg, SourceChangeWarning)\n", "/home/ubuntu/anaconda3/envs/tts/lib/python3.6/site-packages/torch/serialization.py:454: SourceChangeWarning: source code of class 'torch.nn.modules.conv.Conv1d' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n", " warnings.warn(msg, SourceChangeWarning)\n" ] } ], "source": [ "hparams = create_hparams()\n", "hparams.sampling_rate = 22050\n", "checkpoint_path = \"checkpoint_15000\"\n", "model = load_model(hparams)\n", "model.load_state_dict(torch.load(checkpoint_path, map_location = 'cpu')['state_dict']) #added map_location = 'cpu'\n", "_ = model.eval() #it was originally model.cuda().eval().half()\n", "waveglow_path = 'waveglow_256channels.pt'\n", "waveglow = torch.load(waveglow_path, map_location = 'cpu')['model'] #added map_location = 'cpu'\n", "waveglow.eval() #originally waveglow.cuda().eval().half()\n", "for k in waveglow.convinv:\n", " k.float()\n", "#denoiser = Denoiser(waveglow)\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "import soundfile as sf" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import time" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def convert(array):\n", " sf.write('sample.wav', array, 22050)\n", " os.system('ffmpeg -i {0} -filter:a \"atempo=0.80\" {1}'.format('sample.wav', 'sample0.wav'))\n", " #os.system('ffmpeg -i {0} -ar 8000 {1}'.format('sample0.wav', 'sample1.wav'))\n", " data, rate = sf.read('sample0.wav')\n", " os.remove('sample.wav')\n", " os.remove('sample0.wav')\n", " #os.remove('sample1.wav')\n", " return data" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "def speech(t):\n", " start = time.time()\n", " text = t\n", " sequence = np.array(text_to_sequence(text, ['english_cleaners']))[None, :]\n", " print(sequence)\n", " sequence = torch.autograd.Variable(\n", " torch.from_numpy(sequence)).long() #originally torch.from_numpy(sequence)).cuda().long()\n", " mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence)\n", " with torch.no_grad():\n", " audio = waveglow.infer(mel_outputs_postnet, sigma=0.666)\n", " #audio_denoised = denoiser(audio, strength=0.01)[:, 0]\n", " data = convert(audio[0].data.cpu().numpy())\n", " #os.system('ffmpeg -i {0} -filter:a \"atempo=0.85\" {1}'.format('harvard_inference/audio/'+str(i)+'.wav', 'harvard_inference/audio_0.85/'+str(i)+'.wav'))\n", " aud = ipd.Audio(data, rate=22050)\n", " end = time.time()\n", " print(end-start)\n", " return aud" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[[46 11 58 51 41 42 55 56 57 38 51 41 11 62 52 58 55 11 43 55 58 56 57 55\n", " 38 57 46 52 51 11 38 51 41 11 41 46 56 38 53 53 52 46 51 57 50 42 51 57\n", " 7 11 46 11 38 50 11 56 52 55 55 62 11 57 45 38 57 11 46 57 56 11 45 38\n", " 53 53 42 51 46 51 44 11 38 51 41 11 46 11 60 52 58 49 41 11 49 46 48 42\n", " 11 57 52 11 45 42 49 53 11 53 55 42 59 42 51 57 11 46 57 11 46 51 11 57\n", " 45 42 11 43 58 57 58 55 42 7 11 60 45 38 57 11 56 57 62 49 42 11 52 43\n", " 11 41 46 38 53 42 55 56 11 41 46 41 11 62 52 58 11 39 58 62 10 11 43 52\n", " 55 11 46 51 56 57 38 51 40 42 6 11 60 38 56 11 46 57 11 57 45 42 11 56\n", " 51 58 44 44 49 42 55 56 6 11 53 58 49 49 11 58 53 56 11 52 55 11 39 38\n", " 39 62 11 41 55 62 7]]\n", "21.24711561203003\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "speech('I understand your frustration and disappointment. I am sorry that its happening and I would like to help prevent it in the future. What style of diapers did you buy? For instance, was it the snugglers, pull ups or baby dry.')" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'sequence' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msequence\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;31mNameError\u001b[0m: name 'sequence' is not defined" ] } ], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 2 }