tacotron2/final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "#%matplotlib inline\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "import IPython.display as ipd\n",
    "\n",
    "import sys\n",
    "sys.path.append('waveglow/')\n",
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "from hparams import create_hparams\n",
    "from model import Tacotron2\n",
    "from layers import TacotronSTFT, STFT\n",
    "from audio_processing import griffin_lim\n",
    "from train import load_model\n",
    "from text import text_to_sequence\n",
    "from denoiser import Denoiser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hparams = create_hparams()\n",
    "hparams.sampling_rate = 22050\n",
    "checkpoint_path = \"checkpoint_15000\"\n",
    "model = load_model(hparams)\n",
    "model.load_state_dict(torch.load(checkpoint_path, map_location = 'cpu')['state_dict'])  #added map_location = 'cpu'\n",
    "_ = model.eval()          #it was originally model.cuda().eval().half()\n",
    "waveglow_path = 'waveglow_256channels.pt'\n",
    "waveglow = torch.load(waveglow_path, map_location = 'cpu')['model']   #added map_location = 'cpu'\n",
    "waveglow.eval()   #originally waveglow.cuda().eval().half()\n",
    "for k in waveglow.convinv:\n",
    "    k.float()\n",
    "#denoiser = Denoiser(waveglow)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import soundfile as sf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert(array):\n",
    "    sf.write('sample.wav', array, 22050)\n",
    "    os.system('ffmpeg -i {0} -filter:a \"atempo=0.80\" {1}'.format('sample.wav', 'sample0.wav'))\n",
    "    #os.system('ffmpeg -i {0} -ar 8000 {1}'.format('sample0.wav', 'sample1.wav'))\n",
    "    data, rate = sf.read('sample0.wav')\n",
    "    os.remove('sample.wav')\n",
    "    os.remove('sample0.wav')\n",
    "    #os.remove('sample1.wav')\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def speech(t):\n",
    "    start = time.time()\n",
    "    text = t\n",
    "    sequence = np.array(text_to_sequence(text, ['english_cleaners']))[None, :]\n",
    "    print(sequence)\n",
    "    sequence = torch.autograd.Variable(\n",
    "        torch.from_numpy(sequence)).long()   #originally torch.from_numpy(sequence)).cuda().long()\n",
    "    mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence)\n",
    "    with torch.no_grad():\n",
    "        audio = waveglow.infer(mel_outputs_postnet, sigma=0.666)\n",
    "    #audio_denoised = denoiser(audio, strength=0.01)[:, 0]\n",
    "    data = convert(audio[0].data.cpu().numpy())\n",
    "    #os.system('ffmpeg -i {0} -filter:a \"atempo=0.85\" {1}'.format('harvard_inference/audio/'+str(i)+'.wav', 'harvard_inference/audio_0.85/'+str(i)+'.wav'))\n",
    "    aud = ipd.Audio(data, rate=22050)\n",
    "    end = time.time()\n",
    "    print(end-start)\n",
    "    return aud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "speech('I understand your frustration and disappointment. I am sorry that its happening and I would like to help prevent it in the future. What style of diapers did you buy? For instance, was it the snugglers, pull ups or baby dry.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from final import display,speech,play_device\n",
    "import pyaudio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15.046638011932373\n"
     ]
    }
   ],
   "source": [
    "data = speech('Thank you for calling Huggies. How may I help you today .')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "                <audio  controls=\"controls\" >\n",
       "                    <source src=\"data:audio/wav;base64,UklGRjIKAQBXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YQ4KAQDq/wAAQwCeAMAAfAALAGUA4QD4AKkAIQDT/+r/9f8AAAAA3//1/zgAZQA4APX/6v8LADgAFgDT/8j/9f9DAHwALQDf/6b/0/8WABYAAADI/73/yP/T/9P/sf/T/+r/CwA4ADgALQAtACEACwALABYAIQBDABYAAADq/9P/AAAAAAAAAAAWABYACwALAHAA7QCpACEAyP+x//X/AAALAPX/3//1/yEAFgCx/2L/sf9DAGUAFgDf/zgA7QDWAC0AyP8WAKkAywCHAJ4AywDLAHwAFgAhACEA9f/I/9//AAALAPX/yP+9/73/3/8LAEMAWgBPADgACwAAAAAAAAAWAC0AcACpAFoAyP+b/yEA1gDWABYAhP/I/3AA+ACpABYAFgBaAJ4AZQAAAAAAQwCSAGUAOAAhADgAWgBPAHAAwACSAAsAm/+9/xYAIQCx/0z/sf8hACEAyP+m/zgA7QDtADgAsf/f/0MATwDf/4T/m//f/wsA6v+x/5D/hP+b/8j/6v8LAFoAngDhAPgAqQBwAEMAIQBaALQA4QCSAAsA9f8hAIcATwAWABYALQAtABYAAAALAAsA9f8hAGUAwAD4ANYAtACSAGUAhwDAAMAAwADAAJ4AhwCHAFoALQALAAsAOAAtAPX/6v8hAGUATwDf/6b/6v9lAHwACwDI/wsAhwDLAKkAZQA4AC0AQwA4ACEA6v/I//X/QwB8AIcAOAALAC0ALQAWAN//pv/I/xYATwBPAEMALQAhAEMAqQAlAQ8BTwDT/9//IQBlAE8AQwAhAOr/Yv+Q/xYATwAtAKb/QP9M/7H/6v8hAHAAcAAtAPX/CwBwAIcAQwAtAC0AWgBwAHAAkgDAAJ4AOAAAAAAATwBaACEAOACpAMAATwDq/9//AAD1/73/sf/1/xYALQAhAPX/9f/1/9//vf+Q/9P/QwBaACEAsf9A/0D/kP/q/zgALQDq/9P/6v8LACEAIQAtAKkA1gCpAGUAFgAAANP/hP+Q/xYAwADtAGUAm/95/y0AAwExAYcA0/+9/y0AkgB8AEMAWgCeAJIAIQAAADgAkgCSAAAApv+x/9//3/+9/8j/vf+m/1f/Nf+b/y0AhwCHAEMAOABPAHAAZQAtAAAAAAAWACEALQDT/1f/V/+m/y0AZQAAAL3/sf/I/7H/bv95/+r/OAA4AEMAWgB8AHwAZQBlAHwAcABwAKkA7QDhAKkAkgDWAPgA7QDAALQAcAAWAN//3/9PALQAkgBPAGUAywDtAJ4AQwA4AGUACwCE/0z/0/+0ACUBqQDq/2L/bv/T/wAACwBDAGUAfACHAHAAwAA8AWkBJQGpADgATwCHAE8AAAAWAHwAywBaAJD/TP/f/54A1gBaAAAAIQCSALQAkgBlAFoAZQALAOr/FgCHAJIALQCb/4T/6v8AAAAA6v9PAMAAhwAhACEAtAAaAcsAQwBDAMsADwGeAAAApv/q/4cA+ADWAJIAIQD1/+r/9f9DAJIAfABPACEACwAWAAsAFgA4AC0A0/+E/9//cADtAOEATwD1/9P/3/8hAFoAWgBaAIcA1gD4AIcAAADI/7H/ef8q/+b+Nf/f/wAAhP9X/wAA1gDAAOr/ef/f/4cA1gCHAHwA1gAPAbQATwBPAIcADwE8ATwB4QAhAJv/kP/1/zgACwDI/7H/3//1/wAAAAAAAAAAFgBDAGUAZQBwAKkAGgGAAYsBAwFaAAsAOACSAJ4ACwBi/yr/kP8AAAsAbv8I/3n/WgDhAIcA9f/T/wAAvf8q/+b+Yv9lABoBAwFlAIT/jP5I/tv+9f+eAEMAYv8q/6b/FgDT/x//E//f/4cATwCm/5D/QwDhANYAkgCSAJIAfABPAC0AQwBPADgAOAAhABYAQwCHAMsAhwALABYADwHDATEB6v9A/2UASwI4A3gC1gA1/4D+xP7f/zEB/AGWAU8A/f6A/kD/hwBpARoB3//E/oz+V/98AA8B1gBwADgAIQDq/5v/V/+b/1oAUgGtAcsAQP+i/sj/aQGAAQAA8f6x/wMBPAFPAAsA4QCtASUBOABDANYAAwF8AAsATwDAAIcAFgAAAMsAKQKPAq0BOAC9/08A4QBwAL3/0/+eABoBkgCm/73/qQBHAakAbv/E/h//6v8hAAAA9f/T/73/3/8AAAAAvf8q//H+V/9aAFIBogFSASUBaQGAATEBngBlAHwAOABPADwBBwLxAbQAvf8WAAMB4QC9/zX/sf9aABYAV/9u/y0AhwAAAB//z/5X/zgAGgGLAfgALQAAAIcA+ABwAJD/Yv9PAPgAywAWAL3/FgDWAEcBGgGHAOr/hP+E/9//fADhAMAAqQDhAIsB5QExARYAYv+Q/wsAQwBlALQAGgHhAE8AkP8I//3+Yv8AAEMAvf8q/27/TwDAAC0AYv9M/73/vf+E/zgAogFLAg8BE/91/mL/FgC9/4T/cAAHAngCdAGpAO0AJQELAJf+2/4xAXED3gL1/3H99fyi/iUBbQIlAQT+NfwP/m0CuAQTAgv9cfqE/Kb/Yv/q/Bf9ywDpApv/gPsm/kQGFwzqCAMBRP2b/3gC/AEtANP/ZQBM/yL9k/1HAS0D3/9e+637ywDaBIcDsf+u/l4BUwTtAzwBQP/E/s/+ov6u/i0AbQJxA1YCCwDM/SL9z/6LAQsDuAFA/xv+uf4tAF4B/AEHAkcB9f/b/vH+yP+m/xP/m/9SAVYCGgH1/9YAYgLPAbH/QP9SAakA1/ph9u36KQVxCTgDOPpA+SEApgXiA5v/3/+iBMQHhAUaARYADwQMCRcJfAML/XX7Nf94AgMBRP3E+zn9l/4t/Rr7l/vE/osBSwKeAIz+D/5PAAQERAbLAw/+BPuM/tMF1wmuB94Cuf5i/ED8QP8EBKIHAAZM/wf5gPi1/SIDRwQPARv+W/0m/kT9U/va+54A6gUeBar9ZfdE+nUEUAzQCh4CBPvX+qL+TwAhAF4BUwQTBZIA4vrt+rQAfAZTByIDH/9e/gMBXgQiBvUFFwaXB2IIfAaaAnwAOAPbBy4JxAQx/iL61/px/RP/SP4x+xr4nvcH+bz5T/rx+579ufva9Q7yHvbb/iIDCP/l+Lz5PAHXBt4F5QGm/wMBHgIaAQ8ByAXbDS8SIwz9/tr4sQI+Fm0ifxvEB+L6V/92DZEUFA5mA0MAhAW1CXUEbfmw8235FgP5BksCB/nK8azvDvLO9aL4j/mP+QP4Q/Q/8LfvTvST+qL7w/UK7jPqfuzO8uL6cACm/2X3Cu6+6z/zFgP7GBAtmDBdG0j+7frdHJZELUnPJG38pfO1CaEhhyMnENr4O+wN7KjxHvat+FP7xPuS9GvnlOAV6wcChRGMCmn1PuqH9EQJKxRUDc/+P/bL92r+OAPTAmr+QPlL9s71kvTW8SjwCvGG8RnvWeuG68LvO/IZ75nt6fbqC8cfhCU0H2wcMSQuL4wwPCchHdUamh+wIgMe4xLtBu39A/gS85XvMO/S8IPwfuzg5TbiI+ZK8Hj5Kfzx+Oz0bfbI/0EL8hBBDl4HXgQ5BrkHaQQLAL3/3gJiAgT7cPEo7R3wzvU49zfxgufM4MXhZ+bJ6MbqmvZqDc4e4Bem/0T61iCnVwNkkDHK8SfnRhjmTXlRLSaA+KTqpfapA9MCqfol9Snz4OsE3rzT09zt94wNSwVj5eXP5t4fCKkmiyELAw3sjvAQCs4e6BztBjDyrO/e/IgGcAAK8bfpXe9H+Ev2pOpm4K7hp+hU53ncV9+WASoulDITArvQsuLRNv9/QXFkGhbUq+YRNkRsTVRcD5/gvuh9CYIZxQ1S+JXpBeSB4fngUeaZ8NL2Q/HB4w/b9uKa+dQO6xGeA8fzofKWAc0Syhd2EDUFRP18+pr8cADPAcD9R/VV7ffrBvCw8+zucuMe3GbgKOrw9a4HdxaJD7TxWOXFEKtbEXllQBjmb8uyDlpps31VOVDghcWl9iM1gz+fD3HaGc++648
       "                    Your browser does not support the audio element.\n",
       "                </audio>\n",
       "              "
      ],
      "text/plain": [
       "<IPython.lib.display.Audio object>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "display(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play_device(data):\n",
    "    audio_interface = pyaudio.PyAudio()\n",
    "    _audio_stream = audio_interface.open(format=pyaudio.paInt16,channels=1, rate=16000,output=True)\n",
    "    _audio_stream.write(data.tostring())\n",
    "#     _audio_stream.close()\n",
    "play_device(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}