Sound Effects

In the project VoiceAndAudio you created earlier in this chapter, follow these steps to add voice commands to both the service that launches the app and the activity that launches the Glass sound effect test:

1. In the res/menu/main.xml file, add a new menu item.

   <item
           android:id="@+id/soundeffect"
           android:title="@string/soundeffect"/>

2. In res/menu, add a new XML file called soundeffect.xml.

   <?xml version="1.0" encoding="utf-8"?>
   <menu xmlns:android="http://schemas.android.com/apk/res/android " >
    
       <item
           android:id="@+id/disallowed"
           android:title="@string/disallowed"/>
       <item
           android:id="@+id/dismissed"
           android:title="@string/dismissed"/>
       <item
           android:id="@+id/error"
           android:title="@string/error"/>
       <item
           android:id="@+id/selected"
           android:title="@string/selected"/>
       <item
           android:id="@+id/success"
           android:title="@string/success"/>
       <item
           android:id="@+id/tap"
           android:title="@string/tap"/> 
   </menu>

3. Add a new XML file called voice_trigger_start2.xml in res/xml:

   <?xml version="1.0" encoding="utf-8"?>
   <trigger keyword="@string/sound_effect" >
       <input prompt="@string/glass_voice_prompt" />
   </trigger>

4. Make your res/values/strings.xml file look like this:

   <resources>
       <string name="app_name">Voice And Audio</string>
       <string name="say_glasswaretemplate">Voice and Audio</string>
       <string name="glass_voice_prompt">Which sound to hear?</string>
       <string name="soundeffect">Sound Effect</string>
       <string name="stop">STOP</string>
       <string name="disallowed">Disallowed</string>
       <string name="dismissed">Dismissed</string>
       <string name="error">Error</string>
       <string name="selected">Selected</string>
       <string name="success">Success</string>
       <string name="tap">Tap</string>
   </resources>

5. In AndroidManifest.xml, add the following content:

   <activity
       android:name="com.morkout.voiceandaudio.GlassSoundActivity"
       android:enabled="true"
       android:label="@string/app_name" >
       <intent-filter>
           <action android:name="com.google.android.glass.action.VOICE_TRIGGER" />
       </intent-filter>
    
       <meta-data
           android:name="com.google.android.glass.VoiceTrigger"
           android:resource="@xml/voice_trigger_start2" />
   </activity>

6. In MenuActivity.java, add the following statement in the onOptionsItemSelected method:

   case R.id.soundeffect:
       Intent intent2 = new Intent(this, GlassSoundActivity.class);
       startActivity(intent2);
       return true;

7. Finally, create a new Activity subclass file called GlassSoundActivity.java, as shown in Listing 6-3.

   Listing 6-3.  Activity Playing Glass Sound Effects and Processing a Voice Prompt

   public class GlassSoundActivity extends Activity {
       public final static String TAG = "GlassSoundActivity";
    
       @Override
       protected void onCreate(Bundle savedInstanceState) {
           super.onCreate(savedInstanceState);
           openOptionsMenu();
       }
    
       @Override
       public boolean onCreateOptionsMenu(Menu menu) {
           MenuInflater inflater = getMenuInflater();
           inflater.inflate(R.menu.soundeffect, menu);
           return true;
       }
    
       @Override
       public void onResume() {
           super.onResume();
           if (getIntent().getExtras() == null) return;
           ArrayList<String> voiceResults = getIntent().getExtras()
                   .getStringArrayList(RecognizerIntent.EXTRA_RESULTS);
           if (voiceResults != null) {
               if (voiceResults.size() > 0) {
                   AudioManager audio = (AudioManager) this.
                                        getSystemService(Context.AUDIO_SERVICE);
                                         
                   // loop through the recognition results to see if
                   // any of them mataches any Glass sound name
                   for (String result: voiceResults) {
                       Toast.makeText(this, result,
                           Toast.LENGTH_SHORT).show();
    
              if (result.equalsIgnoreCase("disallowed")) {
                  audio.playSoundEffect(Sounds.DISALLOWED);
                  break;
              }
              else if (result.equalsIgnoreCase("dismissed")) {
                   audio.playSoundEffect(Sounds.DISMISSED);
                  break;
              }
              else if (result.equalsIgnoreCase("error")) {
                  audio.playSoundEffect(Sounds.ERROR);
                  break;
              }
              else if (result.equalsIgnoreCase("selected")) {
                  audio.playSoundEffect(Sounds.SELECTED);
                  break;
              }
              else if (result.equalsIgnoreCase("success")) {
                  audio.playSoundEffect(Sounds.SUCCESS);
                  break;
              }
              else if (result.equalsIgnoreCase("tap")) {
                  audio.playSoundEffect (Sounds.TAP);
                  break;
              }
          }}}}
           
       @Override
       public void onOptionsMenuClosed (Menu menu) {
           finish();
       }
    
       @Override
       public boolean onOptionsItemSelected(MenuItem item) {
           AudioManager audio = (AudioManager) this.
                         getSystemService(Context.AUDIO_SERVICE);
                          
           switch (item.getItemId()) {
           case R.id.disallowed:
               audio.playSoundEffect(Sounds.DISALLOWED);
               Toast.makeText(this, "Disallowed",
                                 Toast.LENGTH_SHORT).show();
               return true;
    
           case R.id.dismissed:
               audio.playSoundEffect(Sounds.DISMISSED);
               Toast.makeText(this, "Dismissed",
                                 Toast.LENGTH_SHORT).show();
               return true;
    
           case R.id.error:
               audio.playSoundEffect(Sounds.ERROR);
    
               Toast.makeText(this, "Error", Toast.LENGTH_SHORT).show();
               return true;
    
           case R.id.selected:
               audio.playSoundEffect(Sounds.SELECTED);
               Toast.makeText(this, "Selected",
                   Toast.LENGTH_SHORT).show();
               return true;
    
           case R.id.success:
               audio.playSoundEffect(Sounds.SUCCESS);
               Toast.makeText(this, "Success", Toast.LENGTH_SHORT).show();
               return true;
    
           case R.id.tap:
               audio.playSoundEffect(Sounds.TAP);
               Toast.makeText(this, "Tap", Toast.LENGTH_SHORT).show();
               return true;
    
           default:
               return super.onOptionsItemSelected(item);
           }
       }
   }

Run the app and after saying “OK Glass,” you can try two different voice commands. Saying “Voice and audio” will launch the app service you’re used to, and tapping Glass will show the Glass sound effect menu. Saying “Sound effect” will launch the Glass sound effect activity directly, and because you have a voice prompt defined in step 3 in the previous exercise, you’ll see the prompt shown in Figure 6-1. After your voice input is recognized, for example you say “Dismissed” (also shown in Figure 6-1), the GlassSoundActivity, because of the description in AndroidManifest.xml, will be launched. Then you use getIntent().getExtras().getStringArrayList in onResume to retrieve the recognized result. In the sample, you test to see whether there’s a match between the result and one of Glass’s sound effects and play the sound if so.

For a voice prompt that launches a service, you can get the recognized test in the service’s onStartCommand method. This and other voice input–related details are well documented at Google’s Glass developer site (https://developers.google.com/glass/develop/gdk/starting-glassware), so I won’t repeat them here.

If you’re interested in a 100 percent hands-free app so you can still do karaoke while your hands are coding, debugging, or writing, you should check out the contextual voice command feature request discussion at https://code.google.com/p/google-glass-api/issues/detail?id=273. This is a commonly requested feature, and it’s likely to be improved in a future Glass update. Therefore, I won’t discuss it further since the text would probably be outdated by the time you read the book. Instead, let’s see how you can improve the karaoke feature in the video app by capturing your beautiful voice and playing it back.

Using MediaRecorder and MediaPlayer

The following code is based on the Android developer site, but it has some important changes to make it work for Google Glass, mainly in the settings of the MediaRecorder object and to simplify the record/play process:

1. Add a new menu entry in VoiceAndAudio’s res/menu/main.xml.

   <item
           android:id="@+id/audiocapture"
           android:title="@string/audiocapture"/>

2. Add the following in MenuActivity.java’s onOptionsItemSelected:

   case R.id.audiocapture:
       Intent intent3 = new Intent(this, AudioCaptureActivity.class);
       startActivity(intent3);
       return true;

3. Add <string name="audiocapture">Audio Capture</string> in strings.xml.
4. Add the following in AndroidManifest.xml:

   <uses-permission android:name="android.permission.RECORD_AUDIO" />
   <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
        
   <activity
       android:name="com.morkout.voiceandaudio.AudioCaptureActivity"
       android:enabled="true"
       android:label="@string/app_name" >
   </activity>

5. Create a new AudioCaptureActivity.java file, as shown in Listing 6-4.

   Listing 6-4.  Capturing and Playing Audio Using MediaRecorder and MediaPlayer

   public class AudioCaptureActivity extends Activity {
       private static final String TAG = "AudioCaptureActivity";
       private static String mFileName = null;
    
       private RecordButton mRecordButton = null;
       private MediaRecorder mRecorder = null;
    
       private MediaPlayer   mPlayer = null;
    
       private void onRecord(boolean start) {
           if (start) {
               startRecording();
           } else {
               stopRecording();
           }
       }
    
       private void startPlaying() {
           mPlayer = new MediaPlayer();
    
           mPlayer.setOnCompletionListener(new MediaPlayer.
                         OnCompletionListener() {
               public void onCompletion(MediaPlayer mp) {
                   mRecordButton.setText("Start recording");
                   mPlayer.release();
                   mPlayer = null;
               }
           });
    
           try {
               mPlayer.setDataSource(mFileName);
               mPlayer.prepare();
               mPlayer.start();
           } catch (IOException e) {
               Log.e(TAG, "prepare() failed");
           }
       }
    
       private void startRecording () {
           mRecorder = new MediaRecorder();
           // the settings below are important for the capture
           // and playback to work in Glass
           mRecorder.setAudioSource(MediaRecorder.AudioSource.DEFAULT);
           mRecorder.setOutputFormat(MediaRecorder.OutputFormat.DEFAULT);
           mRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC);
           mRecorder.setOutputFile(mFileName);
    
           try {
               mRecorder.prepare();
           } catch (IOException e) {
               Log.e(TAG, "prepare() failed");
           }
           mRecorder.start();
       }
    
       private void stopRecording() {
           mRecorder.stop();
           mRecorder.release();
           mRecorder = null;
       }
    
       class RecordButton extends Button {
           boolean mStartRecording = true;
    
           OnClickListener clicker = new OnClickListener() {
               public void onClick(View v) {
                   onRecord(mStartRecording);
                   if (mStartRecording) {
                       setText("Stop recording");
                   } else {
                       setText("Playing...");
                       startPlaying();                                    
                   }
                   mStartRecording = !mStartRecording;
               }
           };
    
           public RecordButton(Context ctx) {
               super(ctx);
               setText("Start recording");
               setOnClickListener(clicker);
           }
       }
    
       @Override
       public void onCreate(Bundle icicle) {
           super.onCreate(icicle);
    
           mFileName = Environment.getExternalStorageDirectory().
                         getAbsolutePath();
           mFileName += "/audiorecordtest.3gp";
    
           LinearLayout ll = new LinearLayout(this);
           mRecordButton = new RecordButton(this);
           ll.addView(mRecordButton,
                   new LinearLayout.LayoutParams(
                           ViewGroup.LayoutParams.WRAP_CONTENT,
                           ViewGroup.LayoutParams.WRAP_CONTENT,
                           0));
           setContentView(ll);
       }
    
       @Override
       public void onPause() {
           super.onPause();
           if (mRecorder != null) {
               mRecorder.release();
               mRecorder = null;
           }
    
           if (mPlayer != null) {
               mPlayer.release();
               mPlayer = null;
           }
       }
   }

Run the app and say “OK Glass” and then “Voice and audio.” Tap to show the menu, and swipe to select Audio Capture. You’ll see a button titled “Start recording.” Tap your Glass touchpad to start recording and then say something and tap again; you’ll hear your recorded voice being played back. You can repeat the record-play process as many times as you want.

MediaRecorder and MediaPlayer are powerful high-level Android APIs for both video and audio capture and playback. There is another set of Android APIs, AudioRecord and AudioTrack, that can be used for audio-specific capture and playback. For audio play (no capture) only, SoundPool is yet another API, which I won’t discuss here, but if you’re interested in knowing when you should use which API, you can check out this article: www.wiseandroid.com/post/2010/07/13/Intro-to-the-three-Android-Audio-APIs.aspx.

Using AudioRecord and AudioTrack

AudioRecord and AudioTrack are low-level APIs that deal directly with the audio raw data. So, AudioRecord is appropriate for audio processing, which can lead to many interesting applications like those that will be introduced in the following sections of this chapter.

To see how to use AudioRecord and AudioTrack to capture and play audio, follow these steps:

1. Add another menu entry, audiocapture2, in main.xml. Add a new case statement in MenuActivity.java. Add <string name="audiocapture2">Audio Capture2</string> in strings.xml.
2. Create a new audiorecorder.xml file in the layout folder.

   <?xml version="1.0" encoding="utf-8"?>
   <LinearLayout xmlns:android="http://schemas.android.com/apk/res/android "
       android:layout_width="fill_parent"
       android:layout_height="fill_parent"
       android:orientation="vertical" >
    
       <TextView
           android:layout_width="fill_parent"
           android:layout_height="wrap_content"
           android:text="HB-Record Audio" /> 
       <Button
           android:id="@+id/startrec"
           android:layout_width="fill_parent"
           android:layout_height="wrap_content"
           android:text="Start Recording" /> 
       <Button
           android:id="@+id/stoprec"
           android:layout_width="fill_parent"
           android:layout_height="wrap_content"
           android:text="Stop Recording" />
    
       <Button
           android:id="@+id/playback"
           android:layout_width="fill_parent"
           android:layout_height="wrap_content"
           android:text="Play Back" /> 
   </LinearLayout>

3. Declare a new activity called AudioCapture2Activity in AndroidManifest.xml.
4. Create a new class called AudioCapture2Activity, as in Listing 6-5.

   Listing 6-5.  Capturing and Playing Audio Using AudioRecord and AudioTrack

   public class AudioCapture2Activity extends Activity {
       Button startRec, stopRec, playBack;
       Boolean recording;
       GestureDetector mGestureDetector;
    
       @Override
       public void onCreate(Bundle savedInstanceState) {
           super.onCreate(savedInstanceState);
           setContentView(R.layout.audiorecorder);
    
           startRec = (Button)findViewById(R.id.startrec);
           stopRec = (Button)findViewById(R.id.stoprec);
           playBack = (Button)findViewById(R.id.playback);
    
           startRec.setOnClickListener(startRecOnClickListener);
           stopRec.setOnClickListener(stopRecOnClickListener);
           playBack.setOnClickListener(playBackOnClickListener);
    
           stopRec.setEnabled(false);
           playBack.setEnabled(false);
    
           mGestureDetector = createGestureDetector(this);
       }
    
       // GestureDetector needed since XE16 to swipe to next/prev button
       private GestureDetector createGestureDetector(Context context) {
           GestureDetector gestureDetector = new GestureDetector(context);
           //Create a base listener for generic gestures
           gestureDetector.setBaseListener(new GestureDetector.BaseListener() {
   
               @Override
               public boolean onGesture(Gesture gesture) {
                   if (gesture == Gesture.SWIPE_RIGHT) {
                       if (startRec.isEnabled()) {
                           startRec.setEnabled(false);
                           stopRec.setEnabled(true);
                           stopRec.requestFocus();
                           playBack.setEnabled(false);
                       }
                       else if (stopRec.isEnabled()) {
                           stopRec.setEnabled(false);
                           playBack.setEnabled(true);
                           playBack.requestFocus();
                           startRec.setEnabled(false);
                       }
                       else if (playBack.isEnabled()) {
                           playBack.setEnabled(false);
                           startRec.setEnabled(true);
                           startRec.requestFocus();
                           stopRec.setEnabled(false);
                       }
                       return true;
                   } else if (gesture == Gesture.SWIPE_LEFT) {
                       if (startRec.isEnabled()) {
                           startRec.setEnabled(false);
                           playBack.setEnabled(true);
                           playBack.requestFocus();
                           stopRec.setEnabled(false);
                       }
                       else if (stopRec.isEnabled()) {
                           stopRec.setEnabled(false);
                           startRec.setEnabled(true);
                           startRec.requestFocus();
                           playBack.setEnabled(false);
                       }
                       else if (playBack.isEnabled()) {
                           playBack.setEnabled(false);
                           stopRec.setEnabled(true);
                           stopRec.requestFocus();
                           startRec.setEnabled(false);
                       }
                       return true;
                   }
                   return false;
               }
           });
    
           return gestureDetector;
       }
    
       public boolean onGenericMotionEvent(MotionEvent event) {
           if (mGestureDetector != null) {
               return mGestureDetector.onMotionEvent(event);
           }
           return false;
       }
    
       OnClickListener startRecOnClickListener = new OnClickListener() {
           @Override
           public void onClick(View arg0) {
               startRec.setEnabled(false);
               stopRec.setEnabled(true);
               stopRec.requestFocus();
    
               Thread recordThread = new Thread(new Runnable() {
                   @Override
                   public void run() {
                       recording = true;
                       startRecord();
                   }
               });
    
               recordThread.start();
           }
       };
    
       OnClickListener stopRecOnClickListener = new OnClickListener() {
           @Override
           public void onClick(View arg0) {
               recording = false;
               startRec.setEnabled(false);
               stopRec.setEnabled(false);
               playBack.setEnabled(true);
               playBack.requestFocus();
           }
       };
    
       OnClickListener playBackOnClickListener   = new OnClickListener() {
           @Override
           public void onClick(View v) {
               startRec.setEnabled(true);
               stopRec.setEnabled(false);
               playBack.setEnabled(false);
               startRec.requestFocus();
               playRecord();
           }
       };
    
       private void startRecord() {
           File file = new File(Environment.getExternalStorageDirectory(),
                         " audiorecordtest.pcm");
           try {
               file.createNewFile();
               OutputStream outputStream = new FileOutputStream(file);
               BufferedOutputStream bufferedOutputStream = new
                                 BufferedOutputStream(outputStream);
               DataOutputStream dataOutputStream = new
                                 DataOutputStream(bufferedOutputStream);
                                  
               int minBufferSize = AudioRecord.getMinBufferSize(11025,
                   AudioFormat.CHANNEL_IN_MONO,
                   AudioFormat.ENCODING_PCM_16BIT);
    
               short[] audioData = new short[minBufferSize];
    
               AudioRecord audioRecord = new AudioRecord(
                   MediaRecorder.AudioSource.MIC,
                   11025,
                   AudioFormat.CHANNEL_IN_MONO,
                   AudioFormat.ENCODING_PCM_16BIT,
                   minBufferSize);
    
               audioRecord.startRecording();
    
               while(recording) {
                   int numberOfShort = audioRecord.read(audioData, 0,
                                        minBufferSize);
                   for(int i = 0; i < numberOfShort; i++)
                       dataOutputStream.writeShort(audioData[i]);
               }
               audioRecord.stop();
               dataOutputStream.close();
           }
           catch (IOException e) {
               e.printStackTrace();
           }
       }
    
       void playRecord() {
           File file = new File(Environment.getExternalStorageDirectory (),
                          " audiorecordtest.pcm");
           int shortSizeInBytes = Short.SIZE/Byte.SIZE;
    
           int bufferSizeInBytes = (int)(file.length()/shortSizeInBytes);
           short[] audioData = new short[bufferSizeInBytes];
    
           try {
               InputStream inputStream = new FileInputStream(file);
               BufferedInputStream bufferedInputStream = new
                                 BufferedInputStream(inputStream);
               DataInputStream dataInputStream = new
                                 DataInputStream(bufferedInputStream);
                                  
               int i = 0;
               while(dataInputStream.available() > 0) {
                   audioData[i] = dataInputStream.readShort();
                   i++;
               }
    
               dataInputStream.close();
    
               AudioTrack audioTrack = new AudioTrack(
                   AudioManager.STREAM_MUSIC,
                   11025,
                   AudioFormat.CHANNEL_CONFIGURATION_MONO,
                   AudioFormat.ENCODING_PCM_16BIT,
                   bufferSizeInBytes,
                   AudioTrack.MODE_STREAM);
    
               audioTrack.play();
               audioTrack.write(audioData, 0, bufferSizeInBytes);
           } catch (FileNotFoundException e) {
               e.printStackTrace();
           } catch (IOException e) {
               e.printStackTrace();
           }
       }
   }

The two most important methods here are startRecord and playRecord. Also, notice that startRecord needs to run in a non-UI thread to avoid the blocking of the main thread. Now run the app and select the Audio Capture2 menu item, and you’ll see three buttons (called Start Recording, Stop Recording, and Play Back) that allow you to test the audio capture and play.

If you use adb shell and look at the audio files recorded in the two methods, you may notice the file generated in the second method has a much larger size than the file generated in the first method, especially with longer recording. This is as expected because the raw audio data audiorecordtest.pcm takes more space than the compressed MediaRecorder-generated file audiorecordtest.3gp. Furthermore, if for test purposes you use adb pull to grab the two files, you can most likely play the audiorecordtest.3gp file but will probably need a tool such as open source Audacity (available for download at http://audacity.sourceforge.net/download/) to import the audiorecordtest.pcm file with the settings in Figure 6-2, which match the settings used in the new AudioRecord.

Improving the Karaoke App

Now you’re ready to further improve your karaoke app by capturing your singing while playing the selected video and playing back the recorded audio after the video play is done. Make the following changes in MyVideoApps’ VideoSearchActivity.java file:

1. Add the following code after public void launchVideo(String youtubeurl):

   private static String mFileName = Environment.getExternalStorageDirectory().
       getAbsolutePath() + "/karaoke-recording.3gp";
   private MediaRecorder mRecorder = null;
   private MediaPlayer   mPlayer = null;
   private Boolean mRecording;
    
   private void startPlaying() {
       mPlayer = new MediaPlayer();
       try {
           mPlayer.setDataSource(mFileName);
           mPlayer.prepare();
           mPlayer.start();
       } catch (IOException e) {
           Log.e(TAG, "prepare() failed");
       }
    
   }
    
   private void stopPlaying() {
       if (mPlayer != null) {
           mPlayer.release();
           mPlayer = null;
       }
   }
    
   private void startRecording() {
       mRecorder = new MediaRecorder();
       mRecorder.setAudioSource(MediaRecorder.AudioSource.DEFAULT);
       mRecorder.setOutputFormat(MediaRecorder.OutputFormat.DEFAULT);
       mRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC);
    
       mRecorder.setOutputFile(mFileName);
       try {
           mRecorder.prepare();
       } catch (IOException e) {
           Log.e(TAG, "prepare() failed");
       }
    
       mRecorder.start();
   }
    
   private void stopRecording() {
       if (mRecorder != null) {
           mRecorder.stop();
           mRecorder.release();
           mRecorder = null;
       }
   }

2. Add a new case statement in onActivityResult to handle the return from the video play activity. This will stop the audio recording and start playing the recorded audio.

   protected void onActivityResult(int RequestCode,
              int ResultCode, Intent data) {
       switch(RequestCode) {
       case VIDEO_PLAY_ACTIVITY:
           stopRecording();
           startPlaying();
           break;
       case VOICE_RECOGNIZER_REQUEST_CODE:

3. Add the following code at the end of public void launchVideo(String youtubeurl):

   new Thread(new Runnable() {
          @Override
       public void run() {
           mRecording = true;
           startRecording();
       }
   }).start();

4. Add stopPlaying(); at the end of onPause.

You can also use the second method of capturing and playing audio here. Now you can sing all you can while listening to your favorite YouTube video and find out whether you should go to American Idol right away!