diff --git a/.gitignore b/.gitignore index c65815c23..2b161aa3e 100644 --- a/.gitignore +++ b/.gitignore @@ -49,58 +49,57 @@ local.properties app/src/main/assets/tutors/word_copy/debug_animator_graph.json util/a.java ======= -# Built application files -**/repo* -**/build -**/captures -**/keystore +# Built application files +**/repo* +**/build +**/captures +**/keystore *keystore - -keystore.properties - -# The tutor audio data is now externalized and is managed by the -# RTAssetPackager - -/app/src/main/assets/tutors/trackdata/LIBRARY/audio/* - -# We keep a copy of the live story data to support the RTAssetMAnager -# building the transitions and checking the datasource validity -# however the canonical data is in the RTAssetPublisher - -/app/src/verify/assets/tutors/* - -# External Assets -/app/audio_assets/* - -local.properties - -# Gradle generated files -.gradle/ - -# IntelliJ - User-specific configurations -.idea/* -.idea/gradle.xml -.idea/libraries/ -.idea/workspace.xml -.idea/tasks.xml -.idea/.name -.idea/compiler.xml -.idea/copyright/profiles_settings.xml -.idea/encodings.xml -.idea/misc.xml -.idea/modules.xml -.idea/scopes/scope_settings.xml -.idea/vcs.xml -**/*.iml -*.apk - -# OS-specific files -**/.DS_Store - -app/src/main/assets/tutors/word_copy/debug_animator_graph.json -util/a.java + +keystore.properties + +# The tutor audio data is now externalized and is managed by the +# RTAssetPackager + +/app/src/main/assets/tutors/trackdata/LIBRARY/audio/* + +# We keep a copy of the live story data to support the RTAssetMAnager +# building the transitions and checking the datasource validity +# however the canonical data is in the RTAssetPublisher + +/app/src/verify/assets/tutors/* + +# External Assets +/app/audio_assets/* + +local.properties + +# Gradle generated files +.gradle/ + +# IntelliJ - User-specific configurations +.idea/* +.idea/gradle.xml +.idea/libraries/ +.idea/workspace.xml +.idea/tasks.xml +.idea/.name +.idea/compiler.xml +.idea/copyright/profiles_settings.xml +.idea/encodings.xml +.idea/misc.xml +.idea/modules.xml +.idea/scopes/scope_settings.xml +.idea/vcs.xml +**/*.iml +*.apk + +# OS-specific files +**/.DS_Store + +app/src/main/assets/tutors/word_copy/debug_animator_graph.json +util/a.java LICENCE.md README.md .vscode -.nvimlog tags diff --git a/.project b/.project new file mode 100644 index 000000000..51d8286c2 --- /dev/null +++ b/.project @@ -0,0 +1,28 @@ + + + narrate_mode + Project narrate_mode created by Buildship. + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.buildship.core.gradleprojectnature + + + + 1623181697411 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/.settings/org.eclipse.buildship.core.prefs b/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 000000000..ef3d3744b --- /dev/null +++ b/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,13 @@ +arguments= +auto.sync=false +build.scans.enabled=false +connection.gradle.distribution=GRADLE_DISTRIBUTION(WRAPPER) +connection.project.dir= +eclipse.preferences.version=1 +gradle.user.home= +java.home=C\:/Program Files/Java/jdk-14.0.1 +jvm.arguments= +offline.mode=false +override.workspace.settings=true +show.console.view=true +show.executions.view=true diff --git a/README.md b/README.md index 5b4037205..919cc5e2f 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ # **RoboTutor** -Welcome to RoboTutor_2020: XPRIZE's repo has the version of RoboTutor uploaded on 11/20/2018, but RoboTutor has been updated here since then. **This is the newest version.** +Welcome to RoboTutor: XPRIZE's repo has the version uploaded on 11/20/2018, but RoboTutor has been updated here since then. +For changes since 11/20/2018, see [https://github.com/RoboTutorLLC/RoboTutor](https://github.com/RoboTutorLLC/RoboTutor). For changes prior to 3/16/2020, see [https://github.com/RoboTutorLLC/RoboTutor_2019](https://github.com/RoboTutorLLC/RoboTutor_2019). However, it's no longer the newest. @@ -11,7 +12,7 @@ For changes prior to 3/16/2020, see [https://github.com/RoboTutorLLC/RoboTutor_2 ## Quick Installation To quickly install the most recent version of RoboTutor without having to download the full source code, follow these steps: -1. Go to [this Google Drive folder](https://drive.google.com/drive/u/1/folders/1VyajTK_SShmBB4GXJ74737pBS_IKunL_)(updated 8/26/2020). +1. Go to [this Google Drive folder](https://docs.google.com/document/d/1YoVx1K0LdHVNayiFUhkgffE5v28SPcg0FGIvaU84A_U/edit#) (updated 6/16/2020). 2. Download the APK to your tablet (do not install yet). diff --git a/app/build.gradle b/app/build.gradle index b16c48d58..29e32add1 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -82,6 +82,16 @@ android { ignoreWarnings true } + + configurations { + all { + exclude group: 'com.gitub.adrielcafe', module: 'ffmpeg-android-java' + } + } + + applicationVariants.all { variant -> + variant.getRuntimeConfiguration().exclude group: 'com.github.adrielcafe', module: 'ffmpeg-android-java' + } apply from: "checkstyle.gradle" afterEvaluate{ preBuild.dependsOn('checkstyle') @@ -96,8 +106,12 @@ repositories { dependencies { testImplementation 'junit:junit:4.12' - implementation 'com.writingminds:FFmpegAndroid:0.3.2' - implementation 'com.google.guava:guava:25.0-android' + + implementation ('com.writingminds:FFmpegAndroid:0.3.2') { + exclude group: 'com.gitub.adrielcafe', module: 'ffmpeg-android-java' + } + implementation 'com.google.guava:guava:22.0-android' + implementation 'com.android.support:appcompat-v7:25.2.0' implementation 'com.android.support:percent:25.2.0' implementation 'com.google.code.gson:gson:2.8.7' @@ -106,7 +120,9 @@ dependencies { implementation project(':comp_ltkplus') implementation project(':util') implementation project(':mn_component') - implementation project(':comp_listener') + implementation (project(':comp_listener')) { + exclude group: 'com.gitub.adrielcafe', module: 'ffmpeg-android-java' + } implementation project(':comp_reading') implementation project(':comp_questions') implementation project(':sm_component') diff --git a/app/src/main/assets/tutors/story_reading/animator_graph.json b/app/src/main/assets/tutors/story_reading/animator_graph.json index ba3dc58b7..a46491971 100644 --- a/app/src/main/assets/tutors/story_reading/animator_graph.json +++ b/app/src/main/assets/tutors/story_reading/animator_graph.json @@ -3,109 +3,217 @@ "COMMENT": "Animation Graph for the Reading Tutor", "story_reading": { - "type": "ANIMATOR", "title": "Story Reading Tutor", "COMMENT": "", "version": "1.0.0", "rootnode": "INTRO_STATE", - "queueMap": { - "SPEAK_WORD_BEHAVIOR": { "type": "QUEUE", "COMMENT": "Speak the current word and continue to next.", - "preenter": ["CLR_ONCLICK", "TIMER_CANCEL_HINT_BUTTON"], + "preenter": [ + "CLR_ONCLICK", + "TIMER_CANCEL_HINT_BUTTON" + ], "reuse": true, "tracks": [ - {"name": "LOG_EVENT", "type": "QUEUEDCOMMAND", "id": "SstoryReading", "method": "logState", "parms": "type#prompt,value#SPEAK_WORD_BEHAVIOR:String", "features": ""}, - - {"type": "QUEUEDAUDIO", "command": "PLAY", "soundsource": "{{SstoryReading.currentWord}}.mp3", "soundpackage": "words", "mode": "flow", "features": ""}, - {"type": "QUEUEDCOMMAND", "id": "SstoryReading", "method": "post", "parms": "NEXT_WORD:String" , "features": ""} - + { + "name": "LOG_EVENT", + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "logState", + "parms": "type#prompt,value#SPEAK_WORD_BEHAVIOR:String", + "features": "" + }, + { + "type": "QUEUEDAUDIO", + "command": "PLAY", + "soundsource": "{{SstoryReading.currentWord}}.mp3", + "soundpackage": "words", + "mode": "flow", + "features": "" + }, + { + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "post", + "parms": "NEXT_WORD:String", + "features": "" + } ], "preexit": [], "edges": [] }, - "CONFIG_SPEAK_BUTTON": { "type": "QUEUE", "reuse": true, - "COMMENT": "TBD", + "COMMENT": "NOTE: the button must be in a valid state before showing. TODO: put check in to catch invalid behaviors", "tracks": [ - {"name": "LOG_EVENT", "type": "QUEUEDCOMMAND", "id": "SstoryReading", "method": "logState", "parms": "type#behavior,value#CONFIG_SPEAK_BUTTON:String", "features": ""}, - - // NOTE: the button must be in a valid state before showing - // TODO: put check in to catch invalid behaviors - - {"type": "QUEUEDCOMMAND", "id": "SstoryReading", "method": "setStickyBehavior", "parms": "SPEAK_CLICK:String|SPEAK_WORD_BEHAVIOR:String" , "features": ""}, - {"type": "QUEUEDCOMMAND", "id": "SstoryReading", "method": "setSpeakButton", "parms": "ENABLE:String", "features": ""}, - {"type": "QUEUEDCOMMAND", "id": "SstoryReading", "method": "setSpeakButton", "parms": "SHOW:String", "features": ""} + { + "name": "LOG_EVENT", + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "logState", + "parms": "type#behavior,value#CONFIG_SPEAK_BUTTON:String", + "features": "" + }, + + + { + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "setStickyBehavior", + "parms": "SPEAK_CLICK:String|SPEAK_WORD_BEHAVIOR:String", + "features": "" + }, + { + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "setSpeakButton", + "parms": "ENABLE:String", + "features": "" + }, + { + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "setSpeakButton", + "parms": "SHOW:String", + "features": "" + } ] }, - "NARRATE_SENTENCE_BEHAVIOR": { "type": "QUEUE", "COMMENT": "Speak the current word and continue to next.", - "preenter": [], "reuse": true, "tracks": [ - {"name": "LOG_EVENT", "type": "QUEUEDCOMMAND", "id": "SstoryReading", "method": "logState", "parms": "type#behavior,value#NARRATE_SENTENCE_BEHAVIOR:String", "features": ""}, - - {"type": "QUEUEDCOMMAND", "id": "SstoryReading", "method": "post", "parms": "START_NARRATION:String|1000:long" , "features": ""} + { + "name": "LOG_EVENT", + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "logState", + "parms": "type#behavior,value#NARRATE_SENTENCE_BEHAVIOR:String", + "features": "" + }, + { + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "post", + "parms": "START_NARRATION:String|1000:long", + "features": "" + } ], "preexit": [], "edges": [] }, - "SPEAK_UTTERANCE_BEHAVIOR": { "type": "QUEUE", "COMMENT": "Speak the current word and continue to next.", "preenter": [], "reuse": true, "tracks": [ - {"name": "LOG_EVENT", "type": "QUEUEDCOMMAND", "id": "SstoryReading", "method": "logState", "parms": "type#behavior,value#SPEAK_UTTERANCE_BEHAVIOR:String", "features": ""}, - - {"type": "QUEUEDAUDIO", "command": "PLAY", "soundsource": "{{SstoryReading.utterance}}.mp3", "listeners": "SstoryReading", "oncomplete": "TRACK_SEGMENT" , "soundpackage": "story", "mode": "flow", "features": ""} + { + "name": "LOG_EVENT", + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "logState", + "parms": "type#behavior,value#SPEAK_UTTERANCE_BEHAVIOR:String", + "features": "" + }, + { + "type": "QUEUEDAUDIO", + "command": "PLAY", + "soundsource": "{{SstoryReading.utterance}}.mp3", + "listeners": "SstoryReading", + "oncomplete": "TRACK_SEGMENT", + "soundpackage": "story", + "mode": "flow", + "features": "" + } ], "preexit": [], "edges": [] }, - "SPEAK_SENTENCE_BEHAVIOR": { "type": "QUEUE", "COMMENT": "Speak the current word and continue to next.", "preenter": [], "reuse": true, "tracks": [ - {"name": "LOG_EVENT", "type": "QUEUEDCOMMAND", "id": "SstoryReading", "method": "logState", "parms": "type#behavior,value#SPEAK_SENTENCE_BEHAVIOR:String", "features": ""}, - - {"type": "QUEUEDAUDIO", "command": "PLAY", "soundsource": "{{SstoryReading.sentence}}.mp3", "listeners": "SstoryReading", "oncomplete": "TRACK_SEGMENT", "soundpackage": "story", "mode": "flow", "features": ""}, - {"type": "QUEUEDCOMMAND", "id": "SstoryReading", "method": "nextSentence" , "features": ""} + { + "name": "LOG_EVENT", + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "logState", + "parms": "type#behavior,value#SPEAK_SENTENCE_BEHAVIOR:String", + "features": "" + }, + { + "type": "QUEUEDAUDIO", + "command": "PLAY", + "soundsource": "{{SstoryReading.sentence}}.mp3", + "listeners": "SstoryReading", + "oncomplete": "TRACK_SEGMENT", + "soundpackage": "story", + "mode": "flow", + "features": "" + }, + { + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "nextSentence", + "features": "" + } ], "preexit": [], "edges": [] + }, + "WRONG_WORD_BEHAVIOR": { + "type": "QUEUE", + "COMMENT": "Stop recording in the event of long silence", + "preenter": [], + "reuse": "true", + "tracks": [ + { + "name": "LOG_EVENT", + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "logState", + "parms": "type#behavior,value#WRONG_WORD_BEHAVIOR:String", + "features": "" + }, + { + "type": "QUEUEDCOMMAND", + "id": "SstoryReading", + "method": "wrongWordBehavior", + "parms": "", + "features": "" + } + ], + "peexit": [], + "edges": [] } - }, - "nodeMap": { - "COMMENT": "@@@@@ CNodes @@@@@", - "INTRO_STATE": { "type": "NODE", "COMMENT": "Intro Clip", - "preenter": ["SET_VERSION"], + "preenter": [ + "SET_VERSION" + ], "maptype": "moduleMap", "mapname": "PLAYINTRO", "preexit": [], "edges": [ - {"constraint": "", "edge": "NEXT_STEP"} + { + "constraint": "", + "edge": "NEXT_STEP" + } ] }, - "NEXT_STEP": { "COMMENT": "When user inputs a correct answer...", "type": "node", @@ -114,43 +222,97 @@ "preenter": [], "preexit": [], "edges": [ - {"constraint": "STORY_STARTING", "edge": "BEGIN_STORY"}, - {"constraint": "ECHO_LINEMODE", "edge": "ECHO_LINE_NODE"}, - {"constraint": "PARROT_LINEMODE", "edge": "PARROT_LINE_NODE"}, - {"constraint": "STORY_COMPLETE", "edge": "NEXT_SCENE"}, - {"constraint": "PAGE_COMPLETE", "edge": "NEXT_PAGE_NODE"}, - {"constraint": "PARAGRAPH_COMPLETE", "edge": "NEXT_PARA_NODE"}, - {"constraint": "LINE_COMPLETE", "edge": "NEXT_LINE_NODE"}, - {"constraint": "", "edge": "NEXT_WORD_NODE"} + { + "constraint": "STORY_STARTING", + "edge": "BEGIN_STORY" + }, + { + "constraint": "ECHO_LINEMODE", + "edge": "ECHO_LINE_NODE" + }, + { + "constraint": "PARROT_LINEMODE", + "edge": "PARROT_LINE_NODE" + }, + { + "constraint": "STORY_COMPLETE", + "edge": "NEXT_SCENE" + }, + { + "constraint": "PAGE_COMPLETE", + "edge": "NEXT_PAGE_NODE" + }, + { + "constraint": "PARAGRAPH_COMPLETE", + "edge": "NEXT_PARA_NODE" + }, + { + "constraint": "LINE_COMPLETE", + "edge": "NEXT_LINE_NODE" + }, + { + "constraint": "", + "edge": "NEXT_WORD_NODE" + } ] }, - "BEGIN_STORY": { "type": "NODE", "COMMENT": "", "maptype": "actionMap", "mapname": "START_STORY", - "preenter": ["SET_NARRATOR", "SET_UTTERANCE"], + "preenter": [ + "SET_NARRATOR", + "SET_UTTERANCE" + ], "preexit": [], "edges": [ - {"constraint": "", "edge": "LISTEN"} + { + "constraint": "", + "edge": "LISTEN" + } ] }, - "LISTEN": { "COMMENT": "", "type": "node", - "preenter": ["SET_UTTERANCE_BEHAVIOR", "SET_HYPOTHESIS_BEHAVIOR", "SET_ONCLICK_SPEAK_WORD", "TIMER_HINT_BUTTON"], + "preenter": [ + "SET_UTTERANCE_BEHAVIOR", + "SET_HYPOTHESIS_BEHAVIOR", + "SET_ONCLICK_SPEAK_WORD", + "TIMER_HINT_BUTTON", + "SET_BUTTON_NEXTSENTENCE", + "SET_BUTTON_NEXTPARAGRAPH", + "SET_BUTTON_NEXTPAGE", + "TIMER_STOP_LISTENING" + ], "maptype": "moduleMap", "mapname": "LISTENING", - "preexit": ["CLR_HYPOTHESIS_BEHAVIOR", "CLR_ONCLICK", "TIMER_CANCEL_HINT_BUTTON"], + "preexit": [ + "CLR_HYPOTHESIS_BEHAVIOR", + "CLR_ONCLICK", + "TIMER_CANCEL_HINT_BUTTON", + "TIMER_CANCEL_STOP_LISTENING" + ], "edges": [ - {"constraint": "FTR_RIGHT", "edge": "CORRECT"}, - {"constraint": "SECOND_ERROR", "edge": "WRONG_2"}, - {"constraint": "", "edge": "WRONG"} + { + "constraint": "FTR_RIGHT", + "edge": "CORRECT" + }, + { + "constraint": "NARRATION_CAPTURE_MODE", + "edge": "WRONG" + }, + { + "constraint": "SECOND_ERROR", + "edge": "WRONG_2" + }, + { + "constraint": "", + "edge": "WRONG" + } ] }, - "CORRECT": { "COMMENT": "When user inputs a correct answer...", "type": "node", @@ -159,10 +321,12 @@ "preenter": [], "preexit": [], "edges": [ - {"constraint": "", "edge": "NEXT_STEP"} + { + "constraint": "", + "edge": "NEXT_STEP" + } ] }, - "WRONG": { "COMMENT": "When user says an incorrect word...", "type": "node", @@ -171,22 +335,36 @@ "mapname": "PLAYWRONG", "preexit": [], "edges": [ - {"constraint": "", "edge": "LISTEN"} + { + "constraint": "NARRATION_CAPTURE_MODE", + "edge": "NARRATE_WRONG" + }, + { + "constraint": "", + "edge": "LISTEN" + } ] }, - "WRONG_2": { "COMMENT": "When user says an incorrect word...", "type": "node", "maptype": "moduleMap", - "preenter": ["SET_SPEAK_EVENT"], + "preenter": [ + "SET_SPEAK_EVENT" + ], "mapname": "SPEAK_EVENT_MODULE", "preexit": [], "edges": [ - {"constraint": "", "edge": "LISTEN"} + { + "constraint": "NARRATION_CAPTURE_MODE", + "edge": "NARRATE_WRONG" + }, + { + "constraint": "", + "edge": "LISTEN" + } ] }, - "ECHO_LINE_NODE": { "type": "NODE", "COMMENT": "When in echo mode - we repeat the last line", @@ -195,10 +373,12 @@ "preenter": [], "preexit": [], "edges": [ - {"constraint": "", "edge": "LISTEN"} + { + "constraint": "", + "edge": "LISTEN" + } ] }, - "PARROT_LINE_NODE": { "type": "NODE", "COMMENT": "When in parrot mode - we listen to the student repeat the last line", @@ -207,10 +387,12 @@ "preenter": [], "preexit": [], "edges": [ - {"constraint": "", "edge": "LISTEN"} + { + "constraint": "", + "edge": "LISTEN" + } ] }, - "NEXT_WORD_NODE": { "type": "NODE", "COMMENT": "When module is complete - move to next scene in the scenegraph", @@ -219,10 +401,12 @@ "preenter": [], "preexit": [], "edges": [ - {"constraint": "", "edge": "LISTEN"} + { + "constraint": "", + "edge": "LISTEN" + } ] }, - "NEXT_LINE_NODE": { "type": "NODE", "COMMENT": "When module is complete - move to next scene in the scenegraph", @@ -231,10 +415,16 @@ "preenter": [], "preexit": [], "edges": [ - {"constraint": "", "edge": "LISTEN"} + { + "constraint": "NARRATION_CAPTURE_MODE", + "edge": "NARRATE_RIGHT" + }, + { + "constraint": "", + "edge": "LISTEN" + } ] }, - "NEXT_PARA_NODE": { "type": "NODE", "COMMENT": "When module is complete - move to next scene in the scenegraph", @@ -243,10 +433,16 @@ "preenter": [], "preexit": [], "edges": [ - {"constraint": "", "edge": "LISTEN"} + { + "constraint": "NARRATION_CAPTURE_MODE", + "edge": "NARRATE_RIGHT" + }, + { + "constraint": "", + "edge": "LISTEN" + } ] }, - "NEXT_PAGE_NODE": { "type": "NODE", "COMMENT": "When module is complete - move to next scene in the scenegraph", @@ -255,10 +451,16 @@ "preenter": [], "preexit": [], "edges": [ - {"constraint": "", "edge": "LISTEN"} + { + "constraint": "NARRATION_CAPTURE_MODE", + "edge": "NARRATE_RIGHT" + }, + { + "constraint": "", + "edge": "LISTEN" + } ] }, - "NEXT_SCENE": { "type": "NODE", "COMMENT": "When module is complete - move to next scene in the scenegraph", @@ -268,12 +470,128 @@ "preexit": [], "edges": [ ] - } + }, + "NARRATE_RIGHT": { + "type": "NODE", + "COMMENT": "Node that triggers audio saving for Narration capture", + "maptype": "moduleMap", + "mapname": "SAVEAUDIO", + "preenter": [], + "preexit": [], + "edges": [ + { + "constraint": "", + "edge": "LISTEN" + } + ] + }, + "NARRATE_WRONG": { + "type": "NODE", + "COMMENT": "In narrate mode, a word is pronounced incorrectly.", + "maptype": "moduleMap", + "mapname": "START_UTTERANCE", + "preenter": [], + "preexit": [], + "edges": [ + { + "constraint": "", + "edge": "LISTEN" + } + ] + }, + "RESTART_LINE_NODE": { + "type": "NODE", + "COMMENT": "Goes back to the beginning of the line", + "maptype": "moduleMap", + "mapname": "START_LINE", + "preenter": [], + "preexit": [], + "edges": [ + { + "constraint": "", + "edge": "LISTEN" + } + ] + }, + + "RESTART_UTTERANCE_NODE": { + "type": "NODE", + "COMMENT": "Goes back to the start of the utterance", + "maptype": "moduleMap", + "mapname": "START_UTTERANCE", + "preenter": [], + "preexit": [], + "edges": [ + { + "constrant": "", + "edge": "LISTEN" + } + ] + }, + "NARRATION_END_NODE": { + "type": "NODE", + "COMMENT": "Begins the saving of audio data", + "maptype": "moduleMap", + "mapname": "END_UTTERANCE_CAPTURE_MOD", + "preenter": [], + "preexit": [], + "edges": [ + { + "constraint": "", + "edge": "ECHO_LINE_NODE" + } + ] }, + "NARRATE_HESITATION": { + "type": "NODE", + "COMMENT": "Used if there is hesitation in narration capture mode, because a silence that is too long is not useful for the recording", + "maptype": "moduleMap", + "mapname": "NULL_MODULE", + "preenter": [], + "preexit": [], + "edges": [ + { + "constraint": "NARRATION_CAPTURE_MODE", + "edge": "WRONG" + }, + { + "constraint": "", + "edge": "LISTEN" + } + ] + } + }, "moduleMap": { - "COMMENT": "@@@@@ CModules @@@@@", + + "END_UTTERANCE_CAPTURE_MOD": { + "type": "MODULE", + "reuse": true, + "COMMENT": "", + "tracks": [ + {"type": "COMMAND","id": "SstoryReading","method": "endOfUtteranceCapture","parms": "none","features": ""} + ] + }, + + "SAVEAUDIO": { + "type": "MODULE", + "reuse": true, + "COMMENT": "module that saves audio for narration capture", + "tracks" : [ + {"type": "COMMAND", "id": "SstoryReading", "method": "saveToFile", "parms": "none", "features": "" } + ] + }, + + "CLEARAUDIO": { + "type": "MODULE", + "reuse": true, + "COMMENT": "module that deletes current audio data recorded for this utterance in Narrate Mode", + "tracks": [ + {"type": "COMMAND","id": "SstoryReading","method": "clearAudioData","parms": "none","features": ""} + ] + }, + "PLAYINTRO": { "type": "MODULE", "reuse": true, @@ -287,7 +605,7 @@ {"type": "AUDIO", "command": "PLAY", "soundsource": "Please read aloud.mp3", "mode": "flow", "features": "!FTR_PROMPT&FTR_USER_READ|!FTR_PROMPT&FTR_USER_ECHO|!FTR_PROMPT&FTR_USER_REVEAL"}, //{"type": "AUDIO", "command": "PLAY", "soundsource": "Now lets listen to a story.mp3", "mode": "flow", "features": "!FTR_PROMPT&FTR_USER_HEAR|!FTR_PROMPT&FTR_USER_HIDE"}, {"type": "AUDIO", "command": "PLAY", "soundsource": "Listen carefully.mp3", "mode": "flow", "features": "!FTR_PROMPT&FTR_USER_HEAR|!FTR_PROMPT&FTR_USER_HIDE"}, - {"type": "AUDIO", "command": "PLAY", "soundsource": "Please listen and repeat after me.mp3", "mode": "flow", "features": "!FTR_PROMPT&FTR_USER_PARROT"} + {"type": "AUDIO", "command": "PLAY", "soundsource": "Please listen and repeat after me.mp3", "mode": "flow", "features": "!FTR_PROMPT&FTR_USER_PARROT"} ] }, @@ -418,7 +736,26 @@ {"type": "COMMAND", "id": "SstoryReading", "method": "setHighLight", "parms": "red:String"}, {"type": "COMMAND", "id": "SstoryReading", "method": "continueListening" , "features": ""} ] + }, + + "START_LINE": { + "type": "MODULE", + "reuse": true, + "COMMENT": "goes back to the beginning of the sentence", + "tracks": [ + {"type": "COMMAND", "id": "SstoryReading", "method": "startLine", "parms": "none"} + ] + }, + + "START_UTTERANCE": { + "type": "MODULE", + "reuse": true, + "COMMENT": "", + "tracks": [ + {"type":"COMMAND", "id": "SstoryReading", "method": "restartUtterance", "parms": "none"} + ] } + }, "actionMap": { @@ -458,6 +795,9 @@ "TIMER_HINT_BUTTON": {"type": "TIMER", "id": "HintWordTimer", "startdelay": "0", "period": "3500", "repeat": "false", "action": "CREATEANDSTART", "ontimer": "CONFIG_SPEAK_BUTTON", "features": "FTR_USER_READ|FTR_USER_READING"}, "TIMER_CANCEL_HINT_BUTTON": {"type": "TIMER", "id": "HintWordTimer", "action": "CANCEL", "features": ""}, + "TIMER_STOP_LISTENING": {"type": "TIMER", "id": "HesitationTimer", "startDelay": "0", "period": "2000", "repeat": "false", "action": "CREATEANDSTART","ontimer": "WRONG_WORD_BEHAVIOR", "features": "FTR_USER_READ|FTR_USER_READING|NARRATION_CAPTURE_MODE"}, + "TIMER_CANCEL_STOP_LISTENING": {"type": "TIMER","id": "HesitationTimer", "action": "CANCEL", "features": ""}, + "FLIP_BUTTON_ENABLE": {"type": "COMMAND", "id": "SstoryReading", "method": "setPageFlipButton", "parms": "ENABLE:String", "features": ""}, "FLIP_BUTTON_DISABLE": {"type": "COMMAND", "id": "SstoryReading", "method": "setPageFlipButton", "parms": "DISABLE:String", "features": ""}, "FLIP_BUTTON_SHOW": {"type": "COMMAND", "id": "SstoryReading", "method": "setPageFlipButton", "parms": "SHOW:String", "features": ""}, @@ -471,6 +811,11 @@ "SET_FLIP_NEXTPAGE": {"type": "COMMAND", "id": "SstoryReading", "method": "setVolatileBehavior", "parms": "PAGE_FLIP_CLICK:String|NEXT_PAGE:String" , "features": ""}, "RESET_FLIP_ONCLICK": {"type": "COMMAND", "id": "SstoryReading", "method": "setVolatileBehavior", "parms": "PAGE_FLIP_CLICK:String|NULL:String" , "features": ""}, + "SET_BUTTON_NEXTSENTENCE": {"type": "COMMAND", "id": "SstoryReading", "method": "setVolatileBehavior", "parms": "SENTENCE_SKIP_CLICK:String|NEXT_SENTENCE:String" , "features": "NARRATION_CAPTURE_MODE"}, + "SET_BUTTON_NEXTPARAGRAPH": {"type": "COMMAND", "id": "SstoryReading", "method": "setVolatileBehavior", "parms": "PARAGRAPH_SKIP_CLICK:String|NEXT_PARA:String" , "features": "NARRATION_CAPTURE_MODE"}, + "SET_BUTTON_NEXTPAGE": {"type": "COMMAND", "id": "SstoryReading", "method": "setVolatileBehavior", "parms": "PAGE_SKIP_CLICK:String|NEXT_PAGE:String" , "features": "NARRATION_CAPTURE_MODE"}, + + "SET_SPEAK_EVENT": {"type": "COMMAND", "id": "SstoryReading", "method": "setVolatileBehavior", "parms": "SPEAK_EVENT:String|SPEAK_WORD_BEHAVIOR:String" , "features": ""}, "CLR_SPEAK_EVENT": {"type": "COMMAND", "id": "SstoryReading", "method": "setVolatileBehavior", "parms": "SPEAK_EVENT:String|NULL:String" , "features": ""}, @@ -488,7 +833,9 @@ "CANCEL_FEEDBACK": {"type": "COMMAND", "cmd": "CANCEL_NODE"}, "NEXT_NODE": {"type": "COMMAND", "cmd": "NEXT" }, "WAIT": {"type": "COMMAND", "cmd": "WAIT" }, - "PAUSE": {"type": "COMMAND", "cmd": "WAIT"} + "PAUSE": {"type": "COMMAND", "cmd": "WAIT"}, + + "END_UTTERANCE_CAPTURE": {"type": "COMMAND", "id": "SstoryReading", "method": "endOfUtteranceCapture", "features": ""} }, "constraintMap": { @@ -566,6 +913,18 @@ "FTR_WRONG": { "type": "CONDITION", "test": "FTR_WRONG" + }, + + "NARRATION_CAPTURE_MODE": { + "type": "CONDITION", + "test": "NARRATION_CAPTURE_MODE" + }, + + "UPDATE_NARRATION_CAPTURE": { + "type": "CONDITION", + "test": "{{SstoryReading.narrationCompleteState}}=='TRUE'", + "Then": "true", + "Else": "false" } } } diff --git a/app/src/main/java/cmu/xprize/robotutor/RoboTutor.java b/app/src/main/java/cmu/xprize/robotutor/RoboTutor.java index 8dc019bcf..90f4f8cda 100644 --- a/app/src/main/java/cmu/xprize/robotutor/RoboTutor.java +++ b/app/src/main/java/cmu/xprize/robotutor/RoboTutor.java @@ -84,6 +84,7 @@ import cmu.xprize.util.JSON_Helper; import cmu.xprize.util.TCONST; import cmu.xprize.util.TTSsynthesizer; +import edu.cmu.xprize.listener.AudioWriter; import edu.cmu.xprize.listener.ListenerBase; import static cmu.xprize.comp_logging.PerformanceLogItem.MATRIX_TYPE.LITERACY_MATRIX; @@ -216,13 +217,12 @@ protected void onCreate(Bundle savedInstanceState) { // ACTIVITY = this; + AudioWriter.activity = ACTIVITY; PACKAGE_NAME = getApplicationContext().getPackageName(); Thread.setDefaultUncaughtExceptionHandler(new CrashHandler(hotLogPath, ACTIVITY)); - - // Prep the CPreferenceCache // Update the globally accessible id object for this engine instance. // @@ -289,6 +289,12 @@ protected void onCreate(Bundle savedInstanceState) { catch (Exception e) { Log.wtf(TAG, e); } + + try { + AudioWriter.context = getApplicationContext(); + } catch (Exception e) { + Log.wtf(TAG, e); + } } /** diff --git a/app/src/main/java/cmu/xprize/robotutor/startup/configuration/Configuration.java b/app/src/main/java/cmu/xprize/robotutor/startup/configuration/Configuration.java index 5580971b4..ef9151916 100644 --- a/app/src/main/java/cmu/xprize/robotutor/startup/configuration/Configuration.java +++ b/app/src/main/java/cmu/xprize/robotutor/startup/configuration/Configuration.java @@ -28,6 +28,7 @@ public static void saveConfigurationItems(Context context, ConfigurationItems co .putBoolean(ConfigurationItems.USE_PLACEMENT, configItems.use_placement) .putBoolean(ConfigurationItems.RECORD_AUDIO, configItems.record_audio) .putString(ConfigurationItems.MENU_TYPE, configItems.menu_type) + .putBoolean(ConfigurationItems.CONTENT_CREATION_MODE, configItems.content_creation_mode) .putBoolean(ConfigurationItems.SHOW_HELPER_BUTTON, configItems.show_helper_button) .putBoolean(ConfigurationItems.RECORD_SCREEN_VIDEO, configItems.record_screen_video) .putString(ConfigurationItems.BASE_DIRECTORY, configItems.baseDirectory) @@ -91,6 +92,11 @@ public static String getMenuType(Context context) { .getString(ConfigurationItems.MENU_TYPE, "CD1"); } + public static boolean getContentCreationMode(Context context) { + return context.getSharedPreferences(ROBOTUTOR_CONFIGURATION, MODE_PRIVATE) + .getBoolean(ConfigurationItems.CONTENT_CREATION_MODE, false); + } + public static String getBaseDirectory(Context context) { return context.getSharedPreferences(ROBOTUTOR_CONFIGURATION, MODE_PRIVATE) .getString(ConfigurationItems.BASE_DIRECTORY, "roboscreen"); diff --git a/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationItems.java b/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationItems.java index de986aace..234e18270 100644 --- a/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationItems.java +++ b/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationItems.java @@ -24,6 +24,7 @@ public class ConfigurationItems implements ILoadableObject { public static final String USE_PLACEMENT = "USE_PLACEMENT"; public static final String RECORD_AUDIO = "RECORD_AUDIO"; public static final String MENU_TYPE = "MENU_TYPE"; + public static final String CONTENT_CREATION_MODE = "CONTENT_CREATION_MODE"; public static final String RECORD_SCREEN_VIDEO = "RECORD_SCREEN_VIDEO"; public static final String INCLUDE_AUDIO_OUTPUT_IN_SCREEN_VIDEO = "INCLUDE_AUDIO_OUTPUT_IN_SCREEN_VIDEO"; public static final String SHOW_HELPER_BUTTON = "SHOW_HELPER_BUTTON"; @@ -41,6 +42,7 @@ public class ConfigurationItems implements ILoadableObject { public boolean use_placement; public boolean record_audio; public String menu_type; + public boolean content_creation_mode; public boolean record_screen_video; public boolean show_helper_button; public String baseDirectory; @@ -71,7 +73,8 @@ public ConfigurationItems(String config_version, boolean language_override, boolean language_switcher, boolean no_asr_apps, String language_feature_id, boolean show_demo_vids, boolean use_placement, boolean record_audio, - String menu_type, boolean record_screen_video, boolean include_audio_output_in_screen_video, + String menu_type, boolean content_creation_mode, + boolean record_screen_video, boolean include_audio_output_in_screen_video, boolean show_helper_button, String baseDirectory, boolean pinning_mode) { // this.config_version = config_version; @@ -86,6 +89,7 @@ public ConfigurationItems(String config_version, boolean language_override, this.use_placement = use_placement; this.record_audio = record_audio; this.menu_type = menu_type; + this.content_creation_mode = content_creation_mode; this.record_screen_video = record_screen_video; this.include_audio_output_in_screen_video = include_audio_output_in_screen_video; this.show_helper_button = show_helper_button; @@ -107,6 +111,7 @@ public void setDefaults() { use_placement = true; record_audio = false; menu_type = "CD1"; + content_creation_mode = false; show_helper_button = false; baseDirectory = "roboscreen"; record_screen_video = true; diff --git a/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationQuickOptions.java b/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationQuickOptions.java index 97c9ee7e6..c78365b32 100644 --- a/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationQuickOptions.java +++ b/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationQuickOptions.java @@ -23,6 +23,7 @@ public class ConfigurationQuickOptions { false, false, "CD1", + false, true, false, false, @@ -43,6 +44,7 @@ public class ConfigurationQuickOptions { false, false, "CD1", + false, true, false, false, diff --git a/app/src/main/java/cmu/xprize/robotutor/tutorengine/CDebugLauncher.java b/app/src/main/java/cmu/xprize/robotutor/tutorengine/CDebugLauncher.java index be0a5ca69..20e3e9289 100644 --- a/app/src/main/java/cmu/xprize/robotutor/tutorengine/CDebugLauncher.java +++ b/app/src/main/java/cmu/xprize/robotutor/tutorengine/CDebugLauncher.java @@ -51,8 +51,10 @@ public Boolean launchIfDebug() { this.matrix = mResult.get("skill1"); for (Map.Entry element : mResult.entrySet()) { + debugVars.put(element.getKey(), element.getValue()); + } return true; diff --git a/app/src/main/java/cmu/xprize/robotutor/tutorengine/CMediaManager.java b/app/src/main/java/cmu/xprize/robotutor/tutorengine/CMediaManager.java index c27ba4c2a..0104eb83d 100644 --- a/app/src/main/java/cmu/xprize/robotutor/tutorengine/CMediaManager.java +++ b/app/src/main/java/cmu/xprize/robotutor/tutorengine/CMediaManager.java @@ -33,6 +33,8 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; import cmu.xprize.robotutor.ScreenRecorder; import cmu.xprize.robotutor.tutorengine.graph.type_handler; @@ -78,6 +80,11 @@ public class CMediaManager { final static public String TAG = "CMediaManager"; + int startTime = 0; + + public void setStartTime(int startTime) { + this.startTime = startTime; + } /** * @@ -592,6 +599,8 @@ public int getCurrentPosition(){ protected void createPlayer(String dataSource, String location) { + Log.d("CMediaManager", "CHIRAG dataSource is: " + dataSource); + try { mIsReady = false; @@ -623,6 +632,7 @@ protected void createPlayer(String dataSource, String location) { mPlayer.setOnPreparedListener(this); mPlayer.setOnCompletionListener(this); mPlayer.setLooping(mOwner.isLooping()); + Log.d("CMediaManager","Chirag - isLooping is: " + mOwner.isLooping()); float volume = mOwner.getVolume(); @@ -637,6 +647,7 @@ protected void createPlayer(String dataSource, String location) { } catch (Exception e) { Log.e(GRAPH_MSG, "CMediaManager.mediaplayer.ERROR: " + mOwner.sourceName() + " => " + mOwner.resolvedName() + " => " + e); + Log.getStackTraceString(e); // Do the completion event to keep the tutor moving. // onCompletion(mPlayer); @@ -719,14 +730,20 @@ public void play() { if(!mPlaying && mIsAlive) { if(mIsReady) { + // TODO: this will need a tweak for background music etc. mMediaController.startSpeaking(); Log.v(GRAPH_MSG, "CMediaManager.playermanager.play: " + mDataSource); mPlayer.start(); + mPlaying = true; mDeferredStart = false; + + mPlayer.seekTo(startTime); + + Log.d(TAG, "Player seeked to " + startTime); } else mDeferredStart = true; @@ -772,7 +789,7 @@ public void stop() { // This stops the last audio recording file ScreenRecorder.stopLastAudioFile(); - //#Mod issue #335 - give the tracka chance to shutdown. The audio runs in + //#Mod issue #335 - give the track a chance to shutdown. The audio runs in // JNI code so this seems to allow it to shutdown and not restart if we are // interrupting a clip with another clip. // @@ -783,6 +800,20 @@ public void stop() { e.printStackTrace(); } + + + } + + // Narration capture mode, stopping audio file before it completes and moving onto the next + public void stopEarly(long milliseconds) { + Timer timer = new Timer(); + timer.schedule(new TimerTask() { + @Override + public void run() { + seekTo(mPlayer.getDuration()-1); + + } + }, milliseconds); } diff --git a/app/src/main/java/cmu/xprize/robotutor/tutorengine/CTutor.java b/app/src/main/java/cmu/xprize/robotutor/tutorengine/CTutor.java index 036b20296..85bfabda0 100644 --- a/app/src/main/java/cmu/xprize/robotutor/tutorengine/CTutor.java +++ b/app/src/main/java/cmu/xprize/robotutor/tutorengine/CTutor.java @@ -719,6 +719,7 @@ private void automateScene(ITutorSceneImpl tutorContainer, scene_descriptor scen } } catch(Exception e) { + Log.getStackTraceString(e); Log.d(TAG, "automateScene Error: " + e); } } diff --git a/app/src/main/java/cmu/xprize/robotutor/tutorengine/graph/type_audio.java b/app/src/main/java/cmu/xprize/robotutor/tutorengine/graph/type_audio.java index ee8e7f8b0..8d9e49c76 100644 --- a/app/src/main/java/cmu/xprize/robotutor/tutorengine/graph/type_audio.java +++ b/app/src/main/java/cmu/xprize/robotutor/tutorengine/graph/type_audio.java @@ -1,496 +1,520 @@ -//********************************************************************************* -// -// Copyright(c) 2016-2017 Kevin Willows All Rights Reserved -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//********************************************************************************* - -package cmu.xprize.robotutor.tutorengine.graph; - -import android.util.Log; - -import org.json.JSONObject; - -import java.util.Objects; -import java.util.Timer; -import java.util.TimerTask; - -import cmu.xprize.robotutor.RoboTutor; -import cmu.xprize.robotutor.tutorengine.CDebugLauncher; -import cmu.xprize.robotutor.tutorengine.CMediaController; -import cmu.xprize.robotutor.tutorengine.CMediaManager; -import cmu.xprize.robotutor.tutorengine.IMediaListener; -import cmu.xprize.robotutor.tutorengine.graph.vars.IScope2; -import cmu.xprize.util.CEvent; -import cmu.xprize.util.CFileNameHasher; -import cmu.xprize.util.TCONST; - -import static cmu.xprize.util.TCONST.AUDIO_EVENT; -import static cmu.xprize.util.TCONST.TYPE_AUDIO; - - -/** - * Media players are special objects as there is a system wide limit on how many can be active - * at one time. As a result we centralize creation and management of MediaPlayers to CMediaManager - * where we can cache players across tutors as well as play/pause etc globally - */ -public class type_audio extends type_action implements IMediaListener { - - protected static final String NOOP = "NOOP"; - - // NOTE: we run at a Flash default of 24fps - which is the units in which - // index and duration are calibrated - - protected CFileNameHasher mFileNameHasher; - protected CMediaManager mMediaManager; - protected CMediaManager.PlayerManager mPlayer; - protected boolean mPreLoaded = false; - - private Timer tempTimer; - private String mSoundSource; - private String mSourcePath; - private String mResolvedName; - private String mPathResolved; - private String mRawName; - private String mLocation; - - private boolean _useHashName = true; - private boolean _packageInit = false; - - // json loadable fields - public String command; - public String lang; - public String soundsource; - public String soundpackage; - - public String listeners = ""; - public String oncomplete = NOOP; - - public boolean repeat = false; - public float volume = -1f; - public long index = 0; - - - final static public String TAG = "type_audio"; - - - public type_audio() { - Timer tempTimer = new Timer(); - setTempTimer(tempTimer); - mFileNameHasher = CFileNameHasher.getInstance(); - } - - /** - * TODO: onDestroy not being called when tutor is killed - */ - public void onDestroy() { - stopTempTimer(); - - mMediaManager.detachMediaPlayer(this); - } - - //******************************************************* - //** Global Media Control Start - - private boolean mWasPlaying = false; - - @Override - public String sourceName() { - Log.d("ULANI type_audio", "sourceName: "+soundsource); - return soundsource; - } - - @Override - public String resolvedName() { - Log.d("ULANI type_audio", "resolvedName: "+mResolvedName); - return (mResolvedName == null) ? "" : mResolvedName; - } - - @Override - public void globalPause() { - Log.d("ULANI", "globalPause: wasPlaying= "+mWasPlaying); - if (mPlayer != null) { - if (mPlayer.isPlaying()) { - mWasPlaying = true; - - mPlayer.stop(); - } - } - } - - @Override - public void globalPlay() { - Log.d(TAG, "globalPlay: wasplaying = "+mWasPlaying); - if (mPlayer != null) { - if (mWasPlaying) { - mWasPlaying = false; - - RoboTutor.logManager.postEvent_D(_logType, "target:node.audio,action:globalplay,name:"+mRawName); - mPlayer.play(); - } - } - } - - @Override - public void globalStop() { - Log.d(TAG, "globalStop: wasplaying= "+mWasPlaying); - if (mPlayer != null) { - if (mPlayer.isPlaying()) { - mWasPlaying = true; - - RoboTutor.logManager.postEvent_D(_logType, "target:node.audio,action:globalstop,name:"+mRawName); - mPlayer.releasePlayer(); - } - } - } - - @Override - public boolean isLooping() { - return repeat; - } - - @Override - public float getVolume() { - return volume; - } - - /** - * Listen to the MediaController for completion events. - */ - @Override - public void onCompletion(CMediaManager.PlayerManager playerManager) { - - // Support emitting events if components need state info from the audio - // - if(!oncomplete.equals(NOOP)) { - - CEvent event = new CEvent(TYPE_AUDIO, AUDIO_EVENT, oncomplete); - - dispatchEvent(event); - } - - // If not an AUDIOEVENT then we disconnect the player to allow reuse - // - if (!mode.equals(TCONST.AUDIOEVENT)) { - - // Release the mediaController for reuse - // - if (mPlayer != null) { - mPlayer.detach(); - mPlayer = null; - } - - // Flows automatically emit a NEXT_NODE event to scenegraph. - // - if (mode.equals(TCONST.AUDIOFLOW)) { - RoboTutor.logManager.postEvent_V(_logType, "target:node.audio,event:oncompletion,type:flow,emit:eventNext,name:"+mRawName); - _scope.tutor().eventNext(); - } - else { - RoboTutor.logManager.postEvent_V(_logType, "target:node.audio,event:oncompletion,type:stream,name:"+mRawName); - } - } - // If this is an AUDIOEVENT type then the mPlayer was released already but we need - // to let the independent playerManager know that it is no longer needed. - // - else { - if (playerManager != null) { - playerManager.detach(); - } - } - } - - //** Global Media Control Start - //******************************************************* - - - /** - * This is an optimization used in timeLines to preload the assets - timeline tracks act as the - * owner so that they are informed directly of audio completion events. This is necessary to - * keep them sync'd with the mediaManager attach states - otherwise they don't know when their - * audio players have been detached and may perform operations on a re-purposed players. - */ - public void preLoad(IMediaListener owner) { - System.out.println("PRELOAD"); - System.out.println("OWNER: "+owner.sourceName()+" => "+owner.resolvedName()); - - // Perform late binding to the sound package. - // - initSoundPackage(); - - mPathResolved = getScope().parseTemplate(mSourcePath); - - if(Objects.equals(CDebugLauncher.getDebugVar("use_hash_name"), "false")) { - mPathResolved = mPathResolved.replace("sdcard/Download/RoboTutor/assets/story_questions/audio/en//", "").replace("sdcard/Download/RoboTutor/assets/story_questions/audio/sw//", ""); - _useHashName = false; - } - - RoboTutor.logManager.postEvent_D(_logType, "target:node.audio,action:preload,name:" + mPathResolved); - RoboTutor.logManager.postEvent_D(TCONST.DEBUG_AUDIO_FILE, "target:node.audio,action:preload,name:" + mPathResolved); - - int endofPath = mPathResolved.lastIndexOf("/") + 1; - - // Extract the path and name portions - // NOTE: ASSUME mp3 - trim the mp3 - we just want the filename text to generate the Hash - // TODO: Don't assume mp3 or even an extension - // - String pathPart = mPathResolved.substring(0, endofPath); - - mRawName = mPathResolved.substring(endofPath, mPathResolved.length() - 4); - - // Note we keep this decomposition to provide the resolved name for debug messages - // - if (_useHashName) { - - // Permit actual hash's in the script using the # prefix - // - if (mRawName.startsWith("#")) { - mResolvedName = mRawName.substring(1); - } - // Otherwise generate the hash from the text - else { - mResolvedName = mFileNameHasher.generateHash(mRawName); - } - } else { - mResolvedName = mRawName; - } - - // add the extension back on the generated filename hash - // - mPathResolved = pathPart + mResolvedName + ".mp3"; - - // This allocates a MediaPController for use by this audio_node. The media controller - // is a managed global resource of CMediaManager - // - mPlayer = mMediaManager.attachMediaPlayer(mPathResolved, mLocation, owner); - - mPreLoaded = true; - } - - - @Override - public String applyNode() { - System.out.println("APPLY NODE"); - - String status = TCONST.DONE; - - // If the feature test passes then fire the event. - // Otherwise set flag to indicate event was completed/skipped in this case - // Issue #58 - Make all actions feature reactive. - // - if (testFeatures()) { - - // Non type_timelineFL audio tracks are not preloaded. So do it inline. This just has a - // higher latency between the call and when the audio is actually ready to play. - // - if (!mPreLoaded) { - preLoad(this); - } - mPreLoaded = false; - - // Support having components listen for audio events. - // - if(!listeners.equals("")) { - - String[] compNames = listeners.split(","); - - for(String name : compNames) { - - addViewListener(name); - } - } - - // play on creation if command indicates - if (command.equals(TCONST.PLAY)) { - - play(); - - // Events return done - so they may play on top of each other. - // streams and flows WAIT until completion before continuing. - // - if (mode.equals(TCONST.AUDIOEVENT)) { - status = TCONST.DONE; - } - - // TCONST.STREAMEVENT or TCONST.FLOWEVENT wait for completion - // TCONST.FLOWEVENT automatically advances on completion - else { - status = TCONST.WAIT; - } - } -// else if(command.equals(TCONST.PLAY_CLOZE)){ -// play(TCONST.CLOZE_END); -// } - - } - - return status; - } - - - @Override - public String cancelNode() { - - if(mPlayer != null) { - - stop(); - - mPlayer.detach(); - mPlayer = null; - } - - RoboTutor.logManager.postEvent_D(_logType, "target:node.audio,action:cancelnode,name:" + mRawName); - - return TCONST.NONE; - } - - public void setTempTimer(Timer temp) { - temp.scheduleAtFixedRate( - new TimerTask(){ - public void run(){ - if (mPlayer!= null) { -// Log.d("ULANI", "TEMPTIMER: real time narrationSegment current position = " + mPlayer.getCurrentPosition()); - } - } - },0, // run first occurrence immediately - 100); - this.tempTimer = temp; - } - - public void stopTempTimer(){ - this.tempTimer.cancel(); - } - public void play() { - if(mPlayer != null) { - RoboTutor.logManager.postEvent_I(_logType, "target:node.audio,action:play,name:" + mRawName); - mPlayer.play(); - - // AUDIOEVENT mode tracks are fire and forget - i.e. we disconnect from the player - // and let it continue to completion independently. - // - // This allows this Audio element to be reused immediately - So we can fire another - // instance of the sound while the other is still playing. - // - if(mode == TCONST.AUDIOEVENT) { - RoboTutor.logManager.postEvent_V(_logType, "target:node.audio,type:event,action:complete,name:" + mRawName); - - mPlayer = null; - } - } - } - - public void play(long duration){ - if(mPlayer != null) { - RoboTutor.logManager.postEvent_I(_logType, "target:node.audio,action:play,name:" + mRawName); - mPlayer.play(duration); - - // AUDIOEVENT mode tracks are fire and forget - i.e. we disconnect from the player - // and let it continue to completion independently. - // - // This allows this Audio element to be reused immediately - So we can fire another - // instance of the sound while the other is still playing. - // - if(mode == TCONST.AUDIOEVENT) { - RoboTutor.logManager.postEvent_V(_logType, "target:node.audio,type:event,action:complete,name:" + mRawName); - - mPlayer = null; - } - } - } - - public void stop() { - if(mPlayer != null){ - mPlayer.stop(); - } - } - - - public void pause() { - - if(mPlayer != null) - mPlayer.pause(); - } - - - public void seek(long frame) { - - if(mPlayer != null) - mPlayer.seek(frame); - } - - - public void seekTo(int frameTime) { - - if(mPlayer != null) - mPlayer.seekTo(frameTime); - } - - public void detach() { - - if(mPlayer != null) - mPlayer.detach(); - } - - /** - * Note that we do late binding to the soundpackage as some tutors update the package contents - * on load. e.g. the story_reading tutor creates a custom "story" package that is based on the - * story folder location. - */ - public void initSoundPackage() { - - String langPath; - String assetPath; - - if(!_packageInit) { - - _packageInit = true; - - // If we have set a language then update the sound source to point to the correct subdir - // If no language set then use whichever language is used in the Flash XML - // An audio source can force a language by setting "lang" to a known language Feature ID - // e.g. LANG_SW | LANG_EN | LANG_FR - - // GRAY_SCREEN_BUG _scope.tutorName = "activity_selector" - // GRAY_SCREEN_BUG returns null - mMediaManager = CMediaController.getManagerInstance(_scope.tutorName()); - - // GRAY_SCREEN_BUG X - langPath = mMediaManager.mapSoundPackage(_scope.tutor(), soundpackage, lang); - - // Update the path to the sound source file - // #Mod Dec 13/16 - Moved audio/storyName assets to external storage - // - mSoundSource = TCONST.AUDIOPATH + "/" + langPath + "/" + soundsource; - - assetPath = mMediaManager.mapPackagePath(_scope.tutor(), soundpackage); - - mSourcePath = assetPath + "/" + mSoundSource; - - mLocation = mMediaManager.mapPackageLocation(_scope.tutor(), soundpackage); - } - } - - - - // *** Serialization - - - - @Override - public void loadJSON(JSONObject jsonObj, IScope2 scope) { - - super.loadJSON(jsonObj, scope); - } - -} +//********************************************************************************* +// +// Copyright(c) 2016-2017 Kevin Willows All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//********************************************************************************* + +package cmu.xprize.robotutor.tutorengine.graph; + +import android.util.Log; + +import org.json.JSONObject; + +import java.util.Objects; +import java.util.Timer; +import java.util.TimerTask; + +import cmu.xprize.robotutor.RoboTutor; +import cmu.xprize.robotutor.startup.configuration.Configuration; + +import cmu.xprize.robotutor.tutorengine.CDebugLauncher; +import cmu.xprize.robotutor.tutorengine.CMediaController; +import cmu.xprize.robotutor.tutorengine.CMediaManager; +import cmu.xprize.robotutor.tutorengine.IMediaListener; +import cmu.xprize.robotutor.tutorengine.graph.vars.IScope2; +import cmu.xprize.util.CEvent; +import cmu.xprize.util.CFileNameHasher; +import cmu.xprize.util.IScope; +import cmu.xprize.util.TCONST; +import edu.cmu.xprize.listener.AudioDataStorage; + +import static cmu.xprize.util.TCONST.AUDIO_EVENT; +import static cmu.xprize.util.TCONST.TYPE_AUDIO; + + +/** + * Media players are special objects as there is a system wide limit on how many can be active + * at one time. As a result we centralize creation and management of MediaPlayers to CMediaManager + * where we can cache players across tutors as well as play/pause etc globally + */ +public class type_audio extends type_action implements IMediaListener { + + protected static final String NOOP = "NOOP"; + + // NOTE: we run at a Flash default of 24fps - which is the units in which + // index and duration are calibrated + + protected CFileNameHasher mFileNameHasher; + protected CMediaManager mMediaManager; + protected CMediaManager.PlayerManager mPlayer; + protected boolean mPreLoaded = false; + + private Timer tempTimer; + private String mSoundSource; + private String mSourcePath; + private String mResolvedName; + private String mPathResolved; + private String mRawName; + private String mLocation; + + private boolean _useHashName = true; + private boolean _packageInit = false; + + // json loadable fields + public String command; + public String lang; + public String soundsource; + public String soundpackage; + + public String listeners = ""; + public String oncomplete = NOOP; + + public boolean repeat = false; + public float volume = -1f; + public long index = 0; + + + final static public String TAG = "type_audio"; + + public boolean isNewNarration; + + public static long playDuration = 0; + + // used for manually hijacking this to play audio + public type_audio(boolean isNewNarration) { + this(); + + this.isNewNarration = isNewNarration; + } + + public type_audio() { + Timer tempTimer = new Timer(); + setTempTimer(tempTimer); + mFileNameHasher = CFileNameHasher.getInstance(); + } + + /** + * TODO: onDestroy not being called when tutor is killed + */ + public void onDestroy() { + stopTempTimer(); + + mMediaManager.detachMediaPlayer(this); + } + + //******************************************************* + //** Global Media Control Start + + private boolean mWasPlaying = false; + + @Override + public String sourceName() { + Log.d("ULANI type_audio", "sourceName: "+soundsource); + return soundsource; + } + + @Override + public String resolvedName() { + Log.d("ULANI type_audio", "resolvedName: "+mResolvedName); + return (mResolvedName == null) ? "" : mResolvedName; + } + + @Override + public void globalPause() { + Log.d("ULANI", "globalPause: wasPlaying= "+mWasPlaying); + if (mPlayer != null) { + if (mPlayer.isPlaying()) { + mWasPlaying = true; + + mPlayer.stop(); + } + } + } + + @Override + public void globalPlay() { + Log.d(TAG, "globalPlay: wasplaying = "+mWasPlaying); + if (mPlayer != null) { + if (mWasPlaying) { + mWasPlaying = false; + + RoboTutor.logManager.postEvent_D(_logType, "target:node.audio,action:globalplay,name:"+mRawName); + mPlayer.play(); + } + } + } + + @Override + public void globalStop() { + Log.d(TAG, "globalStop: wasplaying= "+mWasPlaying); + if (mPlayer != null) { + if (mPlayer.isPlaying()) { + mWasPlaying = true; + + RoboTutor.logManager.postEvent_D(_logType, "target:node.audio,action:globalstop,name:"+mRawName); + mPlayer.releasePlayer(); + } + } + } + + @Override + public boolean isLooping() { + return repeat; + } + + @Override + public float getVolume() { + return volume; + } + + /** + * Listen to the MediaController for completion events. + */ + @Override + public void onCompletion(CMediaManager.PlayerManager playerManager) { + + // Support emitting events if components need state info from the audio + // + if(!oncomplete.equals(NOOP)) { + + CEvent event = new CEvent(TYPE_AUDIO, AUDIO_EVENT, oncomplete); + + dispatchEvent(event); + } + + // If not an AUDIOEVENT then we disconnect the player to allow reuse + // + if (!mode.equals(TCONST.AUDIOEVENT)) { + + // Release the mediaController for reuse + // + if (mPlayer != null) { + mPlayer.detach(); + mPlayer = null; + } + + // Flows automatically emit a NEXT_NODE event to scenegraph. + // + if (mode.equals(TCONST.AUDIOFLOW)) { + RoboTutor.logManager.postEvent_V(_logType, "target:node.audio,event:oncompletion,type:flow,emit:eventNext,name:"+mRawName); + _scope.tutor().eventNext(); + } + else { + RoboTutor.logManager.postEvent_V(_logType, "target:node.audio,event:oncompletion,type:stream,name:"+mRawName); + } + } + // If this is an AUDIOEVENT type then the mPlayer was released already but we need + // to let the independent playerManager know that it is no longer needed. + // + else { + if (playerManager != null) { + playerManager.detach(); + } + } + } + + //** Global Media Control Start + //******************************************************* + + + /** + * This is an optimization used in timeLines to preload the assets - timeline tracks act as the + * owner so that they are informed directly of audio completion events. This is necessary to + * keep them sync'd with the mediaManager attach states - otherwise they don't know when their + * audio players have been detached and may perform operations on a re-purposed players. + */ + public void preLoad(IMediaListener owner) { + System.out.println("PRELOAD"); + System.out.println("OWNER: "+owner.sourceName()+" => "+owner.resolvedName()); + + // Perform late binding to the sound package. + // + initSoundPackage(); + + Log.d("msourcepath", "Chirag, it's " + mSourcePath); + + mPathResolved = getScope().parseTemplate(mSourcePath); + + + + if(AudioDataStorage.contentCreationOn || Objects.equals(CDebugLauncher.getDebugVar("use_hash_name"), "false")) { + mPathResolved = mPathResolved.replace("sdcard/Download/RoboTutor/assets/story_questions/audio/en//", "").replace("sdcard/Download/RoboTutor/assets/story_questions/audio/sw//", ""); + _useHashName = false; + } + + RoboTutor.logManager.postEvent_D(_logType, "target:node.audio,action:preload,name:" + mPathResolved); + RoboTutor.logManager.postEvent_D(TCONST.DEBUG_AUDIO_FILE, "target:node.audio,action:preload,name:" + mPathResolved); + + int endofPath = mPathResolved.lastIndexOf("/") + 1; + + // Extract the path and name portions + // NOTE: ASSUME mp3 - trim the mp3 - we just want the filename text to generate the Hash + // TODO: Don't assume mp3 or even an extension + // + String pathPart = mPathResolved.substring(0, endofPath); + + mRawName = mPathResolved.substring(endofPath, mPathResolved.length() - 4); + + // Note we keep this decomposition to provide the resolved name for debug messages + // + if (_useHashName || mRawName.equals("Please read aloud")) { + + // Permit actual hash's in the script using the # prefix + // + if (mRawName.startsWith("#")) { + mResolvedName = mRawName.substring(1); + } + // Otherwise generate the hash from the text + else { + mResolvedName = mFileNameHasher.generateHash(mRawName); + } + } else { + mResolvedName = mRawName; + } + + // add the extension back on the generated filename hash + // + mPathResolved = pathPart + mResolvedName + ".mp3"; + + Log.d("type_Audio", "The resolved name is: " + mResolvedName + ". The resolved path is " + mPathResolved); + // This allocates a MediaPController for use by this audio_node. The media controller + // is a managed global resource of CMediaManager + // + mPlayer = mMediaManager.attachMediaPlayer(mPathResolved, mLocation, owner); + + mPreLoaded = true; + } + + + @Override + public String applyNode() { + System.out.println("APPLY NODE"); + + String status = TCONST.DONE; + + // If the feature test passes then fire the event. + // Otherwise set flag to indicate event was completed/skipped in this case + // Issue #58 - Make all actions feature reactive. + // + if (testFeatures()) { + + // Non type_timelineFL audio tracks are not preloaded. So do it inline. This just has a + // higher latency between the call and when the audio is actually ready to play. + // + if (!mPreLoaded) { + preLoad(this); + } + mPreLoaded = false; + + // Support having components listen for audio events. + // + if(!listeners.equals("")) { + + String[] compNames = listeners.split(","); + + for(String name : compNames) { + + addViewListener(name); + } + } + + // play on creation if command indicates + if (command.equals(TCONST.PLAY)) { + + play(); + + // Events return done - so they may play on top of each other. + // streams and flows WAIT until completion before continuing. + // + if (mode.equals(TCONST.AUDIOEVENT)) { + status = TCONST.DONE; + } + + // TCONST.STREAMEVENT or TCONST.FLOWEVENT wait for completion + // TCONST.FLOWEVENT automatically advances on completion + else { + status = TCONST.WAIT; + } + } +// else if(command.equals(TCONST.PLAY_CLOZE)){ +// play(TCONST.CLOZE_END); +// } + + } + + return status; + } + + + @Override + public String cancelNode() { + + if(mPlayer != null) { + + stop(); + + mPlayer.detach(); + mPlayer = null; + } + + RoboTutor.logManager.postEvent_D(_logType, "target:node.audio,action:cancelnode,name:" + mRawName); + + return TCONST.NONE; + } + + public void setTempTimer(Timer temp) { + temp.scheduleAtFixedRate( + new TimerTask(){ + public void run(){ + if (mPlayer!= null) { +// Log.d("ULANI", "TEMPTIMER: real time narrationSegment current position = " + mPlayer.getCurrentPosition()); + } + } + },0, // run first occurrence immediately + 100); + this.tempTimer = temp; + } + + public void stopTempTimer(){ + this.tempTimer.cancel(); + } + public void play() { + if(mPlayer != null) { + RoboTutor.logManager.postEvent_I(_logType, "target:node.audio,action:play,name:" + mRawName); + mPlayer.play(); + + // AUDIOEVENT mode tracks are fire and forget - i.e. we disconnect from the player + // and let it continue to completion independently. + // + // This allows this Audio element to be reused immediately - So we can fire another + // instance of the sound while the other is still playing. + // + if(mode == TCONST.AUDIOEVENT) { + RoboTutor.logManager.postEvent_V(_logType, "target:node.audio,type:event,action:complete,name:" + mRawName); + + mPlayer = null; + } + } + } + + public void play(long duration){ + if(mPlayer != null) { + RoboTutor.logManager.postEvent_I(_logType, "target:node.audio,action:play,name:" + mRawName); + mPlayer.play(duration); + + // AUDIOEVENT mode tracks are fire and forget - i.e. we disconnect from the player + // and let it continue to completion independently. + // + // This allows this Audio element to be reused immediately - So we can fire another + // instance of the sound while the other is still playing. + // + if(mode == TCONST.AUDIOEVENT) { + RoboTutor.logManager.postEvent_V(_logType, "target:node.audio,type:event,action:complete,name:" + mRawName); + + mPlayer = null; + } + } + } + + public void stop() { + if(mPlayer != null){ + mPlayer.stop(); + } + } + + + public void pause() { + + if(mPlayer != null) + mPlayer.pause(); + } + + + public void seek(long frame) { + + if(mPlayer != null) + mPlayer.seek(frame); + } + + + public void seekTo(int frameTime) { + + if(mPlayer != null) + mPlayer.seekTo(frameTime); + } + + public void detach() { + + if(mPlayer != null) + mPlayer.detach(); + } + + /** + * Note that we do late binding to the soundpackage as some tutors update the package contents + * on load. e.g. the story_reading tutor creates a custom "story" package that is based on the + * story folder location. + */ + public void initSoundPackage() { + + String langPath; + String assetPath; + + if(!_packageInit) { + + _packageInit = true; + + // If we have set a language then update the sound source to point to the correct subdir + // If no language set then use whichever language is used in the Flash XML + // An audio source can force a language by setting "lang" to a known language Feature ID + // e.g. LANG_SW | LANG_EN | LANG_FR + + // GRAY_SCREEN_BUG _scope.tutorName = "activity_selector" + // GRAY_SCREEN_BUG returns null + mMediaManager = CMediaController.getManagerInstance(_scope.tutorName()); + + // GRAY_SCREEN_BUG X + langPath = mMediaManager.mapSoundPackage(_scope.tutor(), soundpackage, lang); + + // Update the path to the sound source file + // #Mod Dec 13/16 - Moved audio/storyName assets to external storage + // + mSoundSource = TCONST.AUDIOPATH + "/" + langPath + "/" + soundsource; + + assetPath = mMediaManager.mapPackagePath(_scope.tutor(), soundpackage); + + mSourcePath = assetPath + "/" + mSoundSource; + + + mLocation = mMediaManager.mapPackageLocation(_scope.tutor(), soundpackage); + } + } + + + + // *** Serialization + + + + @Override + public void loadJSON(JSONObject jsonObj, IScope2 scope) { + + super.loadJSON(jsonObj, scope); + } + + public void hijackScope(IScope2 scope) { + _scope = scope; + } + +} diff --git a/app/src/main/java/cmu/xprize/robotutor/tutorengine/graph/vars/TScope.java b/app/src/main/java/cmu/xprize/robotutor/tutorengine/graph/vars/TScope.java index d18236eb4..a90ea2a25 100644 --- a/app/src/main/java/cmu/xprize/robotutor/tutorengine/graph/vars/TScope.java +++ b/app/src/main/java/cmu/xprize/robotutor/tutorengine/graph/vars/TScope.java @@ -219,9 +219,9 @@ public String parseTemplate(String source) { } // If the symbol references an array object we remember the symbol name and - // continue parsing the input as a new symbol which may be either - // another variable or a number representing the array index - // + // continue parsing the input as a new symbol which may be either + // another variable or a number representing the array index + // else if (tChar == '[') { isArray = true; diff --git a/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java b/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java index 28595cddc..1e335ad9d 100644 --- a/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java +++ b/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java @@ -24,6 +24,7 @@ import android.view.View; import android.widget.Button; +import org.json.JSONException; import org.json.JSONObject; import java.util.Arrays; @@ -35,6 +36,8 @@ import cmu.xprize.comp_logging.ITutorLogger; import cmu.xprize.comp_logging.PerformanceLogItem; import cmu.xprize.robotutor.RoboTutor; +import cmu.xprize.robotutor.startup.configuration.Configuration; +import cmu.xprize.robotutor.tutorengine.CDebugLauncher; import cmu.xprize.robotutor.tutorengine.CMediaController; import cmu.xprize.robotutor.tutorengine.CMediaManager; import cmu.xprize.robotutor.tutorengine.CMediaPackage; @@ -83,6 +86,8 @@ public class TRtComponent extends CRt_Component implements IBehaviorManager, ITu private HashMap _IntegerVar = new HashMap<>(); private HashMap _FeatureMap = new HashMap<>(); + public boolean startLater = false; + static private String TAG = "TRtComponent"; public TRtComponent(Context context) { @@ -335,7 +340,7 @@ public void applyBehaviorNode(String nodeName) { // which is done internally. // case TCONST.QUEUE: - + // CHIRAG - - AFTER THIS, FFW THE AUDIO TO THE CORRECT SPOT if (obj.testFeatures()) { obj.applyNode(); } @@ -465,6 +470,7 @@ public void logState(String logData) { extractFeatureContents(builder, _FeatureMap); RoboTutor.logManager.postTutorState(TUTOR_STATE_MSG, "target#reading_tutor," + logData + builder.toString()); + Log.d("TRt_Component", "Tutor is Listening"); } // ITutorLogger - End @@ -730,6 +736,32 @@ else if (dataNameDescriptor.startsWith(TCONST.SOURCEFILE)) { } else if (dataNameDescriptor.startsWith("{")) { loadJSON(new JSONObject(dataNameDescriptor), null); + } else if (dataNameDescriptor.startsWith(TCONST.UNPACKAGED_ASSET)) { + + // This is for assets that haven't yet been completed (e.g. stories that need narrations) and exists for content creation + String storyFolder = dataNameDescriptor.substring(TCONST.UNPACKAGED_ASSET.length()).toLowerCase(); + + String[] levelval = storyFolder.split("_"); + + String levelFolder = levelval[0]; + + DATASOURCEPATH = TCONST.DOWNLOAD_PATH + "/"; + STORYSOURCEPATH = DATASOURCEPATH + levelFolder + "/" + storyFolder + "/"; + + AUDIOSOURCEPATH = TCONST.DOWNLOAD_PATH + "/" + levelFolder + "/" + storyFolder; + // Probably going to change this to put all the audio in one place + + configListenerLanguage(mMediaManager.getLanguageFeature(mTutor)); + mMediaManager.addSoundPackage(mTutor, MEDIA_STORY, new CMediaPackage(LANG_AUTO, AUDIOSOURCEPATH, LOCAL_STORY_AUDIO)); + + boolean keepOnlyRelevantAudio; + if (CDebugLauncher.getDebugVar("delete_extra_audio") == "true") + keepOnlyRelevantAudio = true; + else + keepOnlyRelevantAudio = false; + + loadStoryANDEnableContentCreation(STORYSOURCEPATH, "ASB_Data", TCONST.EXTERN, Configuration.getContentCreationMode(getContext()), keepOnlyRelevantAudio); + } else if (dataNameDescriptor.startsWith(TCONST.UNPACKAGED_ASSET)) { @@ -753,6 +785,7 @@ else if (dataNameDescriptor.startsWith(TCONST.SOURCEFILE)) { } else { throw (new Exception("BadDataSource")); } + //enableNarrateMode(Configuration.getContentCreationMode(getContext())); } catch (Exception e) { CErrorManager.logEvent(TAG, "Invalid Data Source for : " + mTutor.getTutorName(), e, true); @@ -999,6 +1032,16 @@ public void onButtonClick(String buttonName) { } + /** + * @param sentence Current sentence in a string + * @param index Line number of sentence + * @param wordList Array of words in the sentence + * @param wordIndex Index of current word in Sentence + * @param word Current word in sentence as String + * @param attempts number of attempts + * @param virtual + * @param correct + */ @Override public void updateContext(String sentence, int index, String[] wordList, int wordIndex, String word, int attempts, boolean virtual, boolean correct) { @@ -1076,6 +1119,11 @@ public void echoLine() { mViewManager.echoLine(); } + @Override + public void endOfUtteranceCapture() { + mViewManager.endOfUtteranceCapture(); + } + @Override public void parrotLine() { mViewManager.parrotLine(); @@ -1215,4 +1263,76 @@ private void trackAndLogPerformance(String task, boolean correct) { RoboTutor.perfLogManager.postPerformanceLog(event); } + + @Override + public void constructAudioStoryData() { + mViewManager.constructAudioStoryData(); + } + + @Override + public void clearAudioData() { + mViewManager.clearAudioData(); + } + + @Override + public void startLine() { + mViewManager.startLine(); + } + + @Override + public void restartUtterance() { + mViewManager.restartUtterance(); + } + + @Override + public void prevSentence() {mViewManager.prevSentence();} + + @Override + public void skipSentence() {mViewManager.skipSentence(); } + + @Override + public void updateJSONData(String dataSource, String assetLocation) { + /* + type_audio audioObj = new type_audio(true); + + // json loadable fields are being hijacked here + audioObj.command = "PLAY"; + audioObj.lang = ""; + audioObj.soundsource = dataSource; + audioObj.soundpackage = ""; + audioObj.hijackScope(mTutor.getScope()); + audioObj.applyNode(); + */ + + String jsondata = JSON_Helper.cacheData(STORYSOURCEPATH + "storydata.json"); + + try { + mViewManager.loadJSON(new JSONObject(jsondata), null); + } catch (JSONException e) { + Log.getStackTraceString(e); + } + + } + + + @Override + public void stopAudio() { + mMediaManager.dispMediaPlayers(); + CMediaManager.PlayerManager p = mMediaManager.getPlaying(); + p.stopEarly(60L); + } + + @Override + public void wrongWordBehavior() { + mViewManager.wrongWordBehavior(); + } + + @Override + public void startLate() { + Log.d(TAG, "Forwarding Media to stated starting point: " + firstWordTime); + mMediaManager.dispMediaPlayers(); + + mMediaManager.setStartTime((int) firstWordTime * 10); + } + } diff --git a/app/src/main/res/layout/story_reading.xml b/app/src/main/res/layout/story_reading.xml index 7031ccecc..6aec51ab8 100644 --- a/app/src/main/res/layout/story_reading.xml +++ b/app/src/main/res/layout/story_reading.xml @@ -2,6 +2,7 @@ @@ -18,11 +19,51 @@ android:textColor="#FFF" android:text="This is the Banner Area."/> + + android:layout_height="match_parent" + > + + + diff --git a/app/src/tutor_xmatrices/StoryTransitions.en.csv b/app/src/tutor_xmatrices/StoryTransitions.en.csv index d14334c7d..3303aee76 100644 --- a/app/src/tutor_xmatrices/StoryTransitions.en.csv +++ b/app/src/tutor_xmatrices/StoryTransitions.en.csv @@ -1,2 +1,2 @@ -story.hear::1_1,story.echo::1_1,story.read::1_1,story.hear::1_2,story.echo::1_2,story.read::1_2,story.hear::1_3,story.echo::1_3,story.read::1_3,story.hear::1_11,story.echo::1_11,story.read::1_11,story.hear::1_12,story.echo::1_12,story.read::1_12,story.hear::1_13,story.echo::1_13,story.read::1_13,story.hear::1_14,story.echo::1_14,story.read::1_14,story.hear::1_15,story.echo::1_15,story.read::1_15,story.hear::1_16,story.echo::1_16,story.read::1_16,story.hear::1_17,story.echo::1_17,story.read::1_17,story.hear::1_18,story.echo::1_18,story.read::1_18,story.hear::1_19,story.echo::1_19,story.read::1_19,story.hear::1_20,story.echo::1_20,story.read::1_20,story.hear::1_21,story.echo::1_21,story.read::1_21,story.hear::1_22,story.echo::1_22,story.read::1_22,story.hear::1_23,story.echo::1_23,story.read::1_23,story.hear::1_24,story.echo::1_24,story.read::1_24,story.hear::1_25,story.echo::1_25,story.read::1_25,story.hear::1_26,story.echo::1_26,story.read::1_26,story.hear::1_27,story.echo::1_27,story.read::1_27,story.hear::1_28,story.echo::1_28,story.read::1_28,story.hear::1_29,story.echo::1_29,story.read::1_29,story.hear::1_30,story.echo::1_30,story.read::1_30,story.hear::1_31,story.echo::1_31,story.read::1_31,story.hear::1_32,story.echo::1_32,story.read::1_32,story.hear::1_34,story.echo::1_34,story.read::1_34,story.hear::1_35,story.echo::1_35,story.read::1_35,story.hear::1_36,story.echo::1_36,story.read::1_36,story.hear::1_37,story.echo::1_37,story.read::1_37 +story.hear::1_1,story.echo::1_1,story.read::1_1,story.hear::1_2,story.echo::1_2,story.read::1_2,story.hear::1 _3,story.echo::1_3,story.read::1_3,story.hear::1_11,story.echo::1_11,story.read::1_11,story.hear::1_12,story.echo::1_12,story.read::1_12,story.hear::1_13,story.echo::1_13,story.read::1_13,story.hear::1_14,story.echo::1_14,story.read::1_14,story.hear::1_15,story.echo::1_15,story.read::1_15,story.hear::1_16,story.echo::1_16,story.read::1_16,story.hear::1_17,story.echo::1_17,story.read::1_17,story.hear::1_18,story.echo::1_18,story.read::1_18,story.hear::1_19,story.echo::1_19,story.read::1_19,story.hear::1_20,story.echo::1_20,story.read::1_20,story.hear::1_21,story.echo::1_21,story.read::1_21,story.hear::1_22,story.echo::1_22,story.read::1_22,story.hear::1_23,story.echo::1_23,story.read::1_23,story.hear::1_24,story.echo::1_24,story.read::1_24,story.hear::1_25,story.echo::1_25,story.read::1_25,story.hear::1_26,story.echo::1_26,story.read::1_26,story.hear::1_27,story.echo::1_27,story.read::1_27,story.hear::1_28,story.echo::1_28,story.read::1_28,story.hear::1_29,story.echo::1_29,story.read::1_29,story.hear::1_30,story.echo::1_30,story.read::1_30,story.hear::1_31,story.echo::1_31,story.read::1_31,story.hear::1_32,story.echo::1_32,story.read::1_32,story.hear::1_34,story.echo::1_34,story.read::1_34,story.hear::1_35,story.echo::1_35,story.read::1_35,story.hear::1_36,story.echo::1_36,story.read::1_36,story.hear::1_37,story.echo::1_37,story.read::1_37 story.hear::2_1,story.echo::2_1,story.read::2_1,story.hear::2_2,story.echo::2_2,story.read::2_2,story.hear::2_3,story.echo::2_3,story.read::2_3,story.hear::2_4,story.echo::2_4,story.read::2_4,story.hear::2_5,story.echo::2_5,story.read::2_5,story.hear::2_6,story.echo::2_6,story.read::2_6,story.hear::2_7,story.echo::2_7,story.read::2_7,story.hear::2_8,story.echo::2_8,story.read::2_8,story.hear::2_9,story.echo::2_9,story.read::2_9,story.hear::2_10,story.echo::2_10,story.read::2_10,story.hear::2_11,story.echo::2_11,story.read::2_11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/build.gradle b/build.gradle index 485a0a93a..55761a38a 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ allprojects { rtCompileSdkVersion=26 //Integer rtBuildToolsVersion="26.0.1" //String - rtMinSdkVersion=19 + rtMinSdkVersion=21 // Note that using target version 22 bypasses the new run-time permissions found // in Marshmallow 23 diff --git a/chiragk.keystore b/chiragk.keystore new file mode 100644 index 000000000..67fce589e Binary files /dev/null and b/chiragk.keystore differ diff --git a/comp_bigmath/build.gradle b/comp_bigmath/build.gradle index 1c5fd3329..44b32ce8c 100644 --- a/comp_bigmath/build.gradle +++ b/comp_bigmath/build.gradle @@ -32,11 +32,11 @@ dependencies { exclude group: 'com.android.support', module: 'support-annotations' }) - compile 'com.android.support.constraint:constraint-layout:1.1.2' + api 'com.android.support.constraint:constraint-layout:1.1.2' - compile project(path: ':util') - compile project(path: ':comp_logging') - compile project(path: ':resources') - compile project(path: ':comp_writebox') + api project(path: ':util') + api project(path: ':comp_logging') + api project(path: ':resources') + api project(path: ':comp_writebox') } diff --git a/comp_listener/build.gradle b/comp_listener/build.gradle index 10a0fb85c..9a10ca896 100644 --- a/comp_listener/build.gradle +++ b/comp_listener/build.gradle @@ -31,11 +31,21 @@ android { // if true, only report errors ignoreWarnings true } + +} + +repositories { + maven { + url "https://jitpack.io" + } } dependencies { api fileTree(include: ['*.jar'], dir: 'libs') testImplementation 'junit:junit:4.12' + api('com.github.adrielcafe:AndroidAudioConverter:0.0.8') + implementation 'com.google.code.gson:gson:2.8.5' + api files('libs/jump3rcode.jar') implementation 'com.android.support:appcompat-v7:25.2.0' api project(':util') api files('libs/pocketsphinx-android-5prealpha-nolib.jar') diff --git a/comp_listener/libs/jump3rcode.jar b/comp_listener/libs/jump3rcode.jar new file mode 100644 index 000000000..cc6d1a364 Binary files /dev/null and b/comp_listener/libs/jump3rcode.jar differ diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java new file mode 100644 index 000000000..09d20ce7d --- /dev/null +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java @@ -0,0 +1,188 @@ +package edu.cmu.xprize.listener; + +import android.util.Log; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.Arrays; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * audioData stores the metadata associated with audio (storydata.json stuff) + * if you are looking for actual audio output, see AudioWriter + */ +public class AudioDataStorage { + + public static ArrayList segmentation = new ArrayList(); + static JSONObject storyData; + + static long decoderStartTime; + static long writerStartTime; + + static FileOutputStream outputStream; + static FileChannel outChannel; + + public static boolean contentCreationOn; + + public static void initStoryData(String jsonData) { + try { + storyData = new JSONObject(jsonData); + } catch (Exception e) { + Log.wtf("AudioDataStorage", "Could not load storydata"); + } + } + + public static synchronized void clearAudioData() { + Log.d("AudioDataStorage", "AudioData Cleared"); + // ? chirag what did you write here + } + + public static JSONObject saveAudioData(String fileName, String assetLocation, int currLine, int currPara, int currPage, String sentenceWPunc, int currUtt, List seg) { + // Todo: optimize this code (the process is being Duplicated) + // where? + Log.d("ADSSave", "attempting to save audiodata."); + + String completeFilePath = assetLocation + fileName + ".mp3"; + + Log.d("ADSSave", completeFilePath); + + // write segmentation to .seg file + try { + FileOutputStream os = new FileOutputStream(assetLocation + "/" + fileName.toLowerCase().replace(" ", "_") + ".seg"); + StringBuilder segData = new StringBuilder(""); + + Log.d("AudioDataStorage", "FileOutputStream Created at " + assetLocation + "/" + fileName + ".seg"); + int i = fileName.split(" ").length; + for(ListenerBase.HeardWord word: seg) { + if (i >= 0) { + segData.append(word.hypWord.toLowerCase() + "\t" + word.startFrame + "\t" + word.endFrame); + segData.append("\n"); + } + i--; + } + segData.deleteCharAt(segData.lastIndexOf("\n")); + + byte[] segBytes = segData.toString().getBytes(); + os.write(segBytes); + os.close(); + + // AudioWriter.truncateNarration(completeFilePath, seg.get(seg.size() - 1).endFrame); + } catch(IOException | NullPointerException e) { + Log.wtf("AudioDataStorage", "Failed to write segmentation!"); + Log.d("SegmentationFail", Log.getStackTraceString(e)); + } + + Log.d("AudioDataStorage", "About to update storydata.json"); + + // Update Storydata.json + try { + + JSONObject rawData = storyData + .getJSONArray("data") + .getJSONObject(currPage) + .getJSONArray("text") + .getJSONArray(currPara) + .getJSONObject(currLine); + + JSONObject rawNarration; + JSONArray narrationArray; + try { + narrationArray = rawData + .getJSONArray("narration"); + } catch (JSONException e) { + narrationArray = new JSONArray(); + rawData.put("narration", narrationArray); + } + + rawNarration = new JSONObject(); + + JSONArray segm = new JSONArray(); + long finalEndTime = 0; + int i = fileName.split(" ").length; + for(ListenerBase.HeardWord heardWord : seg) { + if (i >= 0) { + /* + if (i > 1) { + if (seg.get(i).iSentenceWord != heardWord.iSentenceWord - 1) + break; + } + */ + JSONObject segObj = new JSONObject(); + segObj.put("end", heardWord.endFrame); + segObj.put("start", heardWord.startFrame); + segObj.put("word", heardWord.hypWord.toLowerCase()); + segm.put(segObj); + finalEndTime = heardWord.endFrame; + i++; + } + i--; + } + rawNarration.put("segmentation", segm); + rawNarration.put("from", seg.get(0).startFrame); + rawNarration.put("audio", fileName.toLowerCase().replace(" ", "_") + ".mp3"); + rawNarration.put("until", finalEndTime); + rawNarration.put("utterances", fileName.toLowerCase()); + + + narrationArray.put(rawNarration); + + FileOutputStream outJson = new FileOutputStream(assetLocation + "/storydata.json"); + Log.d("AudioDataStorage", "writing out audio to " + assetLocation + "/storydata.json"); + + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + + String storyString = gson.toJson( + new JsonParser().parse(storyData.toString()).getAsJsonObject()) + .replace("\"nameValuePairs\": ", "") + .replace("\"values\": ", ""); + + outJson.write(storyString.getBytes()); + outJson.close(); + + Log.d("AudioDataStorage", "CHIRAG, Storydata.json right here: " + storyData.toString()); + + return storyData + .getJSONArray("data") + .getJSONObject(currPage); + } catch(JSONException e) { + Log.wtf("ADSStoryData", "Failed to update storyData!"); + Log.d("StoryDataFail", Log.getStackTraceString(e)); + + return null; + } catch(IOException e) { + Log.d("ADSStoryDataFail", "Was not able to open file"); + + return null; + } + + } + + public static void updateHypothesis(ListenerBase.HeardWord[] heardWords) { + Log.d("AudioDataStorageHyp", "Hypothesis Updated"); + segmentation.clear(); + if (heardWords != null) { + segmentation.addAll(Arrays.asList(heardWords)); + } + + } + + static void setSampleRate(int samplerate) { + // samplerate is always 16000 from my experience + } + + public static void saveSegmentation() { + + } + + +} diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioWriter.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioWriter.java new file mode 100644 index 000000000..7ef87beba --- /dev/null +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioWriter.java @@ -0,0 +1,377 @@ +package edu.cmu.xprize.listener; + +import android.app.Activity; +import android.content.Context; +import android.media.AudioRecord; +import android.provider.MediaStore; +import android.util.Log; + +import com.github.hiteshsondhi88.libffmpeg.ExecuteBinaryResponseHandler; +import com.github.hiteshsondhi88.libffmpeg.FFmpeg; +import com.github.hiteshsondhi88.libffmpeg.exceptions.FFmpegCommandAlreadyRunningException; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.nio.file.Files; + +import cafe.adriel.androidaudioconverter.AndroidAudioConverter; +import cafe.adriel.androidaudioconverter.callback.IConvertCallback; +import cafe.adriel.androidaudioconverter.callback.ILoadCallback; +import cafe.adriel.androidaudioconverter.model.AudioFormat; + + + +import de.sciss.jump3r.Mp3Encoder; + +public class AudioWriter { + + public static volatile short[] audioData = new short[160 * 60 * 100]; + public static volatile OutputStream outputStream; + public static volatile FileChannel fileChannel; + public static volatile BufferedOutputStream bufferedOutputStream; + public static volatile DataOutputStream dataOutputStream; + + public static String audioFileName; + public static String audioAssetLocation; + public static String completeFilePath; + + public static int dataLen; + + static byte[] savedFileData; + public static String current_log_location; + + public static Activity activity; + + public static Context context; + + public static void closeStreams() throws IOException, NullPointerException { + + outputStream.close(); + bufferedOutputStream.close(); + dataOutputStream.close(); + } + + // AudioWriter is updated with a new filepath + // todo: stop using exceptions as logic + public static void initializePath(String fileName, String assetLocation) { + Log.d("AudioWriter", "initializing to save narration capture"); + dataLen = 0; + // Close up old file + try { + closeStreams(); + } catch (IOException | NullPointerException ignored) {} + + + // Prepare new file + audioFileName = fileName.toLowerCase().replace(" ", "_"); + audioAssetLocation = assetLocation; + completeFilePath = assetLocation + audioFileName + ".wav"; + + // Just in case the new file is already existing, save its contents + try { + File inFile = new File(completeFilePath); + FileInputStream input = new FileInputStream(inFile); + savedFileData = new byte[(int) inFile.length()]; + BufferedInputStream bis =new BufferedInputStream(input); + bis.read(savedFileData,0, savedFileData.length); + bis.close(); + input.close(); + } catch (IOException ignored) {} + + try { + outputStream = new FileOutputStream(completeFilePath); + bufferedOutputStream = new BufferedOutputStream(outputStream); + dataOutputStream = new DataOutputStream(bufferedOutputStream); + + for(int i = 0; i < 44; i++) { + dataOutputStream.writeByte(0); + } + } catch (IOException e) { + Log.wtf("AudioWriterFail", Log.getStackTraceString(e)); + } + + AndroidAudioConverter.load(activity, new ILoadCallback() { + @Override + public void onSuccess() { + Log.d("AudioWriter" , "Success in loading FFMPEG"); + } + + @Override + public void onFailure(Exception error) { + Log.d("AudioWriter", "failed to load FFMPEG \n" + Log.getStackTraceString(error)); + } + }); + } + + public static void addAudio(int noOfShorts, short[] dataBuffer) { + try { + ByteBuffer bytes = ByteBuffer.allocate(noOfShorts * 2); + bytes.order(ByteOrder.LITTLE_ENDIAN); + + for(int i = 0; i < noOfShorts; i++) { + bytes.putShort(dataBuffer[i]); + } + + dataOutputStream.write(bytes.array()); + dataOutputStream.flush(); + dataLen += noOfShorts; + // Log.d("AudioWriter", "Just wrote " + noOfShorts + " shorts to the audio file!"); + } catch (IOException e) { + if(!e.getMessage().equals("Stream Closed")) { + Log.wtf("AudioWriterFail", Log.getStackTraceString(e)); + } + } catch (NullPointerException e) { + Log.d("AudioWriter", "Not writing data because stream does not exist"); + } + } + + public static void destroyContent() { + //initializePath(audioFileName, audioAssetLocation); + } + + public static void addHeader(RandomAccessFile raf) throws IOException { + long dataLen = raf.length() - 8; + int sampleRate = 16000; + int channelNumber = 1; + int bitRate = sampleRate * channelNumber * 16; + long trimmedLen = dataLen - 36; + + byte[] header = new byte[44]; + + header[0] = 'R'; + header[1] = 'I'; + header[2] = 'F'; + header[3] = 'F'; + header[4] = (byte) (dataLen & 0xff); + header[5] = (byte) ((dataLen >> 8) & 0xff); + header[6] = (byte) ((dataLen >> 16) & 0xff); + header[7] = (byte) ((dataLen >> 24) & 0xff); + header[8] = 'W'; + header[9] = 'A'; + header[10] = 'V'; + header[11] = 'E'; + header[12] = 'f'; + header[13] = 'm'; + header[14] = 't'; + header[15] = ' '; + header[16] = (byte) 16; + header[17] = 0; + header[18] = 0; + header[19] = 0; + header[20] = 1; + header[21] = 0; + header[22] = (byte) 1; + header[23] = 0; + header[24] = (byte) (sampleRate & 0xff); + header[25] = (byte) ((sampleRate >> 8) & 0xff); + header[26] = (byte) ((sampleRate >> 16) & 0xff); + header[27] = (byte) ((sampleRate >> 24) & 0xff); + header[28] = (byte) ((bitRate / 8) & 0xff); + header[29] = (byte) (((bitRate / 8) >> 8) & 0xff); + header[30] = (byte) (((bitRate / 8) >> 16) & 0xff); + header[31] = (byte) (((bitRate / 8) >> 24) & 0xff); + header[32] = (byte) ((channelNumber * 16) / 8); + header[33] = 0; + header[34] = 16; + header[35] = 0; + header[36] = 'd'; + header[37] = 'a'; + header[38] = 't'; + header[39] = 'a'; + header[40] = (byte) ((trimmedLen * 2) & 0xff); + header[41] = (byte) (((trimmedLen * 2) >> 8) & 0xff); + header[42] = (byte) (((trimmedLen * 2) >> 16) & 0xff); + header[43] = (byte) (((trimmedLen * 2) >> 24) & 0xff); + + raf.seek(0); + + raf.write(header, 0, 44); + + raf.seek(raf.length()); + raf.close(); + } + + // When the sentence is not being recorded, put the original file contents back + public static void abortOperation() { + try { + closeStreams(); + + // rewrite old contents of the file + FileOutputStream output = new FileOutputStream(completeFilePath); + output.write(savedFileData); + } catch (IOException ignored) { + + } catch (NullPointerException ignored) {// this is thrown if file doesn't exist + } + + } + + /** + * File is saved and put away as an mp3. the viewmanager has now moved to hear mode + */ + public static void pauseRecording() { + dataLen = 0; + // Close up old file + try { + closeStreams(); + } catch (IOException ignored) {} + + //Reopen to add header to the beginning of the file + try { + addHeader(new RandomAccessFile(completeFilePath, "rws")); + convertToMp3(); + } catch (Exception e) { + Log.wtf("AudioWriter", "Could not write header to wav file." + Log.getStackTraceString(e)); + } + } + + private static void convertToMp3() { + try { + Thread.sleep(10); + }catch (InterruptedException e) { + Log.wtf("AudioWriter", Log.getStackTraceString(e)); + } + + File wavFile = new File(completeFilePath); + + Log.d("AudioWriter1", "AndroidAudioConverter loaded: " + AndroidAudioConverter.isLoaded()); + + IConvertCallback callback = new IConvertCallback() { + @Override + public void onSuccess(File convertedFile) { + Log.d("AudioWriter", "successfully wrote mp3: " + convertedFile.getAbsolutePath()); + } + + @Override + public void onFailure(Exception error) { + + Log.d("AudioWriter", "Failed to write mp3 \n" + Log.getStackTraceString(error)); + // Log.getStackTraceString(error); + } + }; + + + AndroidAudioConverter.with(activity) + .setFile(wavFile) + .setFormat(AudioFormat.MP3) + .setCallback(callback) + .convert(); + Log.d("AudioWriter", "Successfully created mp3"); + + } + + void saveUtterance(int centiseconds, int noOfSentenceWords) { + + } + + /** + * + * @param recordingName is the name of recording to keep (with no filetype ending) + */ + public static void endUtterance(String recordingName) { + try { + closeStreams(); + } catch (IOException e) { + Log.wtf("AudioWriter", "attempt to end utterance but no audio input recorded. outputstreams were not open"); + Log.getStackTraceString(e); + return; + } catch (NullPointerException ignored) {} + + try { + addHeader(new RandomAccessFile(completeFilePath, "rws")); + } catch (IOException e) { + Log.wtf("AudioWriter", "Unable to add wav header to file: " + completeFilePath); + Log.getStackTraceString(e); + return; + } + + try { + File unTruncated = new File(completeFilePath); + + audioFileName = recordingName; + completeFilePath = audioAssetLocation + audioFileName + ".wav"; + + boolean success = unTruncated.renameTo(new File(completeFilePath)); + + if(success) { + convertToMp3(); + } else { + throw new IOException("unable to rename recording"); + } + } catch (IOException e) { + Log.wtf("AudioWriter", "Unable to rename file to save partial recording (utterance)"); + Log.getStackTraceString(e); + } + } + + public static boolean renameFile(String prevName, String newName, String assetLocation) { + String previousName = prevName.toLowerCase().replace(" ", "_"); + String newNameFixed = newName.toLowerCase().replace(" ", "_"); + //File oldWavFile = new File(assetLocation + previousName + ".wav"); + //File newWavFile = new File(assetLocation + newNameFixed + ".wav"); + //oldWavFile.renameTo(newWavFile); + + File oldMP3File = new File(assetLocation + previousName + ".mp3"); + File newMP3File = new File(assetLocation + newNameFixed + ".mp3"); + boolean renamed = oldMP3File.renameTo(newMP3File); + + // if (newMP3File.exists() /* && newWavFile.exists() */ ) { + if (renamed) { + return true; + } else { + Log.wtf("AudioWriter", "File Rename Failed:" + previousName + " to " + newNameFixed); + + return false; + } + } + + + public static void pauseNRename(String prevName, String newName, String assetLocation) { + pauseRecording(); + + try { + Thread.sleep(1000); + } catch (Exception e) { + Log.getStackTraceString(e); + } + renameFile(prevName, newName, assetLocation); + } + + public static void truncateNarration(String name, long length) { + FFmpeg ffmpeg = FFmpeg.getInstance(context); + String[] cmd = new String[]{"-t", Long.toString(length), "-i", name, "-acodec", "copy", name}; + try { + // to execute "ffmpeg -version" command you just need to pass "-version" + ffmpeg.execute(cmd, new ExecuteBinaryResponseHandler() { + + @Override + public void onStart() {} + + @Override + public void onProgress(String message) {} + + @Override + public void onFailure(String message) {} + + @Override + public void onSuccess(String message) {} + + @Override + public void onFinish() {} + }); + } catch (FFmpegCommandAlreadyRunningException e) { + Log.wtf("AudioWriter", e); + } + } +} diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/IAsrEventListener.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/IAsrEventListener.java index 2e85e33af..b92d8582a 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/IAsrEventListener.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/IAsrEventListener.java @@ -1,6 +1,10 @@ package edu.cmu.xprize.listener; +import java.util.List; + +import edu.cmu.pocketsphinx.Segment; + public interface IAsrEventListener { /** @@ -26,5 +30,6 @@ public interface IAsrEventListener { void onASREvent(int eventType); + } diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerBase.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerBase.java index 6169ecfb0..79bf0d493 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerBase.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerBase.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.lang.reflect.Field; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -55,7 +56,7 @@ public class ListenerBase { static protected ListenerAssets assets; // created in init phase - protected String captureLabel = ""; // label for capture, logging files - protected boolean IS_LOGGING = false; + protected boolean IS_LOGGING = true; protected File configFile; // config file to use, null => default protected File modelsDir; // saved model directory @@ -83,6 +84,9 @@ public class ListenerBase { // Note: on Android these are case sensitive filenames // static private HashMap dictMap = new HashMap(); + public ArrayList allSegments; + public long offsetTime; + static { dictMap.put("LANG_EN", "CMU07A-CAPS.DIC"); @@ -139,6 +143,7 @@ public void setLanguage(String langFTR) { // initialize recognizer for our task // + setupRecognizer(assets.getExternalDir(), configFile, dictMap.get(langFTR)); } @@ -231,8 +236,9 @@ protected void setupRecognizer(File assetsDir, File configFile, String langDicti .setFloat("-pip", 1f) - .setBoolean("-remove_noise", true) // yes in default - .setBoolean("-remove_silence", true) // yes in default + .setBoolean("-remove_noise", false) // yes in default + // I HAVE SET remove_noise AND remove_noise TO FALSE + .setBoolean("-remove_silence", false) // yes in default .setFloat("-silprob", 1f) // 0.005 in default .setInteger("-topn", 4) @@ -563,8 +569,6 @@ public static String[] textToWords(String text) { /***** Logging */ - - /** * get the path to the hypothesis log file for given utterance label */ @@ -613,4 +617,5 @@ protected void logHyp(String timestamp, String hyp, List segments, Hear } + } diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerPLRT.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerPLRT.java index 497b9e6b6..68bbbe2f1 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerPLRT.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerPLRT.java @@ -69,6 +69,7 @@ public class ListenerPLRT extends ListenerBase { private boolean useTruncations = true; // Flag whether or not to use truncations. private boolean speaking = false; // speaking state. [currently unused] + /** * Attach event listener to receive notification callbacks */ @@ -83,6 +84,7 @@ public void updateNextWordIndex(int next) { * @param wordsToHear -- array of upper-case ASR dictionary words * @param startWord -- 0-based index of word to expect next */ + @Override public void listenFor(String[] wordsToHear, int startWord) { Log.d("STABLE", "ListenFor: " + TextUtils.join(" ", wordsToHear)); @@ -117,6 +119,7 @@ public void listenFor(String[] wordsToHear, int startWord) { // save stream offset of start of utterance, for converting stream-based frame times // to utterance-based times. sentenceStartSamples = recognizer.nSamples; + // record start time now sentenceStartTime = System.currentTimeMillis(); @@ -475,11 +478,15 @@ private void processHypothesis(String[] asrWords, Boolean finalResult) { // if (eventListener != null) { eventListener.onUpdate(heardWords, finalResult); + allSegments = segments; + offsetTime = sentenceStartSamples / SAMPLES_PER_FRAME; } // log the partial hypothesis - if(IS_LOGGING) + if(IS_LOGGING) { logHyp(timestamp, TextUtils.join(" ", asrWords), segments, heardWords); + } + AudioDataStorage.updateHypothesis(heardWords); } } diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java index 46375c7bb..398b8d27c 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java @@ -43,15 +43,19 @@ import android.text.TextUtils; import android.util.Log; +import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; +import java.util.Date; import java.util.HashSet; import cmu.xprize.util.TCONST; @@ -453,7 +457,7 @@ public void run() { try { recorder = new AudioRecord( - AudioSource.VOICE_RECOGNITION, sampleRate, + AudioSource.MIC, sampleRate, // switched VOICE_RECOGNITION to MIC to possibly get better ASR results -- Chirag, 9-23-20 AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, 8192); } @@ -461,6 +465,7 @@ public void run() { Log.d("ASR", "AudioRecorder Create Failed: " + e); } isRunningRecognizer = true; + AudioDataStorage.setSampleRate(sampleRate); // Collect audio samples continuously while not paused and until the // Thread is killed. This allow UI/UX activity while the listener is still @@ -577,6 +582,8 @@ public void run() { decoder.processRaw(buffer, nread, false, false); //Log.d("ASR", "Time in processRaw: " + (System.currentTimeMillis() - ASRTimer)); + AudioWriter.addAudio(nread, buffer); + nSamples += nread; // InSpeech is true whenever there is a signal heard at the mic above threshold @@ -720,6 +727,25 @@ private void publishStableHypothesis(Hypothesis hypothesis) { Log.d("STABLE", "HYP LIST: " + hypString); + /* try { + + SimpleDateFormat formatter= new SimpleDateFormat("yyyy-MM-dd-HH:mm:ss"); + String date = formatter.format(new Date(System.currentTimeMillis())); + + FileWriter writer = new FileWriter(AudioWriter.current_log_location, true); + BufferedWriter bufferedWriter = new BufferedWriter(writer); + //Log.d("Speechrecognizer", "Writing to log file: " + AudioWriter.current_log_location); + bufferedWriter.write(date + " HYP LIST: " + hypString); + writer.close(); + bufferedWriter.close(); + } catch (IOException e) { + Log.getStackTraceString(e); + } catch (Exception e) { + Log.d("SpeechRecognizer", "No log file"); + } + + */ + String[] asrWords = hypString.split("\\s+"); @@ -848,7 +874,7 @@ public static void convertRawToWav(File rawFile, File wavFile) { output = new FileOutputStream(wavFile); // first write appropriate wave file header ByteArrayOutputStream hdrBytes = new ByteArrayOutputStream(); - new WaveHeader(WaveHeader.FORMAT_PCM, (short) 1, 16000, (short) 16, (int) rawFile.length()).write(hdrBytes); + new WaveHeader(WaveHeader.FORMAT_PCM, (short) 1, 16000, (short) 16, (int) rawFile.length()).write(hdrBytes); output.write(hdrBytes.toByteArray()); // then copy raw bytes to output file byte[] buffer = new byte[4096]; diff --git a/comp_questions/src/main/java/cmu/xprize/comp_questions/CQn_Component.java b/comp_questions/src/main/java/cmu/xprize/comp_questions/CQn_Component.java index 23f36d24f..91629a889 100644 --- a/comp_questions/src/main/java/cmu/xprize/comp_questions/CQn_Component.java +++ b/comp_questions/src/main/java/cmu/xprize/comp_questions/CQn_Component.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import cmu.xprize.comp_logging.CErrorManager; @@ -50,6 +51,7 @@ import cmu.xprize.util.JSON_Helper; import cmu.xprize.util.TCONST; import cmu.xprize.util.TTSsynthesizer; +import edu.cmu.pocketsphinx.Segment; import edu.cmu.xprize.listener.IAsrEventListener; import edu.cmu.xprize.listener.ListenerBase; import edu.cmu.xprize.listener.ListenerPLRT; diff --git a/comp_questions/src/main/res/layout/asb_evenpage.xml b/comp_questions/src/main/res/layout/asb_evenpage.xml index 54774b8de..bb4a6d28d 100644 --- a/comp_questions/src/main/res/layout/asb_evenpage.xml +++ b/comp_questions/src/main/res/layout/asb_evenpage.xml @@ -7,6 +7,21 @@ android:layout_height="match_parent" android:background="#FFF"> + + - + android:layout_marginRight="30dp" > + - + \ No newline at end of file diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CASB_Narration.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CASB_Narration.java index b2d6612fe..30e6a29a8 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CASB_Narration.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CASB_Narration.java @@ -40,4 +40,13 @@ public void loadJSON(JSONObject jsonObj, IScope scope) { JSON_Helper.parseSelf(jsonObj, this, CClassMap.classMap, scope); } + CASB_Narration(String audio, int from, int until, String utterances, CASB_Seg[] segmentation) { + this.audio = audio; + this.from = from; + this.until = until; + this.utterances = utterances; + this.segmentation = segmentation; + } + + public CASB_Narration() {} } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CASB_Seg.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CASB_Seg.java index ffced5f17..d8db9b021 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CASB_Seg.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CASB_Seg.java @@ -37,4 +37,13 @@ public void loadJSON(JSONObject jsonObj, IScope scope) { JSON_Helper.parseSelf(jsonObj, this, CClassMap.classMap, scope); } + public CASB_Seg() { + + } + + CASB_Seg(int start, int end, String word) { + this.start = start; + this.end = end; + this.word = word; + } } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_Component.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_Component.java index 3709d336c..fe391c9d2 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_Component.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_Component.java @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import cmu.xprize.comp_logging.CErrorManager; @@ -48,6 +49,8 @@ import cmu.xprize.util.JSON_Helper; import cmu.xprize.util.TTSsynthesizer; import cmu.xprize.util.TCONST; +import edu.cmu.pocketsphinx.Segment; +import edu.cmu.xprize.listener.AudioDataStorage; import edu.cmu.xprize.listener.IAsrEventListener; import edu.cmu.xprize.listener.ListenerBase; import edu.cmu.xprize.listener.ListenerPLRT; @@ -109,8 +112,6 @@ public class CRt_Component extends ViewAnimator implements IEventListener, IVMan // public CData_Index[] dataSource; - - // This is used to map "type" (class names) in the index to real classes // static public HashMap viewClassMap = new HashMap(); @@ -533,6 +534,7 @@ public void setPageFlipButton(String command) { mViewManager.setPageFlipButton(command); } + // Tutor methods End //************************************************************************ //************************************************************************ @@ -581,6 +583,7 @@ public void loadStory(String EXTERNPATH, String viewType, String assetLocation, // ZZZ it loads the story data JUST FINE String jsonData = JSON_Helper.cacheDataByName(EXTERNPATH + TCONST.STORYDATA); + AudioDataStorage.initStoryData(jsonData); Log.d(TCONST.DEBUG_STORY_TAG, "logging jsonData:"); mViewManager.loadJSON(new JSONObject(jsonData), null); @@ -603,6 +606,44 @@ public void loadStory(String EXTERNPATH, String viewType, String assetLocation, } + public void loadStoryANDEnableContentCreation(String EXTERNPATH, String viewType, String assetLocation, Boolean narrationCaptureMode, boolean keepOnlyRelevantAudio) { + + Log.d(TCONST.DEBUG_STORY_TAG, String.format("assetLocation=%s -- EXTERNPATH=%s", assetLocation, EXTERNPATH)); + + Class storyClass = viewClassMap.get(viewType); + + try { + // Generate the View manager for the storyName - specified in the data + // + // ooooh maybe check if it's math and make text closer to image + mViewManager = (ICRt_ViewManager)storyClass.getConstructor(new Class[]{CRt_Component.class, ListenerBase.class}).newInstance(this,mListener); + + // ZZZ it loads the story data JUST FINE + String jsonData = JSON_Helper.cacheDataByName(EXTERNPATH + TCONST.STORYDATA); + AudioDataStorage.initStoryData(jsonData); + Log.d(TCONST.DEBUG_STORY_TAG, "logging jsonData:"); + + mViewManager.loadJSON(new JSONObject(jsonData), null); + + } catch (Exception e) { + // TODO: Manage Exceptions + CErrorManager.logEvent(TAG, "Story Parse Error: ", e, false); + } + + if (assetLocation.equals(TCONST.EXTERN_SHARED)) { + Log.d(TCONST.DEBUG_STORY_TAG, "SHARED!"); + // we are done using sharedAssetLocation + EXTERNPATH = null; + } + + mViewManager.enableNarrationCaptureMode(narrationCaptureMode, keepOnlyRelevantAudio); + // + // ZZZ what are these values? + // ZZZ EXTERNPATH = TCONST.EXTERN + // ZZZ assetLocation contains storydata.json and images + mViewManager.initStory(this, EXTERNPATH, assetLocation); + + } /** * TODO: this currently only supports extern assets - need to allow for internal assets @@ -779,6 +820,7 @@ private void enQueue(Queue qCommand) { } private void enQueue(Queue qCommand, Long delay) { + if (!_qDisabled) { queueMap.put(qCommand, qCommand); @@ -837,4 +879,36 @@ public void loadJSON(JSONObject jsonData, IScope scope) { JSON_Helper.parseSelf(jsonData, this, CClassMap.classMap, scope); } + + /** + * Bypasses the normal json procedure to efficiently play narrations at runtime + * Overridden by TClass + * + * @param dataSource + * @param assetLocation + */ + public void updateJSONData(String dataSource, String assetLocation) { + + } + + public void stopAudio() {} + + public void wrongWordBehavior() { + + } + + public long firstWordTime; + + public void startLate() { + + } + + List getSegments() { + return mListener.allSegments; + } + + long getoffsetTime() { + return mListener.offsetTime; + } + } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java index 01209c052..b83c8b725 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java @@ -20,10 +20,14 @@ import android.content.Context; import android.graphics.BitmapFactory; +import android.graphics.Color; import android.graphics.PointF; import android.text.Html; import android.text.Layout; +import android.text.SpannableStringBuilder; +import android.text.Spanned; import android.text.TextUtils; +import android.text.style.BackgroundColorSpan; import android.util.Log; import android.view.View; import android.view.ViewGroup; @@ -33,11 +37,18 @@ import org.json.JSONObject; +import java.io.BufferedReader; +import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; +import java.util.List; import java.util.Locale; import cmu.xprize.util.CPersonaObservable; @@ -45,13 +56,15 @@ import cmu.xprize.util.IScope; import cmu.xprize.util.JSON_Helper; import cmu.xprize.util.TCONST; +import edu.cmu.pocketsphinx.Segment; +import edu.cmu.xprize.listener.AudioDataStorage; +import edu.cmu.xprize.listener.AudioWriter; import edu.cmu.xprize.listener.ListenerBase; import static cmu.xprize.util.TCONST.FTR_USER_READ; import static cmu.xprize.util.TCONST.FTR_USER_READING; import static cmu.xprize.util.TCONST.QGRAPH_MSG; - /** * This view manager provides student UX for the African Story Book format used * in the CMU XPrize submission @@ -140,6 +153,16 @@ public class CRt_ViewManagerASB implements ICRt_ViewManager, ILoadableObject { private ArrayList wordsSpoken; private ArrayList futureSpoken; + boolean alreadyNarrated; + boolean keepOnlyRelevantAudio; // true if NARRATION CAPTURE MODE is deleting all audio that is wrong, false if the entire file including mistakes is to be kept + boolean deleteRecording; // if there is a mistake in NARRATION CAPTURE MODE, then the current recording is deleted + + int currUtt; + ArrayList capturedUtt = new ArrayList<>(); + + boolean narrationTracking; + boolean silenceTimerRunning = false; + // json loadable // ZZZ where the money gets loaded @@ -162,6 +185,21 @@ public class CRt_ViewManagerASB implements ICRt_ViewManager, ILoadableObject { static final String TAG = "CRt_ViewManagerASB"; + public ImageButton backButton; + public ImageButton forwardButton; + + public String buttonState; + + String narrationFileName; + + boolean isUserNarrating; + ListenerBase.HeardWord[] allHeardWords; + final int SEGMENT_GAP_LENGTH = 100; // length of silence used to separate segments in narration capture mode + ArrayList seamIndices = new ArrayList<>(); // list of indices of seams for narration capture mode + + // Amount to pad the end time of the last word to avoid truncating it during playback + static final long END_UTTERANCE_DELAY_MS = 200; + /** * * @param parent @@ -216,8 +254,40 @@ public void initStory(IVManListener owner, String assetPath, String location) { //TODO: CHECK mParent.animatePageFlip(true,mCurrViewIndex); + + deleteRecording = true; + } + boolean isNarrationCaptureMode; + // NARRATION CAPTURE mode -- as in CONTENT CREATION and not NARRATING -- gets activated here + public void enableNarrationCaptureMode(boolean capturingNarration, boolean keepExtraAudio) { + if (capturingNarration) { + mParent.setFeature(TCONST.FTR_NARRATION_CAPTURE, TCONST.ADD_FEATURE); + Log.d("CRT_ViewManagerASB", "Narrate Mode has been activated through setNarrateMode()"); + isNarrationCaptureMode = true; + isUserNarrating = true; + + AudioDataStorage.contentCreationOn = true; + + SimpleDateFormat formatter= new SimpleDateFormat("yyyy-MM-dd-HH:mm:ss"); + String date = formatter.format(new Date(System.currentTimeMillis())); + String hyplogFile = TCONST.HYP_LOG_FILE_LOCATION + date + ".log"; + // Not working code below + /* try { + FileOutputStream fos = new FileOutputStream(hyplogFile); + fos.close(); + AudioWriter.current_log_location = hyplogFile; + } catch (IOException e) { + isNarrationCaptureMode = false; + Log.getStackTraceString(e); + } */ + } else { + isNarrationCaptureMode = false; + } + + keepOnlyRelevantAudio = keepExtraAudio; + } /** * NOTE: we reset mCurrWord - last parm in seekToStoryPosition @@ -246,6 +316,8 @@ public void startStory() { storyBooting = false; speakOrListen(); } + + Log.d("InitStory", "Story has initialized"); } @@ -253,7 +325,13 @@ public void speakOrListen() { if (hearRead.equals(TCONST.FTR_USER_HEAR)) { - mParent.applyBehavior(TCONST.NARRATE_STORY); + if(isNarrationCaptureMode) { + isUserNarrating = true; + endOfUtteranceCapture(); + } + if(!hearRead.equals(FTR_USER_READ)) { + mParent.applyBehavior(TCONST.NARRATE_STORY); + } } if (hearRead.equals(FTR_USER_READ)) { @@ -397,6 +475,8 @@ public void flipPage() { mPageFlip = (ImageButton) mOddPage.findViewById(R.id.SpageFlip); mSay = (ImageButton) mOddPage.findViewById(R.id.Sspeak); + + } else { mCurrViewIndex = mEvenIndex; @@ -629,6 +709,7 @@ private void seekToStoryPosition(int currPage, int currPara, int currLine, int c mParaCount = data[currPage].text.length; mLineCount = data[currPage].text[currPara].length; + // WARNING: referring to sentences as "lines" is dangerously misleading rawNarration = data[currPage].text[currPara][currLine].narration; rawSentence = data[currPage].text[currPara][currLine].sentence; if (data[currPage].prompt != null) page_prompt = data[currPage].prompt; @@ -639,9 +720,9 @@ private void seekToStoryPosition(int currPage, int currPara, int currLine, int c // so it must reflect the current sentence without punctuation! // // To keep indices into wordsToSpeak in sync with wordsToDisplay we break the words to - // display if they contain apostrophes or hyphens into sub "words" - e.g. "thing's" -> "thing" "'s" - // these are reconstructed by the highlight logic without adding spaces which it otherwise inserts - // automatically. + // // display if they contain apostrophes or hyphens into sub "words" - e.g. "thing's" -> "thing" "'s" + // // these are reconstructed by the highlight logic without adding spaces which it otherwise inserts + // // automatically. // wordsToDisplay = splitRawSentence(rawSentence); @@ -721,6 +802,9 @@ private void seekToStoryPosition(int currPage, int currPara, int currLine, int c // UpdateDisplay(); + if (isNarrationCaptureMode) + feedSentence(mCurrWord); + // Once past the storyName initialization stage - Listen for the target word - // if (!storyBooting) @@ -739,9 +823,14 @@ private void initSegmentation(int _uttNdx, int _segNdx) { segmentNdx = _segNdx; numSegments = segmentArray.length; utterancePrev = utteranceNdx == 0 ? 0 : rawNarration[utteranceNdx - 1].until; - segmentPrev = utterancePrev; + segmentPrev = rawNarration[utteranceNdx].from; // utterancePrev; + + mParent.firstWordTime = currUtterance.from; + - // Clean the extension off the end - could be either wav/mp3 + mParent.post(TCONST.STOP_AUDIO, (currUtterance.until * 10) + END_UTTERANCE_DELAY_MS); // absolute position in file (in ms) to stop playing + + // Clean the extension off the end - could be either wav/mp3 + // String filename = currUtterance.audio.toLowerCase(); @@ -753,6 +842,7 @@ private void initSegmentation(int _uttNdx, int _segNdx) { // mParent.publishValue(TCONST.RTC_VAR_UTTERANCE, filename); + // NOTE: Due to inconsistencies in the segmentation data, you cannot depend on it // having precise timing information. As a result the segment may timeout before the // audio has completed. To avoid this we use oncomplete in type_audio to push an @@ -765,6 +855,8 @@ private void initSegmentation(int _uttNdx, int _segNdx) { private void trackNarration(boolean start) { + narrationTracking = true; + if (start) { mHeardWord = 0; @@ -775,14 +867,18 @@ private void trackNarration(boolean start) { spokenWords = new ArrayList(); - // Tell the script to speak the new uttereance - // - mParent.applyBehavior(TCONST.SPEAK_UTTERANCE); + // Tell the script to speak the new uttereance + // when its not in narrate mode, that + // + mParent.applyBehavior(TCONST.SPEAK_UTTERANCE); + mParent.post(TCONST.START_LATER, 0L); postDelayedTracker(); } else { - // NOTE: The narration mode uses the ASR logic to simplify operation. In doing this + + + // NOTE: The narration mode uses the AS R logic to simplify operation. In doing this /// it uses the wordsToSpeak array to progressively highlight the on screen text based /// on the timing found in the segmentation data. // @@ -812,7 +908,7 @@ private void trackNarration(boolean start) { onUpdate(spokenWords.toArray(new String[spokenWords.size()])); // If the segment word is complete continue to the next segment - note that this is - // generally the case. Words are not usually split by pubctuation + // generally the case. Words are not usually split by punctuation // if (splitIndex >= splitSegment.length) { @@ -824,7 +920,7 @@ private void trackNarration(boolean start) { // Note the last segment is not timed. It is driven by the TRACK_COMPLETE event // from the audio mp3 playing. This is required as the segmentation data is not // sufficiently accurate to ensure we don't interrupt a playing utterance. - // + //track segmentNdx++; if (segmentNdx >= numSegments) { @@ -842,6 +938,7 @@ private void trackNarration(boolean start) { } else { endOfSentence = true; + narrationTracking = false; } } // All the segments except the last one are timed based on the segmentation data. @@ -865,7 +962,14 @@ private void postDelayedTracker() { narrationSegment = rawNarration[utteranceNdx].segmentation[segmentNdx]; - segmentCurr = utterancePrev + narrationSegment.end; + segmentCurr = narrationSegment.end; //utterancePrev + narrationSegment.end; + + // If last word of the utterance, add a small amount of delay due to ASR inaccuracy + if (segmentNdx == rawNarration[utteranceNdx].segmentation.length - 1) { + segmentCurr += END_UTTERANCE_DELAY_MS / 10; + } + + Log.d(TAG, "Posting delayed tracker: SegmentCurr: " + segmentCurr + " segmentPrev: " + segmentPrev + " Delay: " + (segmentCurr - segmentPrev)); mParent.post(TCONST.TRACK_NARRATION, new Long((segmentCurr - segmentPrev) * 10)); @@ -925,6 +1029,13 @@ public void execCommand(String command, Object target ) { mParent.nextNode(); break; + case TCONST.STOP_AUDIO: + mParent.stopAudio(); + break; + + case TCONST.START_LATER: + mParent.startLate(); + break; case TCONST.SPEAK_EVENT: case TCONST.UTTERANCE_COMPLETE_EVENT: @@ -945,10 +1056,12 @@ private void publishStateValues() { String cummulativeState = TCONST.RTC_CLEAR; - // ensure encho state has a valid value. + // ensure echo state has a valid value. // mParent.publishValue(TCONST.RTC_VAR_ECHOSTATE, TCONST.FALSE); mParent.publishValue(TCONST.RTC_VAR_PARROTSTATE, TCONST.FALSE); + mParent.publishValue(TCONST.RTC_VAR_NARRATESTATE, TCONST.FALSE); + mParent.publishValue(TCONST.RTC_VAR_NARRATECOMPLETESTATE, TCONST.FALSE); if (prompt != null) { mParent.publishValue(TCONST.RTC_VAR_PROMPT, prompt); @@ -964,41 +1077,46 @@ private void publishStateValues() { // if (mCurrWord >= mWordCount) { - // In echo mode - After line has been echoed we switch to Read mode and - // read the next sentence. - // - if (mParent.testFeature(TCONST.FTR_USER_ECHO) || mParent.testFeature(TCONST.FTR_USER_REVEAL) || mParent.testFeature(TCONST.FTR_USER_PARROT)) { + // In echo mode - After line has been echoed we switch to Read mode and + // read the next sentence. + // + if (mParent.testFeature(TCONST.FTR_USER_ECHO) || mParent.testFeature(TCONST.FTR_USER_REVEAL) || mParent.testFeature(TCONST.FTR_USER_PARROT)) { - // Read Mode - When user finishes reading switch to Narrate mode and - // narrate the same sentence - i.e. echo - // - if (hearRead.equals(FTR_USER_READ)) { + // Read Mode - When user finishes reading switch to Narrate mode and + // narrate the same sentence - i.e. echo + // + if (hearRead.equals(FTR_USER_READ)) { - if (!mParent.testFeature(TCONST.FTR_USER_PARROT)) mParent.publishValue(TCONST.RTC_VAR_ECHOSTATE, TCONST.TRUE); + if(mParent.testFeature(TCONST.FTR_NARRATION_CAPTURE)) { + mParent.publishValue(TCONST.RTC_VAR_NARRATECOMPLETESTATE, TCONST.TRUE); + } - hearRead = TCONST.FTR_USER_HEAR; - mParent.retractFeature(FTR_USER_READING); + if (!mParent.testFeature(TCONST.FTR_USER_PARROT)) + mParent.publishValue(TCONST.RTC_VAR_ECHOSTATE, TCONST.TRUE); - Log.d("ISREADING", "NO"); + hearRead = TCONST.FTR_USER_HEAR; + mParent.retractFeature(FTR_USER_READING); - cummulativeState = TCONST.RTC_LINECOMPLETE; - mParent.publishValue(TCONST.RTC_VAR_WORDSTATE, TCONST.LAST); + Log.d("ISREADING", "NO"); - mListener.setPauseListener(true); - } - // Narrate mode - swithc back to READ and set line complete flags - // - else { - hearRead = FTR_USER_READ; - mParent.publishFeature(FTR_USER_READING); + cummulativeState = TCONST.RTC_LINECOMPLETE; + mParent.publishValue(TCONST.RTC_VAR_WORDSTATE, TCONST.LAST); - if (mParent.testFeature(TCONST.FTR_USER_PARROT)) mParent.publishValue(TCONST.RTC_VAR_PARROTSTATE, TCONST.TRUE); + mListener.setPauseListener(true); + } + // Narrate mode - swithc back to READ and set line complete flags + // + else { + hearRead = FTR_USER_READ; + mParent.publishFeature(FTR_USER_READING); - Log.d("ISREADING", "YES"); + if (mParent.testFeature(TCONST.FTR_USER_PARROT)) mParent.publishValue(TCONST.RTC_VAR_PARROTSTATE, TCONST.TRUE); - cummulativeState = TCONST.RTC_LINECOMPLETE; - mParent.publishValue(TCONST.RTC_VAR_WORDSTATE, TCONST.LAST); - } + Log.d("ISREADING", "YES"); + + cummulativeState = TCONST.RTC_LINECOMPLETE; + mParent.publishValue(TCONST.RTC_VAR_WORDSTATE, TCONST.LAST); + } } else { cummulativeState = TCONST.RTC_LINECOMPLETE; mParent.publishValue(TCONST.RTC_VAR_WORDSTATE, TCONST.LAST); @@ -1028,6 +1146,7 @@ private void publishStateValues() { // Publish the cumulative state out to the scripting scope in the tutor // mParent.publishValue(TCONST.RTC_VAR_STATE, cummulativeState); + buttonState = cummulativeState; } @@ -1058,6 +1177,7 @@ public void nextPage() { // Actually do the page animation // mParent.animatePageFlip(true, mCurrViewIndex); + } @Override public void prevPage() { @@ -1087,6 +1207,99 @@ private void incPage(int direction) { // NOTE: we reset mCurrPara, mCurrLine and mCurrWord // seekToStoryPosition(mCurrPage, TCONST.ZERO, TCONST.ZERO, TCONST.ZERO); + updateButtonPositions(); + } + + private void updateButtonPositions() { + Log.d("CRt_ViewManager", "isNarrateMode is" + isNarrationCaptureMode); + if (mCurrPage % 2 == 0) { + backButton = (ImageButton) mOddPage.findViewById(R.id.backButton); + forwardButton = (ImageButton) mOddPage.findViewById(R.id.forwardButton); + } else { + backButton = (ImageButton) mEvenPage.findViewById(R.id.backButton); + forwardButton = (ImageButton) mEvenPage.findViewById(R.id.forwardButton); + } + + if (isNarrationCaptureMode) { + try { + backButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + Log.d(TAG, "Back Button Pressed"); + if (hearRead != null) { + if (hearRead.equals(TCONST.FTR_USER_READ)) { + AudioWriter.abortOperation(); + acceptedList.clear(); + /* + } else if (hearRead.equals(TCONST.FTR_USER_HEAR)) { + mParent.post(TCONST.STOP_AUDIO, new Long(currUtterance.until * 10)); + hearRead = TCONST.FTR_USER_HEAR; + } + + */ + + if (mCurrLine > 0) { + seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine - 1, 0); + } else if (mCurrPara > 0) { + seekToStoryPosition(mCurrPage, mCurrPara - 1, 0, 0); + } else if (mCurrPage > 0) { + Log.d(TAG, "mCurrPage: " + mCurrPage + " mCurrPara: " + mCurrPara + " mCurrLine: " + mCurrLine); + seekToStoryPosition(mCurrPage - 1, 0, 0, 0); + } + } + } + + + } + }); + + forwardButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + Log.d(TAG, "Skip Button Pressed"); + + Log.d(TAG, "hearRead: " + hearRead); + if (hearRead != null) { + if (hearRead.equals(TCONST.FTR_USER_READ)) { + AudioWriter.abortOperation(); + acceptedList.clear(); + /* + } else if (hearRead.equals(TCONST.FTR_USER_HEAR)) { + mParent.post(TCONST.STOP_AUDIO, new Long(currUtterance.until * 10)); + + } + + */ + if (mCurrLine < mLineCount - 1) { + seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine + 1, 0); + } else if (mCurrPara < mParaCount - 1) { + seekToStoryPosition(mCurrPage, mCurrPara + 1, 0, 0); + + } else if (mCurrPage < mPageCount - 1) { + seekToStoryPosition(mCurrPage + 1, 0, 0, 0); + + } + } + } + + UpdateDisplay(); + + + } + }); + + forwardButton.setVisibility(View.VISIBLE); + backButton.setVisibility(View.VISIBLE); + } catch (NullPointerException e) { + Log.d("CRt_ViewManagerASB", "Couldn't find forwardbutton and backbutton"); + } + } else { + // make buttons invisible + forwardButton.setVisibility(View.GONE); + backButton.setVisibility(View.GONE); + } + + } @@ -1113,6 +1326,7 @@ public void nextPara() { if (mCurrPara < mParaCount-1) { incPara(TCONST.INCR); } + } @Override @@ -1121,6 +1335,7 @@ public void prevPara() { if (mCurrPara > 0) { incPara(TCONST.DECR); } + } // NOTE: we reset mCurrLine and mCurrWord @@ -1157,6 +1372,7 @@ public void nextLine() { if (mCurrLine < mLineCount-1) { incLine(TCONST.INCR); } + } @Override public void prevLine() { @@ -1196,13 +1412,20 @@ private void incLine(int incr) { @Override public void echoLine() { + isUserNarrating = true; + + if (isNarrationCaptureMode) { + + // endOfUtteranceCapture(); + // sets data narration + } // reset the echo flag // mParent.publishValue(TCONST.RTC_VAR_ECHOSTATE, TCONST.FALSE); // Update the state vars // - seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, TCONST.ZERO); + seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, TCONST.ZERO); // } @@ -1307,7 +1530,7 @@ private void startListening() { // for the current target word. // 1. Start with the target word on the target sentence // 2. Add the words from there to the end of the sentence - just to permit them - // 3. Add the words alread spoken from the other lines - just to permit them + // 3. Add the words already spoken from the other lines - just to permit them // // "Permit them": So the language model is listening for them as possibilities. // @@ -1366,6 +1589,7 @@ public void setHighLight(String highlight, boolean update) { * Update the displayed sentence */ private void UpdateDisplay() { + Log.d(TAG, "Updating display"); if (showWords) { String fmtSentence = ""; @@ -1402,7 +1626,66 @@ private void UpdateDisplay() { if (showFutureContent) content += TCONST.SENTENCE_SPACE + futureSentencesFmtd; - mPageText.setText(Html.fromHtml(content)); + try { + if (isNarrationCaptureMode && hearRead.equals(TCONST.FTR_USER_HEAR)) { + try { + int progress = 0; + int segmentNum = 0; + for (int i = 0; i <= mCurrWord; i++) { + if (rawNarration[segmentNum].segmentation.length > progress) { + progress++; + } else { + segmentNum++; + progress = 1; + } + Log.d(TAG, "Highlighting Information -- Progress: " + progress + ", SegmentNum: " + segmentNum); + + } + + SpannableStringBuilder preHighlight = new SpannableStringBuilder(Html.fromHtml(fmtSentence)); + + int startIndex = 0; + int endIndex = 0; + int wordIndex = 0; + for (String word : wordsToDisplay) { + if (wordIndex < mCurrWord + 1 - progress) { + if (0 < wordIndex) + startIndex++; + startIndex += word.length(); + } + + if (wordIndex < mCurrWord + 1 - progress + rawNarration[segmentNum].segmentation.length) { + if (0 < wordIndex) + endIndex++; + endIndex += word.length(); + } + wordIndex++; + } + + // Don't highlight the space between seams to emphasize the difference + if (startIndex > 0) startIndex++; + + Log.d(TAG, "Highlighting Update -- StartIndex: " + startIndex + "EndIndex: " + endIndex); + + preHighlight.setSpan(new BackgroundColorSpan(Color.parseColor("#00FFFF")), startIndex, endIndex, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); + + fmtSentence = preHighlight.toString(); + + preHighlight.insert(0, Html.fromHtml(completedSentencesFmtd)); + preHighlight.append(TCONST.SENTENCE_SPACE + Html.fromHtml(futureSentencesFmtd)); + mPageText.setText(preHighlight); + } catch (Exception e) { + Log.d(TAG, "Highlighting Fault: " + Log.getStackTraceString(e)); + mPageText.setText(Html.fromHtml(content)); + } + } else { + + mPageText.setText(Html.fromHtml(content)); + } + } catch (NullPointerException e) { + mPageText.setText(Html.fromHtml(content)); + } + Log.d(TAG, "Story Sentence Text: " + content); } @@ -1473,7 +1756,7 @@ private PointF broadcastActiveTextPos(TextView text, String[] words){ /** - * This is where we process words being narrated + * This is where we process words being narrated (by the pre-made narration, not the live user) * VMC_QA why does this get called twice for the last word??? */ @Override @@ -1489,8 +1772,22 @@ public void onUpdate(String[] heardWords) { while (mHeardWord < heardWords.length) { - if (wordsToSpeak[mCurrWord].equals(heardWords[mHeardWord])) { // VMC_QA these are not equal. one of these is out of bounds (probably wordsToSpeak) + Log.d("CRt_ViewManager", "Heardwords length: " + heardWords.length + ". Current heardWord: " + mHeardWord + + "wordsToSpeak length: " + wordsToSpeak.length + "current word " + mCurrWord); + // todo: (chirag) figure out the root cause of the problem instead of this bandaid solution + boolean match; + try { + match = wordsToSpeak[mCurrWord].equals(heardWords[mHeardWord]); + } catch(ArrayIndexOutOfBoundsException e) { + match = true; + } + + + //if (wordsToSpeak[mCurrWord].equals(heardWords[mHeardWord])) { // VMC_QA these are not equal. one of these is out of bounds (probably wordsToSpeak) + // wordsToSpeak is in fact out of bounds here. It is 1 shorter than heardWords */ + // the above was commented out by Chirag in order to ensure that if heardwords is too long (because it often accidentally contains an extra word) + if(match) { nextWord(); mHeardWord++; @@ -1506,6 +1803,8 @@ public void onUpdate(String[] heardWords) { attemptNum = 0; result = true; } + + mParent.updateContext(rawSentence, mCurrLine, wordsToSpeak, mCurrWord - 1, heardWords[mHeardWord - 1], attemptNum, false, result); } @@ -1514,6 +1813,7 @@ public void onUpdate(String[] heardWords) { mParent.UpdateValue(result); } + //Timer silenceTimer = new Timer(); /** * This is where the incoming PLRT ASR data is processed. @@ -1529,7 +1829,10 @@ public void onUpdate(String[] heardWords) { */ @Override public void onUpdate(ListenerBase.HeardWord[] heardWords, boolean finalResult) { + //silenceTimer.cancel(); + allHeardWords = heardWords; + AudioDataStorage.updateHypothesis(heardWords); boolean result = true; String logString = ""; @@ -1544,6 +1847,15 @@ public void onUpdate(ListenerBase.HeardWord[] heardWords, boolean finalResult) { while ((mCurrWord < wordsToSpeak.length) && (mHeardWord < heardWords.length)) { + // wordIndex is the sentence index of the first heard word, and is used to deduce the position in the sentence in narration capture mode + int wordIndex = wordsToSpeak.length; + for (int i = 0; i < wordsToSpeak.length; i++) { + if(wordsToSpeak[i].equals(heardWords[mHeardWord].hypWord)) { + wordIndex = i; + break; + } + } + if (wordsToSpeak[mCurrWord].equals(heardWords[mHeardWord].hypWord)) { nextWord(); @@ -1556,6 +1868,39 @@ public void onUpdate(ListenerBase.HeardWord[] heardWords, boolean finalResult) { result = true; mParent.updateContext(rawSentence, mCurrLine, wordsToSpeak, mCurrWord - 1, heardWords[mHeardWord - 1].hypWord, attemptNum, heardWords[mHeardWord - 1].utteranceId == "", result); + // In narration capture mode, it is acceptable if the narrator says the wrong word because they may be starting from a different point than predicted + } else if (isNarrationCaptureMode && wordIndex <= mCurrWord) { + + boolean isAtSeam = false; + for(int i : seamIndices) { + if(wordIndex == i) { + isAtSeam = true; + break; + } + } + + if(isAtSeam) { + + incWord(wordIndex+1 - mCurrWord); + mHeardWord++; + mParent.updateContext(rawSentence, mCurrLine, wordsToSpeak, mCurrWord - 1, heardWords[mHeardWord - 1].hypWord, attemptNum, heardWords[mHeardWord - 1].utteranceId == "", result); + mListener.updateNextWordIndex(mHeardWord); // what does this do ?? - chirag + + Log.i("ASR", "Wrong But Continuing"); + attemptNum = 0; + result = true; + + } else { + mListener.setPauseListener(true); + + Log.i("ASR", "WRONG"); + attemptNum++; + result = false; + mParent.updateContext(rawSentence, mCurrLine, wordsToSpeak, mCurrWord, heardWords[mHeardWord].hypWord, attemptNum, heardWords[mHeardWord].utteranceId == "", result); + break; + } + + } else { mListener.setPauseListener(true); @@ -1578,10 +1923,39 @@ public void onUpdate(ListenerBase.HeardWord[] heardWords, boolean finalResult) { } catch (Exception e) { - Log.e("ASR", "onUpdate Fault: " + e); + Log.e("ASR", "onUpdate Fault: " + Log.getStackTraceString(e)); } + } + @Override + public void wrongWordBehavior() { + /* + if (isNarrationCaptureMode && hearRead.equals(FTR_USER_READ)) { + + Log.d("NARRATE HESITATION", "wrong word behavior. "); + + mListener.setPauseListener(true); + + Log.i("NARRATE HESITATION", "WRONG"); + attemptNum++; + if (allHeardWords != null) { + mParent.updateContext(rawSentence, mCurrLine, wordsToSpeak, mCurrWord, allHeardWords[mHeardWord-1].hypWord, attemptNum, allHeardWords[mHeardWord-1].utteranceId == "", false); + Log.d("ASR", "Wrong Word Behavior: word" + allHeardWords[mHeardWord-1]); + } else { + mParent.updateContext(rawSentence, mCurrLine, wordsToSpeak, mCurrWord, wordsToSpeak[0], attemptNum, false, false); + Log.d("ASR", "Wrong Word Behavior: word" + wordsToSpeak[0]); + + } + + // Publish the outcome + mParent.publishValue(TCONST.RTC_VAR_ATTEMPT, attemptNum); + mParent.UpdateValue(false); + + mParent.onASREvent(TCONST.RECOGNITION_EVENT); + } + */ + } public void generateVirtualASRWord() { @@ -1640,4 +2014,533 @@ public void loadJSON(JSONObject jsonData, IScope scope) { JSON_Helper.parseSelf(jsonData, this, CClassMap.classMap, scope); } + + /** + * In Narration capture mode, the audio recording data is saved to the storydata file and updated + * by RoboTutor in order to then echo the file. Currently, RoboTutor uses a greedy tiling algorithm + * to load narrations and tile them. This means that the longest segment that starts with + * the first word is prioritized. After that, the longest segment that starts with one word after + * the last word of the previous segment is prioritized. This occurs until the end of the sentence + * is reached. + */ + @Override + public void constructAudioStoryData() { + + if (isUserNarrating) { + + // runtime greedy tiling algorithm to build narrations + int startSegment = 0; + ArrayList prev_i_loc = new ArrayList(); + int endSegment = wordsToSpeak.length - 1; + + ArrayList narrationList = new ArrayList<>(); + + boolean narrationCovered = false; + + do { + + CASB_Narration narration = new CASB_Narration(); + ArrayList segList = new ArrayList<>(); + + // build phrase file name + StringBuilder fileName = new StringBuilder(mAsset); + for (int i = startSegment; i <= endSegment; i++) { + fileName.append(wordsToSpeak[i].toLowerCase()).append("_"); + } + fileName.deleteCharAt(fileName.lastIndexOf("_")); + String fileString = fileName.toString() + ".mp3"; + + File audioFile = new File(fileString); + + Log.d(TAG, "NARRATION CAPTURE MODE Greedy algorithm " + "endsegment: " + endSegment + " startsegment: " + startSegment); + Log.d(TAG, "NARRATION CAPTURE Searching for narration " + fileString); + + if(audioFile.exists()) { + Log.d(TAG, "Narration Construction: found file: " + fileString); + prev_i_loc.add(startSegment); + startSegment = endSegment + 1; + endSegment = wordsToSpeak.length - 1; + + // add segmentation data + try { + narration.audio = fileString.replace(mAsset, ""); + + FileInputStream segmentationDataFile = new FileInputStream(new File(fileName.toString() + ".seg")); + BufferedReader reader = new BufferedReader(new InputStreamReader(segmentationDataFile)); + String line; + + while ((line = reader.readLine()) != null) { + String[] data = line.split("\t"); + + segList.add(new CASB_Seg(Integer.parseInt(data[1]), Integer.parseInt(data[2]), data[0])); + } + + narration.from = segList.get(0).start; + narration.until = segList.get(segList.size()-1).end; + narration.utterances = fileName.toString().replace("_", " "); + narration.segmentation = segList.toArray(new CASB_Seg[segList.size()]); + + narrationList.add(narration); + + } catch (FileNotFoundException e) { + Log.wtf(TAG, "Unable to construct narration for " + fileString + " because " + fileName.toString() + ".seg does not exist"); + return; + } catch (IOException e) { + Log.wtf(TAG, "Unable to construct narration for " + fileString); + Log.getStackTraceString(e); + return; + } + + } else if(endSegment > startSegment) { + // if the file doesn't exist, look for a file that is 1 word shorter + endSegment--; + + } else { + // if we are down to the last word then an adequate recording doesn't exist. Start search again, but start from a shorter recording than last time + try { + narrationList.remove(narrationList.size() - 1); + endSegment = startSegment - 2; + if (prev_i_loc.size() > 0) { + startSegment = prev_i_loc.get(prev_i_loc.size() - 1); + prev_i_loc.remove(prev_i_loc.size() - 1); + } else { + Log.d(TAG, "Could not create narration"); + break; + } + } catch (ArrayIndexOutOfBoundsException e) { + Log.d(TAG, "Narration Capture: There is no possible narration that can be built"); + break; + } + } + + + if(startSegment == wordsToSpeak.length) + narrationCovered = true; + } while(!narrationCovered); + + data[mCurrPage].text[mCurrPara][mCurrLine].narration = narrationList.toArray(new CASB_Narration[narrationList.size()]); + + isUserNarrating = false; + + resetNarration(); + } + + } + + /** + * for an incorrect utterance in NARRATION CAPTURE MODE + * Obsolete + */ + @Override + public void clearAudioData() { + //AudioDataStorage.clearAudioData(); + //AudioWriter.destroyContent(); + } + + /** + * Line is restarted after a wrong narration in NARRATION CAPTURE MODE + */ + @Override + public void startLine() { + deleteRecording = keepOnlyRelevantAudio; + seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, TCONST.ZERO); + deleteRecording = true; + } + + int narrateWordsToDelete; + ArrayList acceptedUtterances = new ArrayList<>(); + ArrayList acceptedList = new ArrayList<>(); + + + /** + * + * Determines the point at which the current utterance should be end (at a silence) + * @param lastUtteranceOfSentence + * @return + */ + private ArrayList determineUtterance(boolean lastUtteranceOfSentence) { + + int offsetTime = (int) mParent.getoffsetTime() / 10; + List segments = mParent.getSegments(); + // creates a 2d HeardWord list of all continuous utterances spoken + if(allHeardWords == null) { + return new ArrayList(); + } + AudioDataStorage.updateHypothesis(allHeardWords); + for(ListenerBase.HeardWord word : allHeardWords) { + Log.d("HeardWords", word.hypWord); + } + + ArrayList> uttMap = new ArrayList<>(); + + uttMap.add(new ArrayList()); + + int start = 1; + for(int i = 0; i < AudioDataStorage.segmentation.size(); i++) { + if (AudioDataStorage.segmentation.get(i).matchLevel == ListenerBase.HeardWord.MATCH_EXACT) { + uttMap.get(0).add(AudioDataStorage.segmentation.get(i)); + start = i + 1; + break; + } + } + + for (int i = start; i < AudioDataStorage.segmentation.size(); i++) { + + // heardword currently being 'investigated' + ListenerBase.HeardWord currHeardWord = AudioDataStorage.segmentation.get(i); + // if the word is not an exact match, ignore + if (currHeardWord.matchLevel == ListenerBase.HeardWord.MATCH_EXACT) { + ArrayList latestSubSeq = uttMap.get(uttMap.size() - 1); + + if (latestSubSeq.size() == 0) { + latestSubSeq.add(currHeardWord); + // check word is the next sentence word + } else if (latestSubSeq.get(latestSubSeq.size() - 1).iSentenceWord == currHeardWord.iSentenceWord - 1) { + latestSubSeq.add(currHeardWord); + } else { + ArrayList nextSubseq = new ArrayList<>(); + nextSubseq.add(currHeardWord); + uttMap.add(nextSubseq); + } + } else { + uttMap.add(new ArrayList()); + } + } + + StringBuilder uttMapDiagram = new StringBuilder(""); + + ArrayList latestUtterance = new ArrayList<>(); + + int maxSeqLength = wordsToSpeak.length - prevStartFrom; + + int seqIndex = 0; + for (ArrayList seq : uttMap) { + uttMapDiagram.append("\n"); + for (ListenerBase.HeardWord hd : seq) { + uttMapDiagram.append(hd.hypWord).append(" (").append(hd.iSentenceWord).append(") "); + } + + if (seq.size() > 0) { + + // firstindex is the index of the word in the sentence where the narrator began speaking + int firstindex = seq.get(0).iSentenceWord; + // The sequence must pick up where the narrator left off + if (firstindex == TCONST.ZERO || firstindex > (wordsToSpeak.length - prevStartFrom - 1)) { + + // Unless the narrator reached the end, it is necessary to find the point at which + // to end the current utterance + if(!lastUtteranceOfSentence) { + // Truncate this subsequence so that it ends in a silence + // Start at the end of the subsequence and travel backwards to find the *last silence* + for (int i = seq.size() - 1; i > 0; i--) { + + // find the corresponding RAW SEGMENT for this particular heardWord + // this way you can find out whether it is preceded/followed by a silence + int correspondingSegmentIndex = 0; + for(Segment s: segments) { + if(s.getStartFrame() - offsetTime == seq.get(i).startFrame) { + break; + } + correspondingSegmentIndex++; + } + try { + // if (seq.size() < 2) break; // no use trying to test a 1 word utterance + if (seq.get(i).hypWord.contains("START_")) { + seq.remove(i); + } else if (i + 1 == seq.size() && segments.get(correspondingSegmentIndex + 1).getWord().equals("")) { + // word is the last word of subsequence and ends with a silence, so the whole + // subsequence is a usable utterance + break; + } else if (seq.get(i).startFrame - seq.get(i-1).endFrame >= SEGMENT_GAP_LENGTH && segments.get(correspondingSegmentIndex - 1).getWord().equals("")) { + // word is preceded by a silence (AND A 100 CENTISECOND GAP), so remove the current word + // and cut off the utterance here + seq.remove(i); + break; + } else if (seq.size() > maxSeqLength) { + seq.remove(i); + } else { + // none of the conditions are met for this to be an end of the utterance, + // so simply remove this unusable word and try the one behind it + seq.remove(i); + } + } catch (IndexOutOfBoundsException e) { + seq.remove(i); + } + } + } + + // If this is the farthest reaching subsequence so far save it + if (latestUtterance.size() > 0) { + if (seq.size() > 1 && seq.get(seq.size() - 1).iSentenceWord > latestUtterance.get(latestUtterance.size() - 1).iSentenceWord) { + latestUtterance.clear(); + latestUtterance.addAll(seq); + } + } else { + latestUtterance.addAll(seq); + } + + } + } + seqIndex++; + } + Log.d("Utterance_Map", uttMapDiagram.toString()); + Log.d(TAG, Integer.toString(latestUtterance.size())); + + Log.d("CRt_ViewmanagerASB", "Utterance Map"); + + + return latestUtterance; + } + + /** + * Restart utterance if the narration is wrong in Narration Capture Mode, preventing the user from having to restart the entire sentence + */ + @Override + public void restartUtterance() { + Log.d(TAG, "restarting utterance"); + AudioDataStorage.updateHypothesis(allHeardWords); + ArrayList latestUtterance = determineUtterance(false); + + if (latestUtterance.size() > 0) { + StringBuilder newFileName = new StringBuilder(); + for (ListenerBase.HeardWord h : latestUtterance) { + newFileName.append(h.hypWord.toLowerCase()).append(" "); + } + newFileName.deleteCharAt(newFileName.length() - 1); + Log.d(TAG, "Narration file name: " + newFileName.toString()); + String oldFileName = narrationFileName; // make a copy to be safe + renameNarration(oldFileName, newFileName.toString()); + narrationFileName = newFileName.toString(); + + isUserNarrating = true; + narrateWordsToDelete = mCurrWord - (latestUtterance.get(latestUtterance.size() - 1).iSentenceWord + prevStartFrom + 1); + capturedUtt = latestUtterance; + + // output a .seg file + StringBuilder withPunctuation = new StringBuilder(); + for (ListenerBase.HeardWord w : latestUtterance) { + withPunctuation.append(w.hypWord).append(" "); + } + withPunctuation.deleteCharAt(withPunctuation.length() - 1); + AudioDataStorage.saveAudioData(narrationFileName, mAsset, mCurrLine, mCurrPara, mCurrPage, withPunctuation.toString(), currUtt, latestUtterance); + + acceptedUtterances.add(narrationFileName); + acceptedList.add(new int[]{prevStartFrom, latestUtterance.get(latestUtterance.size() - 1).iSentenceWord + prevStartFrom}); + int[] segment = acceptedList.get(acceptedList.size() -1 ); + Log.d(TAG, "Added to acceptedList: segment[0] " + segment[0] + " segment[1] " + segment[1] + ". AcceptedList size is " + acceptedList.size()); + // acceptedList.get(acceptedList.size() - 1)[1] = mCurrWord - 1; + + seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, prevStartFrom + latestUtterance.get(latestUtterance.size() - 1).iSentenceWord + 1); + UpdateDisplay(); + } else { + seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, prevStartFrom); + } + + } + + + public void endOfUtteranceCapture() { + + capturedUtt = AudioDataStorage.segmentation; + + // creates a 2d HeardWord list of all continuous utterances spoken + // todo (chirag): write segmentation data to file also + + Log.d(TAG, "end of utterance capture"); + AudioDataStorage.updateHypothesis(allHeardWords); + ArrayList latestUtterance = determineUtterance(true); + + if (latestUtterance.size() > 0) { + + while (!(latestUtterance.get(latestUtterance.size() - 1).hypWord.equals(wordsToSpeak[wordsToSpeak.length - 1]))) { + if(latestUtterance.get(latestUtterance.size() - 1).hypWord.equals(wordsToSpeak[wordsToSpeak.length - 2])) { + + Log.d(TAG, "Sentence end reached, however last word omitted."); + hearRead = TCONST.FTR_USER_READ; + seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, TCONST.ZERO); + return; + + } + latestUtterance.remove(latestUtterance.size() - 1); + } + + StringBuilder newFileName = new StringBuilder(); + for (ListenerBase.HeardWord h : latestUtterance) { + newFileName.append(h.hypWord.toLowerCase()).append(" "); + } + newFileName.deleteCharAt(newFileName.length() - 1); + String oldFileName = narrationFileName; // make a copy to be thread safe + renameNarration(oldFileName, newFileName.toString()); + narrationFileName = newFileName.toString(); + + isUserNarrating = true; + narrateWordsToDelete = mCurrWord - (latestUtterance.get(latestUtterance.size() - 1).iSentenceWord + 1); + capturedUtt = latestUtterance; + + // output a .seg file + StringBuilder withPunctuation = new StringBuilder(); + for (ListenerBase.HeardWord w : latestUtterance) { + withPunctuation.append(w.hypWord).append(" "); + } + withPunctuation.deleteCharAt(withPunctuation.length() - 1); + AudioDataStorage.saveAudioData(narrationFileName, mAsset, mCurrLine, mCurrPara, mCurrPage, withPunctuation.toString(), currUtt, latestUtterance); + acceptedList.add(new int[] {prevStartFrom, wordsToSpeak.length - 1}); + + boolean narrationCovered = false; + int startSegment = 0; + int endSegment = wordsToSpeak.length - 1; + ArrayList seams = new ArrayList(); + seams.add(-1); + /* + while(!narrationCovered) { + boolean segmentCovered = false; + for(int[] segment : acceptedList) { + if (segment[0] == startSegment && segment[1] == endSegment) { + segmentCovered = true; + seams.add(endSegment); + if(endSegment == wordsToSpeak.length - 1) { + narrationCovered = true; + } else { + startSegment = endSegment + 1; + endSegment = wordsToSpeak.length - 1; + break; + } + } + } + if(!segmentCovered) { + if((endSegment - startSegment) < 2) { + if (startSegment > 0) { + startSegment = seams.get(seams.size() - 2) + 1; + endSegment = seams.get(seams.size() - 1) - 1; + } else { + Log.d(TAG, "Sentence end reached however narration was not complete."); + hearRead = TCONST.FTR_USER_READ; + seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, TCONST.ZERO); + return; + } + } else { + endSegment--; + } + } + } + + */ + Log.d(TAG, "End Of Utterance Capture. Successfully captured narration of sentence"); + UpdateDisplay(); + acceptedList.clear(); + seamIndices.clear(); + mParent.publishValue(TCONST.RTC_VAR_NARRATECOMPLETESTATE,TCONST.FALSE); + constructAudioStoryData(); + } else { + Log.wtf(TAG, "Unable to save final part of narration"); + } + + + } + + + void renameNarration(String oldFileName, String newFileName) { + AudioWriter.pauseNRename(oldFileName, newFileName, mAsset); + } + + int prevStartFrom = 0; + + /** + * Gives the file location to save the current recording to the AudioWriter so that the data collected is immediately written to file + * @param startFrom is the location of the sentence to start the recording from + */ + public void feedSentence(int startFrom) { + seamIndices.add(startFrom); + // acceptedList.add(new int[]{mCurrWord, 0}); + Log.d(TAG, "Chirag: currUtt is " + currUtt); + Log.d(TAG, "Chirag: prevStartFrom will be" + startFrom); + + if (!storyBooting) { + if (hearRead.equals(TCONST.FTR_USER_HEAR)) { + AudioWriter.pauseRecording(); + Log.d("Narrate-Debug", "Recording paused (in hear mode)"); + return; + } + } + + StringBuilder fileNameBuilder = new StringBuilder(); + int i = 0; + for (String word : wordsToSpeak) { + // This uses the wordsToSpeak String[] because it does not contain punctuation + //if (i >= startFrom) { + fileNameBuilder.append(word).append(" "); + //} + //i++; + } + fileNameBuilder.setLength(fileNameBuilder.length() - 1); + + fileNameBuilder.append("TEMP"); + String fileName = fileNameBuilder.toString(); + + Log.d("CRt_ saveToFile", "Telling Audiowriter to being writing file. File is: " + fileName); + AudioWriter.initializePath(fileName, mAsset); + + narrationFileName = fileName; + + prevStartFrom = startFrom; + } + + /** + * DO NOT CONFUSE THIS WITH prevLine(). + * THIS IS USED TO GO BACK A SENTENCE REGARDLESS OF CURRENT STORY POSITION. IT'S ORIGINAL PURPOSE IS TO ALLOW FOR NAVIGATION WITHIN THE STORY + * prevLine() GOES BACK TO THE PREVIOUS SENTENCE IF AND ONLY IF THERE IS A SENTENCE WITHIN THE PARAGRAPH BEFORE IT + */ + @Override + public void prevSentence() { + + if (!narrationTracking) { + + Log.d("NavButton", "Back Button has been pressed"); + AudioWriter.abortOperation(); + hearRead = TCONST.FTR_USER_HEAR; + // It is zero-index + if (mCurrWord > prevStartFrom) { + seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, prevStartFrom); + } else if (mCurrLine > 0) { + prevLine(); + } else if (mCurrPara > 0) { + prevPara(); + } else if (mCurrPage > 0) { + prevPage(); + } + } + } + + @Override + public void skipSentence() { + if (!narrationTracking) { + Log.d("NavButton", "Forward Button has been pressed"); + AudioWriter.abortOperation(); + mCurrWord = mWordCount; // go to the end of the sentence + publishStateValues(); + hearRead = TCONST.FTR_USER_HEAR; + // mParent.applyBehavior(TCONST.UTTERANCE_COMPLETE_EVENT); + /* + if (buttonState.equals(TCONST.RTC_PAGECOMPLETE)) { + + nextPage(); + } else if (buttonState.equals(TCONST.RTC_PARAGRAPHCOMPLETE)) { + nextPara(); + } else if (buttonState.equals(TCONST.RTC_LINECOMPLETE)) { + nextLine(); + } + */ + } + } + + private void resetNarration() { + rawNarration = data[mCurrPage].text[mCurrPara][mCurrLine].narration; + rawSentence = data[mCurrPage].text[mCurrPara][mCurrLine].sentence; + if (data[mCurrPage].prompt != null) page_prompt = data[mCurrPage].prompt; + UpdateDisplay(); + } + + public void checkForGarbage(List s) { + + } } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerMari.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerMari.java index 6e9d2019f..ba35decc3 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerMari.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerMari.java @@ -203,6 +203,26 @@ public void execCommand(String command, Object target ) { } } + @Override + public void constructAudioStoryData() { + + } + + @Override + public void clearAudioData() { + + } + + @Override + public void startLine() { + + } + + @Override + public void restartUtterance() { + + } + @Override public void onUpdate(ListenerBase.HeardWord[] heardWords, boolean finalResult) { @@ -594,6 +614,33 @@ public void loadJSON(JSONObject jsonData, IScope scope) { System.out.println(sb.toString()); sentences = new ArrayList(Arrays.asList(sb.toString().split("\\."))); } + + public void prevSentence() { + + } + + @Override + public void enableNarrationCaptureMode(boolean isNarrateMode, boolean keepExtraAudio) { + + } + + public void enableNarrateMode(boolean isNarrateMode) { + + } + + public void skipSentence() { + + } + + @Override + public void endOfUtteranceCapture() { + + } + + @Override + public void wrongWordBehavior() { + + } } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java b/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java index a41ce0916..f650595ea 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java @@ -18,7 +18,10 @@ package cmu.xprize.rt_component; +import java.util.List; + import cmu.xprize.util.ILoadableObject; +import edu.cmu.pocketsphinx.Segment; import edu.cmu.xprize.listener.ListenerBase; public interface ICRt_ViewManager extends ILoadableObject { @@ -75,4 +78,21 @@ public interface ICRt_ViewManager extends ILoadableObject { public void setPageFlipButton(String command); public void execCommand(String _command, Object _target); + + public void constructAudioStoryData(); + public void clearAudioData(); + public void startLine(); + + public void restartUtterance(); + + public void prevSentence(); + + public void enableNarrationCaptureMode(boolean isNarrationCaptureMode, boolean keepExtraAudio); + + public void skipSentence(); + + public void endOfUtteranceCapture(); + + public void wrongWordBehavior(); + } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java b/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java index 36fd9b1b1..265c455ad 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java @@ -54,4 +54,11 @@ public interface IRtComponent { public void continueListening(); + public void constructAudioStoryData(); + public void clearAudioData(); + public void startLine(); + public void prevSentence(); + public void restartUtterance(); + public void skipSentence(); + public void endOfUtteranceCapture(); } diff --git a/comp_reading/src/main/res/layout/asb_evenpage.xml b/comp_reading/src/main/res/layout/asb_evenpage.xml index c01a8c6f1..79dea480a 100644 --- a/comp_reading/src/main/res/layout/asb_evenpage.xml +++ b/comp_reading/src/main/res/layout/asb_evenpage.xml @@ -69,7 +69,41 @@ android:layout_height="match_parent" android:layout_marginLeft="30dp" android:layout_marginRight="30dp" - android:layout_centerVertical="true" /> + android:layout_centerVertical="true"> + + + + + + + diff --git a/comp_reading/src/main/res/layout/asb_oddpage.xml b/comp_reading/src/main/res/layout/asb_oddpage.xml index 096798f12..8d86aabbe 100644 --- a/comp_reading/src/main/res/layout/asb_oddpage.xml +++ b/comp_reading/src/main/res/layout/asb_oddpage.xml @@ -7,6 +7,21 @@ android:id="@+id/story_reading" android:background="#FFF"> + + + android:layout_centerInParent="true"> + + + + + + + diff --git a/comp_reading/src/main/res/layout/rt__component.xml b/comp_reading/src/main/res/layout/rt__component.xml index cd4cbc122..713ba168d 100644 --- a/comp_reading/src/main/res/layout/rt__component.xml +++ b/comp_reading/src/main/res/layout/rt__component.xml @@ -1,6 +1,7 @@ diff --git a/docs/Narration Capture Mode/README.md b/docs/Narration Capture Mode/README.md new file mode 100644 index 000000000..899602480 --- /dev/null +++ b/docs/Narration Capture Mode/README.md @@ -0,0 +1,121 @@ +# Documentation for Narration Capture Mode + +Narration capture mode exists to create narrations of existing stories and store the relevant data for RoboTutor to use. To use narration capture mode, a storydata.json file should already be created for a given story, but the necessary narrations and audio file information should be missing. + +## Usage + +Narration capture mode requires 3 files to be modified: debug.json, config.json, and storydata.json. + +Directions for Use +STEP 1: Create storydata.json for a particular story, but leave out segmentation and narration data in the storydata.json file + +STEP 2: In the downloads folder of the android device, create a folder with the story title, and in it create another folder with the story title + an underscore + a level number (e.g. if the story was titled, say, "butterflies" and the level was "1", you should have the file structure "Downloads/butterfly/butterflies_1"). Save the storydata.json in the subfolder. + +STEP 3: Configure debug.json to look like the following. Replace "storyName_level" with your story name and level identical to the subfolder created earlier. +``` + { + "type": "TRANSITION", + "skill": "stories", + "tutor_id": "story.echo::storyName_level", + "tutor_desc":"story.echo", + "tutor_data": "[unpackaged_asset]storyName_level", + "use_hash_name": "false" + } +``` +STEP 4: Configure config.json to include the following variable: +``` + "content_creation_mode": false +``` +STEP 5: Launch RoboTutor. + +For more information on storydata.json and config.json, visit the [github authoring documentation](https://drive.google.com/drive/u/0/folders/0B7kzHmZs33scWjF3Qkw2MFo1Wm8?resourcekey=0-ZK-ICf9-mziZzT1Atavx0A). Information on debug.json can be found [here](https://docs.google.com/document/d/1qF4lGDrR7wzWOTY7lcwdhF6ttJegUvQxSEVUF-s8UoI/edit). + +## Narration Audio Input + +Audio input is captured by "intercepting" the audio data where it is fed to the recognizer and saving it into a temporary file. The handling of audio data is done by the [AudioWriter.java](../comp_listener/src/main/java/edu/cmu/xprize/listener/AudioWriter.java) class, using the method `addAudio()`. The SpeechRecognizer passes the audio buffer to this class, as well as recognizing it. + +From [SpeechRecognizer.java](../comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java): +``` +decoder.processRaw(buffer, nread, false, false); + //Log.d("ASR", "Time in processRaw: " + (System.currentTimeMillis() - ASRTimer)); + +AudioWriter.addAudio(nread, buffer); + +nSamples += nread; +``` +### Storing the input + +The audio is written to a temporary file. The filepath is initialized via `AudioWriter.initializePath()` when each successive segment begins. This generally occurs in [CRt_ViewmanagerASB.java](../comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java) in the method `seekToStoryPosition()`. When seeking to the next point in the story, the new path is initialized. + +When the narration is stopped earlier than anticipated (see [Tiling](#tiling) for more information about why this happens) the narration temporary file is renamed and then converted to an mp3, as RoboTutor's playback mechanism can only work with mp3 files. This happens by calling `AudioWriter.pauseRecording()`, which adds a header to the raw audio, making it a playable wav file, and then converting the wav to an mp3. Additionally, the mp3 is renamed to adequately reflect the usable contents of the file. That is, the file is named exactly what the utterance in it contains with all lowercase characters and underscores for spaces. (A file containing the words "I row a boat" becomes the file "i_row_a_boat.mp3") + +Finally, the process is restarted as a new path is initialized and more data is fed. + +## Using the Tutor Logic and ASR Output + +Narration capture mode builds on the current logic of the reading tutor. The user creating a narration has an experience very similar to that of a student using the platform to practice their reading skills. + +Most tutor logic is handled in the class [CRt_ViewmanagerASB.java](../comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java), so this is where the narration capture mode logic exists as well. Additionally, some changes are made to the [animator graph](../app/src/main/assets/tutors/story_reading/animator_graph.json) for reading. + +*Note: The word "cursor" is used for the current point in the story that the tutor is listening for (and that the user is expected to say). It is the word that is being prompted and would be underlined during story reading.* + +Every time the cursor moves to a point that is not the next word in the current sentence, a new path to write audio is generated through the method `feedSentence()`. It takes a single parameter - the word in the sentence to being listening from. The utterance can be "completed" in 2 ways - either the narrator says the wrong word, in which case the utterance is completed before the end of the sentence and the method `restartUtterance()` is called, or the narrator states the correct word and `endOfUtteranceCapture()` is called. + +When the narrator states an incorrect word, the method `restartUtterance()` is called, as the WRONG node in the animator graph shows. (this only occurs in narration capture mode). Here, an incorrect word is any word that is not the word after the current sentence, as well as not being the first word of the sentence. We allow the user to start at the beginning of the sentence at any time to allow for smoother narrations. This behavior can be seen in the method `onUpdate()` in CRt_ViewManagerASB.java, where the recognizer passes every updated hypothesis after multimatch output. + +`restartUtterance()` separates the recorded audio into an **utterance map**, where the different continuous subsequences of the sentence are separated from non. For example (where each subsequence in the utterance map is denoted by a new line): +    *Sentence text: "Once upon a time a beautiful princess"* +     *Narrator: " Once upon a... Once... upon... a time... Once upon a time a beautiful frog* +     *Utterance map:* +          *Once upon a* +          *Once upon a time* +          *Once upon a time a beautiful* +          *frog* + +Next, it determines the usable portion of the subsequences - A subsequence that does not end in a silence is not usable, as [tiling](#tiling) recordings using such subsequences would be prone to inaccuracy. It thus tries to find the longest subsequence possible that ends in a silence. If that means shortening one of the existing subsequences, that is fine. Finally, it moves the cursor to the location in the text analogous to the word after the accepted subsequence, for the narrator to begin narrating again. + +If the narrator reaches the end of the sentence, a similar process via `endOfUtteranceCapture()` is used to find the portion of the recording when the narrator narrated until the end of the sentence. + +*NOTE: a bug sometimes causes RoboTutor to act as if that the narrator has reached the end of the sentence even when they have not (according to the ASR output). This does not interfere with other parts of the program becuase they do not rely on ASR precision as much as narration capture mode does* + +The segmentation data for the corresponding part of the sentence is fed to the `AudioDataStorage` class, which writes such data to a .seg file named identically to the mp3 file in the story folder and also edits the storydata.json file at runtime to add in the reent segmentation data (although this latter function does not happen at the same time, but rather later during narration tiling). + +Notably, the recordings are not edited at all (no trimming), meaning that they contain all the mistakes that the narrator made as well as attempts to restart. For this reason, only part of the file is played back - the usable "phrase". Phrases begin at a "seam" - either the word after the previous phrase ended OR the beginning of the sentence. Additionally, a phrase begins and ends with silence. + +## Tiling + +Narration capture mode seeks to tile multiple recordings of partial narrations to form a full narration of the sentence. These recordings are referred to as "utterances". The word "phrase" refers to the usable portion of these utterances. + +The tiling algorithm, `constructAudioStoryData()` of CRt_ViewManagerASB.java tiles the utterances in the story folder. It uses the names of the utterances to infer their contents, and determines the resulting sequence of utterances to play back as a narration of words. It simply concatenates non-overlapping utterances to "cover" the whole sentence. Due to the difficulty of finding an optimal solution, narration capture mode uses a greedy tiling heuristic: It attempts to find a narration for words 1..n for the largest possible n. Next it attempts to find the longest possible narration from words n..m. If no utterance exists for any n..m where m >=n, it starts from scratch, after reducing the max possible n to n-1. It continues this pattern until it reaches the end of the sentence. + +Say you have the following utterances for the sentence "My father buys meat on Saturday.": +     *my_father.mp3* +     *buys_meat_on_saturday.mp3* +     *my_father_buys.mp3* +     *meat_on.mp3* +     *saturday.mp3* +The narration tiling algorithm will tile the utterances so playback of the sentence would involve playing back the following utterances in order: *my_father_buys.mp3, meat_on.mp3, saturday.mp3.* This is not optimal but easiest to compute. In its current state, tiling does not allow for overlapping phrases. + +The tiling mechanism occurs after the sentence has been completely narrated (the narrator reached the end of the sentence), and modifies the internal representation of the story by adding in the information about the recordings and the segmentation (gathered from the names of the recordings themselves and the .seg files associated with each recording). + +## Narration Behavior + +Narration capture mode uses the reading tutor's "echo" mode to play back audio. Functionality is almost exactly the same, except that the playback has been modified to start at the beginning of the first word and end at the end of the last word, instead of playing the entire file back at once. This can be seen through the posted behaviors `TCONST.START_LATER` AND `TCONST.STOP_EARLY`. `START_LATER` instantaneously forwards the recording to a specified start time as soon as playback begins (see [`TRtComponent.startLate()`](../app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java)) while stop early sets a timer for the length of playback and instantly forwards to the end of the file once that timer goes off. The timer uses the existing `post` mechanism in CRt_Component.java. + +Additionally, the highlighting of text has been modified during playback. The utterance being currently narrated has a cyan background in addition to the default text effects. + +## Issues + +1. **ASR inaccuracy** +The ASR is extremely faulty and turns otherwise fluent readers into robotic, monotone machines. Becuase of false rejections and outright lack of speech detection at times, the ASR forces the narrator to pause too long between words or over-enunciate. This causes obvious issues with developing a good narration, as an ideal narration is smooth and reflects speaking in real life. + +## Still to do + +1. **UI Enhancement** + Currently, narratin capture mode has a limited UI identical to the reading tutor. This means that it auto-advances at the end of sentences. 2 features should be implemented to give the narrator more control +       a. Better navigation (buttons are being implemented have not been robustly tested) +       b. Tap to play back existing narrations +       c. Start and stop narrations at will + +2. **Testing** + Tiling should be tested on canned speech. diff --git a/util/build.gradle b/util/build.gradle index b79be7994..372c4124c 100644 --- a/util/build.gradle +++ b/util/build.gradle @@ -30,6 +30,12 @@ android { testOptions { unitTests.returnDefaultValues = true } + + configurations { + all { + exclude group: 'com.gitub.adrielcafe', module: 'ffmpeg-android-java' + } + } } dependencies { diff --git a/util/src/main/java/cmu/xprize/util/JSON_Helper.java b/util/src/main/java/cmu/xprize/util/JSON_Helper.java index 3646b5653..797374c8e 100644 --- a/util/src/main/java/cmu/xprize/util/JSON_Helper.java +++ b/util/src/main/java/cmu/xprize/util/JSON_Helper.java @@ -413,7 +413,7 @@ else if (fieldClass.equals(HashMap.class)) { // You can have a global hash RH type where you defince the Right hand type once // This permits native types - i.e. arrays etc on the RH side. - // Note that type info gets lost in the compile so it is not available for introspection + // Note that type info gets lost in the api so it is not available for introspection // Check for global type Class elemClass = null; diff --git a/util/src/main/java/cmu/xprize/util/TCONST.java b/util/src/main/java/cmu/xprize/util/TCONST.java index e4b76e64e..9ace24049 100644 --- a/util/src/main/java/cmu/xprize/util/TCONST.java +++ b/util/src/main/java/cmu/xprize/util/TCONST.java @@ -226,6 +226,7 @@ public class TCONST { public static final String FTR_USER_REVEAL = "FTR_USER_REVEAL"; public static final String FTR_USER_PARROT = "FTR_USER_PARROT"; public static final String FTR_USER_READING = "FTR_USER_READING"; + public static final String FTR_NARRATION_CAPTURE = "NARRATION_CAPTURE_MODE"; // UHQ public static final String FTR_GEN = "FTR_GEN"; public static final String FTR_PIC = "FTR_PIC"; @@ -233,6 +234,7 @@ public class TCONST { public static final String STOP_AUDIO = "STOP_AUDIO"; public static final String RTC_VAR_CLOZEWORD = ".clozeWord"; public static final String REMOVE_CLOZE_FROM_BLANK = "REMOVE_CLOZE_FROM_BLANK"; + public static final String START_LATER = "START_LATER"; public static final String NARRATE_STORY = "NARRATE_STORY"; @@ -833,6 +835,10 @@ public class TCONST { public static final String PAGEFLIP_BUTTON = "PAGE_FLIP_CLICK"; public static final String SPEAK_BUTTON = "SPEAK_CLICK"; + public static final String SKIP_SENTENCE = "SENTENCE_SKIP_CLICK"; + public static final String SKIP_PARA = "PARAGRAPH_SKIP_CLICK"; + public static final String SKIP_PAGE = "PAGE_SKIP_CLICK"; + public static final String EMPTY = ""; public static final int INCR = 1; @@ -840,6 +846,9 @@ public class TCONST { public static final String RTC_VAR_ECHOSTATE = ".echoState"; public static final String RTC_VAR_PARROTSTATE = ".parrotState"; + public static final String RTC_VAR_NARRATESTATE = ".narrateState"; + public static final String RTC_VAR_NARRATECOMPLETESTATE = ".narrationCompleteState"; + // Generic question state flag public static final String RTC_VAR_QUESTIONSTATE = ".questionState"; public static final String RTC_VAR_CLOZESTATE = ".clozeState"; @@ -876,6 +885,7 @@ public class TCONST { public static final String RTC_VAR_REMAINING = ".remainingWords"; public static final String RTC_VAR_SENTENCE = ".sentence"; public static final String RTC_VAR_UTTERANCE = ".utterance"; + public static final String RTC_VAR_UTTERANCE_DURATION = ".duration"; //Akira Game Prompt Situation public static final String PROMPT_1LEFT = "PROMPT_1LEFT"; @@ -931,4 +941,6 @@ public enum Thumb { public static final String WRITING_PLACEMENT_INDEX = "WRITING_PLACEMENT_INDEX"; public static final String DEBUG_CSV = "DEBUG_CSV"; + + public static final String HYP_LOG_FILE_LOCATION = "/sdcard/Download/RTNarrateHypLogs/"; }