From 8f0277a52b34cb6a43252cbbb277c4a38e1b17ca Mon Sep 17 00:00:00 2001 From: chirag03k Date: Mon, 22 Jun 2020 13:01:13 -0700 Subject: [PATCH 01/45] I had to change gradle configuration slightly in order to be operable with windows. TODO: Change it back when I make the final commit. --- app/build.gradle | 5 ++++- build.gradle | 2 +- chiragk.keystore | Bin 0 -> 2601 bytes .../xprize/rt_component/CRt_ViewManagerASB.java | 2 +- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 5 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 chiragk.keystore diff --git a/app/build.gradle b/app/build.gradle index a5d46588e..29d2a540d 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -61,7 +61,7 @@ android { // define apk naming behavior // define apk naming behavior applicationVariants.all { variant -> - variant.outputs.each { output -> + /* variant.outputs.each { output -> def project = "robotutor" def SEP = "." def buildType = variant.variantData.variantConfiguration.buildType.name @@ -69,6 +69,9 @@ android { def newApkName = project + SEP + buildType + SEP + version + ".apk" output.outputFile = new File(output.outputFile.parent, newApkName) + } */ + variant.outputs.all { + outputFileName = "robotutor.${variant.variantData.variantConfiguration.buildType.name}.${variant.versionName}.apk" } } diff --git a/build.gradle b/build.gradle index 233119729..ea1343843 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ buildscript { } dependencies { - classpath 'com.android.tools.build:gradle:2.3.3' + classpath 'com.android.tools.build:gradle:3.6.3' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/chiragk.keystore b/chiragk.keystore new file mode 100644 index 0000000000000000000000000000000000000000..67fce589e7e74931228d2f33e8189405cf02eabc GIT binary patch literal 2601 zcmY+EX*3iH8^;GT!(a%5$eNI~u@=U-)^M%SSngOdS%zfabupI8-jF4`CPXyMSd*=x zT#T){h^7pcUDnBx>wVAr-h1wc=RD{9{{QFs_=DrvrJ0#n;5f)fFhoAZG-V&k#L85T zgNy=kkfBrj8jb_`{VM{YfjE%kDYicy8!*TJt~lA5n9Fg%LpTn&50?Y8{~w<|KL>=J zOx_5)y_KG_A^bDk>+F38ZLQ=XW){FJ8z2t&=7G}4R+sVsHZa;}c~RpHf0`ho7JoqN zMudDvPg%#5HmS6dwRE?B* z8Ll?)%C;i(jvAFs8Dng|u5Ol_|6_rTVS>zQXgR1=@=F%>aXU&{b)RZAfe0S26RI0Lz)98k?;c)nbuJY0^2^0AvIlo8QWB+$iC)rz zoq-!U+~D}EIeyt&L)XP+@8*8|u$rV*Uy79PG)?f(&W*5=U5Lb!`NVhlRnR1_Frprm z>i6Wf{xQ<>@vtVoh4B5rS1_bBq-WQ3q~i<<;F)A(F?&@Z_f-X2I2rrFHUL*5^n#jMd=y-5&7wo>60_W%O7$k z?Am`4*ps`>CujUkgL^gI4))SyNoC)ly-?VZO8>S-JL0-g0jE7AcTclCc_~jDJqJC z@7+$8VfQ0fOi_80W(Mk;3ywSeg5uqH@k^T5bu)=PfiyY1N3Won zgwf3f85T3E#SOp^Y1>C(P2fNsAE*{#kKm)DKd@eypU)f!%3^Yw9!}r&fk~m>af{kjd-=5DRrt}E)*g!TJpSVP8&7 z;Ixldne0N9*=@ktKlA`G+6ey(@_FCr4ux9pC9Wl?7f*ya^_Xq5+Jazj&JK`=o0%13 zQtzr5z(xO7B?P8&MvM!90r&v0fMCE4fH%MoF8NQ)uL1+|+GG7aB~&!DR5Ud-G_GiA zsi~go)Bf)gEBMr?#-~W1nF(;ZwEsz%{^eNke;nKJpy&9BL*8_PWkN}dXN_BWqfX?1 z9UG4W0wSP2kd#AYwVaX+2!UyC3YSdPDd_!;@E$_|Nslh?#F86-#bt7JNkbgfD(E1r z^@lbszLM7vfGb`|NUyG7sR_0>(pMzWHXg3!HZ0XO#Is%80{AEid&4i?-i186H}@1w~IJ67NrcZjTb~td4rw2DaE<1sPJtMO#R~LukjFA}SrMVwn?9pIoLf*mOU1 zc!bBzrRz?m7;PA2R7-&_f_y^1|MV;o^(Lbo9HUw8u-z7{NYRXo5@mS8aJ$(!&0vQ~HlKNJQ>hGv*C6fW8Gg8+mdD|`hWTy`D$eMng?qHnoGE37!8YRB-#%8`LNNI-H^f8trOpJ zV%>G}qR`C^g3UgV`ej7#yJydl;*)hHE^^NnsUa3MYrWDzH+rlEl51!>lo;vdahD%` z<6EeE5lwrrsxg*ES1W>mxLNH;;N>yMQBZ}dbI6NYkjT!3*WL`j)wzzvJz)F!FbM$axzeHHmHGz|Zw`7Dx7 zw0it&dccF=UDK7c#Hy;g2mRFK#M!*(qPsO)&kBjgfQS9@XCG;)mtqjry*V zTgCUV?Rx34L9u{R{o4{0M(Ufv%6BQe-J=&)I6XKgcP)>{1VApdOOIpP^}JZFMwKx# zFx?4I($i-FU=hT`Ee>t9o63%@g^_W3heH+PhJu!Iqum#RY%*?~Xf*JPWkaE{D=!Rq*CYNTxE8wxZZaZEaFq1~$zUVXAAi}));e Date: Mon, 6 Jul 2020 15:05:37 -0700 Subject: [PATCH 02/45] Added AudioDataStorage class --- app/build.gradle | 2 + .../tutors/story_reading/animator_graph.json | 29 ++++++++++++-- .../cmu/xprize/listener/AudioDataStorage.java | 38 +++++++++++++++++++ .../cmu/xprize/listener/SpeechRecognizer.java | 2 + .../rt_component/CRt_ViewManagerASB.java | 6 +++ 5 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java diff --git a/app/build.gradle b/app/build.gradle index 29d2a540d..6e921129b 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -70,6 +70,8 @@ android { def newApkName = project + SEP + buildType + SEP + version + ".apk" output.outputFile = new File(output.outputFile.parent, newApkName) } */ + // This allows the build to work on Windows. + // TODO: delete this for final commit variant.outputs.all { outputFileName = "robotutor.${variant.variantData.variantConfiguration.buildType.name}.${variant.versionName}.apk" } diff --git a/app/src/main/assets/tutors/story_reading/animator_graph.json b/app/src/main/assets/tutors/story_reading/animator_graph.json index ba3dc58b7..dff9df137 100644 --- a/app/src/main/assets/tutors/story_reading/animator_graph.json +++ b/app/src/main/assets/tutors/story_reading/animator_graph.json @@ -268,12 +268,33 @@ "preexit": [], "edges": [ ] + }, + + "NARRATE_RIGHT": { + "type": "NODE", + "COMMENT": "node that triggers audio saving for NARRATE mode", + "maptype": "moduleMap", + "mapname": "", + "preenter": [], + "preexit": [], + "edges": [ + {"constraint": "", "edge": "NEXT_SENTENCE"} + ] } }, "moduleMap": { - "COMMENT": "@@@@@ CModules @@@@@", + + "SAVEAUDIO": { + "type": "MODULE", + "reuse": true, + "COMMENT": "module that saves audio for NARRATE mode", + "tracks" : [ + {"type": "COMMAND", "id": "StoryReading", "method": "saveAudio", "parms": "none", "features": "" } + ] + }, + "PLAYINTRO": { "type": "MODULE", "reuse": true, @@ -418,7 +439,9 @@ {"type": "COMMAND", "id": "SstoryReading", "method": "setHighLight", "parms": "red:String"}, {"type": "COMMAND", "id": "SstoryReading", "method": "continueListening" , "features": ""} ] - } + }, + + }, "actionMap": { @@ -478,7 +501,7 @@ "SET_SPEAK_SENTENCE": {"type": "COMMAND", "id": "SstoryReading", "method": "setVolatileBehavior", "parms": "SPEAK_CLICK:String|SPEAK_SENTENCE_BEHAVIOR:String" , "features": ""}, "RESET_SPEAK_ONCLICK": {"type": "COMMAND", "id": "SstoryReading", "method": "setVolatileBehavior", "parms": "SPEAK_CLICK:String|NULL:String" , "features": ""}, - "SET_NARRATOR": {"type": "COMMAND", "id": "SstoryReading", "method": "setStickyBehavior", "parms": "NARRATE_STORY:String|NARRATE_SENTENCE_BEHAVIOR:String" , "features": ""}, + "SET_NARRATOR": {"type": "COMMAND", "id": "SstoryReading", "method": "setStickyBehavior", "parms": "NARRATE_STO RY:String|NARRATE_SENTENCE_BEHAVIOR:String" , "features": ""}, "SET_UTTERANCE": {"type": "COMMAND", "id": "SstoryReading", "method": "setStickyBehavior", "parms": "SPEAK_UTTERANCE:String|SPEAK_UTTERANCE_BEHAVIOR:String" , "features": ""}, "EMPTY_ACTION": {"type": "COMMAND", "cmd": "NOOP"}, diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java new file mode 100644 index 000000000..21de13ea7 --- /dev/null +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java @@ -0,0 +1,38 @@ +package edu.cmu.xprize.listener; + +import java.util.Arrays; + +public class AudioDataStorage { + + /** + * audioData stores the data + * finIndex is used to add to audioData without overwriting existing data + * Volatile because it is accessed by the RecognizerThread but also by the _______ thread + */ + public static volatile short[] audioData = new short[160 * 25 * 100]; // creates a buffer that can store 25 seconds worth of audio + public static volatile int finIndex = 0; + + // I initialize the array and add to it + + public static synchronized void addAudioData(short[] buffer) { + try { + for (short s : buffer) { + audioData[finIndex] = s; + finIndex++; + } + } catch (ArrayIndexOutOfBoundsException e) { + // ?? + // Should I save the audio file that we have and start writing to a new file? + } + } + + public static synchronized void clearAudioData() { + Arrays.fill(audioData, (short) 0); + finIndex = 0; + } + + public static void saveAudioData(String filepath) { + + } + +} diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java index 46375c7bb..fc0fe60f3 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java @@ -577,6 +577,8 @@ public void run() { decoder.processRaw(buffer, nread, false, false); //Log.d("ASR", "Time in processRaw: " + (System.currentTimeMillis() - ASRTimer)); + AudioDataStorage.addAudioData(buffer); + nSamples += nread; // InSpeech is true whenever there is a signal heard at the mic above threshold diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java index 991bebc4d..f731faec3 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java @@ -1640,4 +1640,10 @@ public void loadJSON(JSONObject jsonData, IScope scope) { JSON_Helper.parseSelf(jsonData, this, CClassMap.classMap, scope); } + + // TODO: make this method + public void saveToFile() { + // Where to save the data? + // How to get the utterance id? + } } From d306ccebb374952cd489cd8923c186d90db73772 Mon Sep 17 00:00:00 2001 From: chirag03k Date: Sun, 12 Jul 2020 17:02:11 -0700 Subject: [PATCH 03/45] Added ability to write to file and modules to animator graph --- .../tutors/story_reading/animator_graph.json | 29 ++++++++++++++++--- .../cmu/xprize/listener/AudioDataStorage.java | 25 ++++++++++++++-- .../edu/cmu/xprize/listener/ListenerBase.java | 2 -- .../rt_component/CRt_ViewManagerASB.java | 24 +++++++++++++-- 4 files changed, 70 insertions(+), 10 deletions(-) diff --git a/app/src/main/assets/tutors/story_reading/animator_graph.json b/app/src/main/assets/tutors/story_reading/animator_graph.json index dff9df137..7f2a07bf6 100644 --- a/app/src/main/assets/tutors/story_reading/animator_graph.json +++ b/app/src/main/assets/tutors/story_reading/animator_graph.json @@ -274,12 +274,25 @@ "type": "NODE", "COMMENT": "node that triggers audio saving for NARRATE mode", "maptype": "moduleMap", - "mapname": "", + "mapname": "SAVEAUDIO", "preenter": [], "preexit": [], "edges": [ {"constraint": "", "edge": "NEXT_SENTENCE"} ] + }, + + + "NARRATE_WRONG": { + "type": "NODE", + "COMMENT": "For the 'mean' mode of NARRATE that deletes audio as soon as it hears something wrong.", + "maptype": "moduleMap", + "mapname": "CLEARAUDIO", + "preenter": [], + "preexit": [], + "edges": [ + {"constraint": "","edge": ""} //TODO: Go to previous sentence.. how? + ] } }, @@ -291,7 +304,16 @@ "reuse": true, "COMMENT": "module that saves audio for NARRATE mode", "tracks" : [ - {"type": "COMMAND", "id": "StoryReading", "method": "saveAudio", "parms": "none", "features": "" } + {"type": "COMMAND", "id": "SstoryReading", "method": "saveAudio", "parms": "none", "features": "" } + ] + }, + + "CLEARAUDIO": { + "type": "MODULE", + "reuse": true, + "COMMENT": "module that deletes current audio data", + "tracks": [ + {"type": "COMMAND","id": "SstoryReading","method": "clearAudio","parms": "none","features": ""} ] }, @@ -439,8 +461,7 @@ {"type": "COMMAND", "id": "SstoryReading", "method": "setHighLight", "parms": "red:String"}, {"type": "COMMAND", "id": "SstoryReading", "method": "continueListening" , "features": ""} ] - }, - + } }, diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java index 21de13ea7..1d6fc8bf3 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java @@ -1,5 +1,12 @@ package edu.cmu.xprize.listener; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.ShortBuffer; +import java.nio.channels.FileChannel; import java.util.Arrays; public class AudioDataStorage { @@ -9,9 +16,10 @@ public class AudioDataStorage { * finIndex is used to add to audioData without overwriting existing data * Volatile because it is accessed by the RecognizerThread but also by the _______ thread */ - public static volatile short[] audioData = new short[160 * 25 * 100]; // creates a buffer that can store 25 seconds worth of audio + public static volatile short[] audioData = new short[160 * 60 * 100]; // creates a buffer that can store 60 seconds worth of audio public static volatile int finIndex = 0; + // I initialize the array and add to it public static synchronized void addAudioData(short[] buffer) { @@ -32,7 +40,20 @@ public static synchronized void clearAudioData() { } public static void saveAudioData(String filepath) { - + try { + FileOutputStream os = new FileOutputStream(filepath + ".pcm"); + ByteBuffer dataBuffer = ByteBuffer.allocate(audioData.length * 2); + ShortBuffer dataBufferShort = dataBuffer.asShortBuffer(); + dataBuffer.order(ByteOrder.LITTLE_ENDIAN); // I *think* little endian is what .wav uses, not sure about .mp3 + dataBufferShort.put(audioData); + FileChannel out = os.getChannel(); + out.close(); + os.close(); + + // TODO: now it needs to be converted to either .wav or .mp3. I think the header can take care of that + } catch (IOException e) { + e.printStackTrace(); // ? + } } } diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerBase.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerBase.java index 6169ecfb0..ff8e509c6 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerBase.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerBase.java @@ -563,8 +563,6 @@ public static String[] textToWords(String text) { /***** Logging */ - - /** * get the path to the hypothesis log file for given utterance label */ diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java index f731faec3..f7c627cbb 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java @@ -45,6 +45,7 @@ import cmu.xprize.util.IScope; import cmu.xprize.util.JSON_Helper; import cmu.xprize.util.TCONST; +import edu.cmu.xprize.listener.AudioDataStorage; import edu.cmu.xprize.listener.ListenerBase; import static cmu.xprize.util.TCONST.FTR_USER_READ; @@ -1643,7 +1644,26 @@ public void loadJSON(JSONObject jsonData, IScope scope) { // TODO: make this method public void saveToFile() { - // Where to save the data? - // How to get the utterance id? + // Step 1. get sentence text + StringBuilder fileNameBuilder = new StringBuilder(); + for (String word : wordsToDisplay) { // This uses the wordsToDisplay String[] because it contains punctuation + fileNameBuilder.append(word).append(" "); + } + fileNameBuilder.setLength(fileNameBuilder.length() - 1); + fileNameBuilder.append(".mp3"); + + String fileName = fileNameBuilder.toString(); + + // Step 2. Get the story name + String fullPath = new StringBuilder(assetLocation) + .append("/") + .append(fileNameBuilder) + .toString(); + + AudioDataStorage.saveAudioData(fullPath); + } + + public void clearAudioData() { + AudioDataStorage.clearAudioData(); } } From dbe18c5a4f9ac62c9cc763f7a39a91c8f6a0b75b Mon Sep 17 00:00:00 2001 From: chirag03k Date: Sun, 12 Jul 2020 23:27:22 -0700 Subject: [PATCH 04/45] Added functionality to return to the start of the line if something is wrong --- .../tutors/story_reading/animator_graph.json | 33 +++++++++++++++++-- .../rt_component/CRt_ViewManagerASB.java | 5 +++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/app/src/main/assets/tutors/story_reading/animator_graph.json b/app/src/main/assets/tutors/story_reading/animator_graph.json index 7f2a07bf6..81f7b4387 100644 --- a/app/src/main/assets/tutors/story_reading/animator_graph.json +++ b/app/src/main/assets/tutors/story_reading/animator_graph.json @@ -231,6 +231,7 @@ "preenter": [], "preexit": [], "edges": [ + {"constraint": "NARRATE_MODE", "edge": "NARRATE_RIGHT"}, {"constraint": "", "edge": "LISTEN"} ] }, @@ -278,7 +279,7 @@ "preenter": [], "preexit": [], "edges": [ - {"constraint": "", "edge": "NEXT_SENTENCE"} + {"constraint": "", "edge": "LISTEN"} ] }, @@ -291,7 +292,19 @@ "preenter": [], "preexit": [], "edges": [ - {"constraint": "","edge": ""} //TODO: Go to previous sentence.. how? + {"constraint": "","edge": "RESTART_LINE_NODE"} //TODO: Go to previous sentence.. how? + ] + }, + + "RESTART_LINE_NODE": { + "type": "NODE", + "COMMENT": "Goes back to the beginning of the line", + "maptype": "moduleMap", + "mapname": "START_LINE", + "preenter": [], + "preexit": [], + "edges": [ + {"constraint": "", "edge": "LISTEN"} ] } }, @@ -461,6 +474,15 @@ {"type": "COMMAND", "id": "SstoryReading", "method": "setHighLight", "parms": "red:String"}, {"type": "COMMAND", "id": "SstoryReading", "method": "continueListening" , "features": ""} ] + }, + + "START_LINE": { + "type": "MODULE", + "reuse": true, + "COMMENT": "goes back to the beginning of the sentence", + "tracks": [ + {"type": "COMMAND", "id": "SstoryReading", "method": "startLine", "pamrms": "none"} + ] } }, @@ -610,6 +632,13 @@ "FTR_WRONG": { "type": "CONDITION", "test": "FTR_WRONG" + }, + + "NARRATE_MODE": { + "type": "CONDITION", + "If": "{{SstoryReading.narrateState}}=='TRUE'", + "Then": "true", + "Else": "false" } } } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java index f7c627cbb..ca1297c97 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java @@ -1666,4 +1666,9 @@ public void saveToFile() { public void clearAudioData() { AudioDataStorage.clearAudioData(); } + + private void startLine() { + // Goes back to the beginning of the line + seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, TCONST.ZERO); + } } From 3a2ac22ac90c7f161bbfb1e3eb1886e8186560c7 Mon Sep 17 00:00:00 2001 From: chirag03k Date: Sun, 12 Jul 2020 23:39:51 -0700 Subject: [PATCH 05/45] Made sure that methods were corroborated by the interfaces implemented by CRt_ViewManagerASB.java --- .../tutors/story_reading/animator_graph.json | 2 +- .../tutorengine/widgets/core/TRtComponent.java | 15 +++++++++++++++ .../xprize/rt_component/CRt_ViewManagerASB.java | 6 ++++-- .../cmu/xprize/rt_component/ICRt_ViewManager.java | 4 ++++ .../cmu/xprize/rt_component/IRtComponent.java | 4 ++++ 5 files changed, 28 insertions(+), 3 deletions(-) diff --git a/app/src/main/assets/tutors/story_reading/animator_graph.json b/app/src/main/assets/tutors/story_reading/animator_graph.json index 81f7b4387..1451e4d5f 100644 --- a/app/src/main/assets/tutors/story_reading/animator_graph.json +++ b/app/src/main/assets/tutors/story_reading/animator_graph.json @@ -292,7 +292,7 @@ "preenter": [], "preexit": [], "edges": [ - {"constraint": "","edge": "RESTART_LINE_NODE"} //TODO: Go to previous sentence.. how? + {"constraint": "","edge": "RESTART_LINE_NODE"} ] }, diff --git a/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java b/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java index 9dd1f0199..5088a0b52 100644 --- a/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java +++ b/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java @@ -1196,4 +1196,19 @@ private void trackAndLogPerformance(String task, boolean correct) { RoboTutor.perfLogManager.postPerformanceLog(event); } + + @Override + public void saveToFile() { + mViewManager.saveToFile(); + } + + @Override + public void clearAudioData() { + mViewManager.clearAudioData(); + } + + @Override + public void startLine() { + mViewManager.startLine(); + } } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java index ca1297c97..ce1a8b14d 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java @@ -1642,7 +1642,7 @@ public void loadJSON(JSONObject jsonData, IScope scope) { JSON_Helper.parseSelf(jsonData, this, CClassMap.classMap, scope); } - // TODO: make this method + @Override public void saveToFile() { // Step 1. get sentence text StringBuilder fileNameBuilder = new StringBuilder(); @@ -1663,11 +1663,13 @@ public void saveToFile() { AudioDataStorage.saveAudioData(fullPath); } + @Override public void clearAudioData() { AudioDataStorage.clearAudioData(); } - private void startLine() { + @Override + public void startLine() { // Goes back to the beginning of the line seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, TCONST.ZERO); } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java b/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java index a41ce0916..f7814ce53 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java @@ -75,4 +75,8 @@ public interface ICRt_ViewManager extends ILoadableObject { public void setPageFlipButton(String command); public void execCommand(String _command, Object _target); + + void saveToFile(); + void clearAudioData(); + void startLine(); } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java b/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java index 36fd9b1b1..863dfdfb4 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java @@ -54,4 +54,8 @@ public interface IRtComponent { public void continueListening(); + void saveToFile(); + void clearAudioData(); + void startLine(); + } From 5ec981dd42bb9b1127a937039feca6fec2d71196 Mon Sep 17 00:00:00 2001 From: chirag03k Date: Sun, 19 Jul 2020 12:52:19 -0700 Subject: [PATCH 06/45] Enabled Narrate Mode Through Config File Added Config.json variable - content_creation_mode in order to enable narrate mode Added empty methods to CRt_ViewManagerMari.java - they may need to be filled in if this class is actually used Added FTR_NARRATE_MODE to TCONST --- .../tutors/story_reading/animator_graph.json | 6 ++---- .../startup/configuration/Configuration.java | 6 ++++++ .../startup/configuration/ConfigurationItems.java | 6 +++++- .../configuration/ConfigurationQuickOptions.java | 6 ++++-- .../edu/cmu/xprize/listener/SpeechRecognizer.java | 2 +- .../xprize/rt_component/CRt_ViewManagerASB.java | 11 +++++++++-- .../xprize/rt_component/CRt_ViewManagerMari.java | 15 +++++++++++++++ util/src/main/java/cmu/xprize/util/TCONST.java | 3 +++ 8 files changed, 45 insertions(+), 10 deletions(-) diff --git a/app/src/main/assets/tutors/story_reading/animator_graph.json b/app/src/main/assets/tutors/story_reading/animator_graph.json index 1451e4d5f..b50fd3193 100644 --- a/app/src/main/assets/tutors/story_reading/animator_graph.json +++ b/app/src/main/assets/tutors/story_reading/animator_graph.json @@ -343,7 +343,7 @@ {"type": "AUDIO", "command": "PLAY", "soundsource": "Please read aloud.mp3", "mode": "flow", "features": "!FTR_PROMPT&FTR_USER_READ|!FTR_PROMPT&FTR_USER_ECHO|!FTR_PROMPT&FTR_USER_REVEAL"}, //{"type": "AUDIO", "command": "PLAY", "soundsource": "Now lets listen to a story.mp3", "mode": "flow", "features": "!FTR_PROMPT&FTR_USER_HEAR|!FTR_PROMPT&FTR_USER_HIDE"}, {"type": "AUDIO", "command": "PLAY", "soundsource": "Listen carefully.mp3", "mode": "flow", "features": "!FTR_PROMPT&FTR_USER_HEAR|!FTR_PROMPT&FTR_USER_HIDE"}, - {"type": "AUDIO", "command": "PLAY", "soundsource": "Please listen and repeat after me.mp3", "mode": "flow", "features": "!FTR_PROMPT&FTR_USER_PARROT"} + {"type": "AUDIO", "command": "PLAY", "soundsource": "Please listen and repeat after me.mp3", "mode": "flow", "features": "!FTR_PROMPT&FTR_USER_PARROT"} ] }, @@ -636,9 +636,7 @@ "NARRATE_MODE": { "type": "CONDITION", - "If": "{{SstoryReading.narrateState}}=='TRUE'", - "Then": "true", - "Else": "false" + "test": "NARRATE_MODE" } } } diff --git a/app/src/main/java/cmu/xprize/robotutor/startup/configuration/Configuration.java b/app/src/main/java/cmu/xprize/robotutor/startup/configuration/Configuration.java index 6b94ab2bd..28d9228ad 100644 --- a/app/src/main/java/cmu/xprize/robotutor/startup/configuration/Configuration.java +++ b/app/src/main/java/cmu/xprize/robotutor/startup/configuration/Configuration.java @@ -23,6 +23,7 @@ public static void saveConfigurationItems(Context context, ConfigurationItems co .putBoolean(ConfigurationItems.USE_PLACEMENT, configItems.use_placement) .putBoolean(ConfigurationItems.RECORD_AUDIO, configItems.record_audio) .putString(ConfigurationItems.MENU_TYPE, configItems.menu_type) + .putBoolean(ConfigurationItems.CONTENT_CREATION_MODE, configItems.content_creation_mode) .apply(); } @@ -80,4 +81,9 @@ public static String getMenuType(Context context) { return context.getSharedPreferences(ROBOTUTOR_CONFIGURATION, MODE_PRIVATE) .getString(ConfigurationItems.MENU_TYPE, "CD1"); } + + public static boolean getContentCreationMode(Context context) { + return context.getSharedPreferences(ROBOTUTOR_CONFIGURATION, MODE_PRIVATE) + .getBoolean(ConfigurationItems.CONTENT_CREATION_MODE, false); + } } diff --git a/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationItems.java b/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationItems.java index 3a490f0ce..c63ed1aec 100644 --- a/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationItems.java +++ b/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationItems.java @@ -25,6 +25,7 @@ public class ConfigurationItems implements ILoadableObject { public static final String USE_PLACEMENT = "USE_PLACEMENT"; public static final String RECORD_AUDIO = "RECORD_AUDIO"; public static final String MENU_TYPE = "MENU_TYPE"; + public static final String CONTENT_CREATION_MODE = "CONTENT_CREATION_MODE"; public String config_version; public boolean language_override; @@ -37,6 +38,7 @@ public class ConfigurationItems implements ILoadableObject { public boolean use_placement; public boolean record_audio; public String menu_type; + public boolean content_creation_mode; public ConfigurationItems() { String dataPath = TCONST.DOWNLOAD_PATH + "/config.json"; @@ -56,7 +58,7 @@ public ConfigurationItems(String config_version, boolean language_override, boolean language_switcher, boolean no_asr_apps, String language_feature_id, boolean show_demo_vids, boolean use_placement, boolean record_audio, - String menu_type) { + String menu_type, boolean content_creation_mode) { this.config_version = config_version; this.language_override = language_override; @@ -69,6 +71,7 @@ public ConfigurationItems(String config_version, boolean language_override, this.use_placement = use_placement; this.record_audio = record_audio; this.menu_type = menu_type; + this.content_creation_mode = content_creation_mode; } public void setDefaults() { @@ -84,6 +87,7 @@ public void setDefaults() { use_placement = true; record_audio = false; menu_type = "CD1"; + content_creation_mode = false; } @Override diff --git a/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationQuickOptions.java b/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationQuickOptions.java index aa6f13616..d1341130c 100644 --- a/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationQuickOptions.java +++ b/app/src/main/java/cmu/xprize/robotutor/startup/configuration/ConfigurationQuickOptions.java @@ -22,7 +22,8 @@ public class ConfigurationQuickOptions { false, false, false, - "CD1" + "CD1", + false ); // EN version, and they both have the debugger menu. @@ -37,6 +38,7 @@ public class ConfigurationQuickOptions { false, false, false, - "CD1" + "CD1", + false ); } diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java index fc0fe60f3..a552f5e2f 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java @@ -850,7 +850,7 @@ public static void convertRawToWav(File rawFile, File wavFile) { output = new FileOutputStream(wavFile); // first write appropriate wave file header ByteArrayOutputStream hdrBytes = new ByteArrayOutputStream(); - new WaveHeader(WaveHeader.FORMAT_PCM, (short) 1, 16000, (short) 16, (int) rawFile.length()).write(hdrBytes); + new WaveHeader(WaveHeader.FORMAT_PCM, (short) 1, 16000, (short) 16, (int) rawFile.length()).write(hdrBytes); output.write(hdrBytes.toByteArray()); // then copy raw bytes to output file byte[] buffer = new byte[4096]; diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java index ce1a8b14d..250eec66c 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java @@ -217,6 +217,12 @@ public void initStory(IVManListener owner, String assetPath, String location) { //TODO: CHECK mParent.animatePageFlip(true,mCurrViewIndex); + + // Turns on narrate mode if it is specified in Config.json + if (mContext.getSharedPreferences("ROBOTUTOR CONFIGURATION", Context.MODE_PRIVATE) + .getBoolean("CONTENT_CREATION_MODE", false)) { + mParent.setFeature(TCONST.FTR_NARRATE_MODE, TCONST.ADD_FEATURE); + } } @@ -950,6 +956,7 @@ private void publishStateValues() { // mParent.publishValue(TCONST.RTC_VAR_ECHOSTATE, TCONST.FALSE); mParent.publishValue(TCONST.RTC_VAR_PARROTSTATE, TCONST.FALSE); + mParent.publishValue(TCONST.RTC_VAR_NARRATESTATE, TCONST.FALSE); if (prompt != null) { mParent.publishValue(TCONST.RTC_VAR_PROMPT, prompt); @@ -1646,11 +1653,11 @@ public void loadJSON(JSONObject jsonData, IScope scope) { public void saveToFile() { // Step 1. get sentence text StringBuilder fileNameBuilder = new StringBuilder(); - for (String word : wordsToDisplay) { // This uses the wordsToDisplay String[] because it contains punctuation + for (String word : wordsToSpeak) { // This uses the wordsToDisplay String[] because it contains punctuation fileNameBuilder.append(word).append(" "); } fileNameBuilder.setLength(fileNameBuilder.length() - 1); - fileNameBuilder.append(".mp3"); + fileNameBuilder.append(".pcm"); String fileName = fileNameBuilder.toString(); diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerMari.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerMari.java index 6e9d2019f..f68479c0a 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerMari.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerMari.java @@ -203,6 +203,21 @@ public void execCommand(String command, Object target ) { } } + @Override + public void saveToFile() { + + } + + @Override + public void clearAudioData() { + + } + + @Override + public void startLine() { + + } + @Override public void onUpdate(ListenerBase.HeardWord[] heardWords, boolean finalResult) { diff --git a/util/src/main/java/cmu/xprize/util/TCONST.java b/util/src/main/java/cmu/xprize/util/TCONST.java index 05513cd43..475692833 100644 --- a/util/src/main/java/cmu/xprize/util/TCONST.java +++ b/util/src/main/java/cmu/xprize/util/TCONST.java @@ -225,6 +225,7 @@ public class TCONST { public static final String FTR_USER_REVEAL = "FTR_USER_REVEAL"; public static final String FTR_USER_PARROT = "FTR_USER_PARROT"; public static final String FTR_USER_READING = "FTR_USER_READING"; + public static final String FTR_NARRATE_MODE = "NARRATE_MODE"; // UHQ public static final String FTR_GEN = "FTR_GEN"; public static final String FTR_PIC = "FTR_PIC"; @@ -839,6 +840,8 @@ public class TCONST { public static final String RTC_VAR_ECHOSTATE = ".echoState"; public static final String RTC_VAR_PARROTSTATE = ".parrotState"; + public static final String RTC_VAR_NARRATESTATE = ".narrateState"; + // Generic question state flag public static final String RTC_VAR_QUESTIONSTATE = ".questionState"; public static final String RTC_VAR_CLOZESTATE = ".clozeState"; From 515ead086c3f27a4190fbde14412b50db351a6f0 Mon Sep 17 00:00:00 2001 From: chirag03k Date: Tue, 21 Jul 2020 14:44:19 -0700 Subject: [PATCH 07/45] Merge remote-tracking branch 'remotes/origin/launch_specified_totor' into narrate_mode --- README.md | 5 ++- build.gradle | 4 ++ .../cmu/xprize/listener/AudioDataStorage.java | 39 +++++++++++++++---- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d8eda2c7b..1a89a212e 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ # **RoboTutor** -Welcome to RoboTutor_2020: XPRIZE's repo has the version of RoboTutor uploaded on 11/20/2018, but RoboTutor has been updated here since then. **This is the newest version.** +Welcome to RoboTutor: XPRIZE's repo has the version uploaded on 11/20/2018, but RoboTutor has been updated here since then. +For changes since 11/20/2018, see [https://github.com/RoboTutorLLC/RoboTutor](https://github.com/RoboTutorLLC/RoboTutor). For changes prior to 3/16/2020, see [https://github.com/RoboTutorLLC/RoboTutor_2019](https://github.com/RoboTutorLLC/RoboTutor_2019). However, it's no longer the newest. @@ -11,7 +12,7 @@ For changes prior to 3/16/2020, see [https://github.com/RoboTutorLLC/RoboTutor_2 ## Quick Installation To quickly install the most recent version of RoboTutor without having to download the full source code, follow these steps: -1. Go to [this Google Drive folder](https://drive.google.com/drive/u/1/folders/1VyajTK_SShmBB4GXJ74737pBS_IKunL_)(updated 8/26/2020). +1. Go to [this Google Drive folder](https://docs.google.com/document/d/1YoVx1K0LdHVNayiFUhkgffE5v28SPcg0FGIvaU84A_U/edit#) (updated 6/16/2020). 2. Download the APK to your tablet (do not install yet). diff --git a/build.gradle b/build.gradle index ea1343843..9245dc691 100644 --- a/build.gradle +++ b/build.gradle @@ -40,7 +40,11 @@ allprojects { // +<<<<<<< HEAD rtVersionName="3.4.2.1" //TODO: move this out of build so we don't have to rebuild to unzip assets +======= + rtVersionName="3.3.1.2" +>>>>>>> 4b280492... Merge remote-tracking branch 'remotes/origin/launch_specified_totor' into narrate_mode } } diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java index 1d6fc8bf3..27eea7222 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java @@ -16,9 +16,10 @@ public class AudioDataStorage { * finIndex is used to add to audioData without overwriting existing data * Volatile because it is accessed by the RecognizerThread but also by the _______ thread */ - public static volatile short[] audioData = new short[160 * 60 * 100]; // creates a buffer that can store 60 seconds worth of audio + public static final int MAX_LEN = 160* 60 * 100; // can store 60 seconds worth of audio + public static volatile short[] audioData = new short[MAX_LEN]; public static volatile int finIndex = 0; - + public static final int HEADER_LENGTH = 44; // I initialize the array and add to it @@ -41,18 +42,42 @@ public static synchronized void clearAudioData() { public static void saveAudioData(String filepath) { try { - FileOutputStream os = new FileOutputStream(filepath + ".pcm"); - ByteBuffer dataBuffer = ByteBuffer.allocate(audioData.length * 2); - ShortBuffer dataBufferShort = dataBuffer.asShortBuffer(); + // Remove trailing silence + short[] trimmedAudio = null; + for(int x = audioData.length - 1; x > 0; x++) { + if (audioData[x] != 0) { + trimmedAudio = new short[x + 1]; + for(int y = 0; y> 8) & 0xff), (byte) ((dataLen >> 16) & 0xff),(byte) ((dataLen >> 24) & 0xff), + 'W','A','V','E','f','m','t',' ', (byte) 16, 0, 0, 1, 0, 1, 0, }; + dataBuffer.put(header); */ + + ShortBuffer dataBufferShort = dataBuffer.asShortBuffer(); + dataBufferShort.put(trimmedAudio); + FileChannel out = os.getChannel(); + out.write(dataBuffer); out.close(); os.close(); - // TODO: now it needs to be converted to either .wav or .mp3. I think the header can take care of that } catch (IOException e) { e.printStackTrace(); // ? + } catch(NullPointerException e) { + e.printStackTrace(); + // Log empty audio } } From d9317526645ea3432f313b347df3a81e3b4914ea Mon Sep 17 00:00:00 2001 From: chirag03k Date: Mon, 3 Aug 2020 12:02:13 -0700 Subject: [PATCH 08/45] Implemented 'Mean Mode' except UI - Added ability to launch tutor using debug.json - Finished saving feature (storydata.json is edited accordingly) - Added segmentation information - Added necessary framework for story navigation (UI needs to be made) --- .../robotutor/tutorengine/CDebugLauncher.java | 93 ++++++++++++++++- .../widgets/core/TRtComponent.java | 22 +++++ .../tutor_xmatrices/StoryTransitions.en.csv | 2 +- .../cmu/xprize/listener/AudioDataStorage.java | 99 ++++++++++++++++--- .../edu/cmu/xprize/listener/ListenerBase.java | 2 +- .../edu/cmu/xprize/listener/ListenerPLRT.java | 1 + .../rt_component/CRt_ViewManagerASB.java | 22 +++-- .../xprize/rt_component/ICRt_ViewManager.java | 8 +- .../cmu/xprize/rt_component/IRtComponent.java | 2 +- .../src/main/java/cmu/xprize/util/TCONST.java | 1 + 10 files changed, 225 insertions(+), 27 deletions(-) diff --git a/app/src/main/java/cmu/xprize/robotutor/tutorengine/CDebugLauncher.java b/app/src/main/java/cmu/xprize/robotutor/tutorengine/CDebugLauncher.java index 20773e5a4..947987cba 100644 --- a/app/src/main/java/cmu/xprize/robotutor/tutorengine/CDebugLauncher.java +++ b/app/src/main/java/cmu/xprize/robotutor/tutorengine/CDebugLauncher.java @@ -22,6 +22,9 @@ public class CDebugLauncher { String tutorId; String matrix; + // For content_creation_mode + String storyDataPath; + public Boolean launchIfDebug() { try { InputStream inputStream = new FileInputStream("/sdcard/Download/debug.json"); @@ -47,6 +50,7 @@ public Boolean launchIfDebug() { this.tutorId = mResult.get("tutor_id"); this.matrix = mResult.get("skill1"); + this.storyDataPath = mResult.get("story_data_path"); return true; } catch (Exception e) { Log.wtf("CDebugLauncher", "/sdcard/Download/debug.json does not exist"); @@ -74,4 +78,91 @@ public String getTutorId() { public String getMatrix() { return this.matrix; } -} \ No newline at end of file + + public Integer getNext_node_times() { + Integer t = this.next_node_times; + this.next_node_times = 0; + return t; + } + + /* + modify the datasource + */ + public String hijackJsonData(String jsonData){ + // not in debug mod + if(!this.launchIfDebug()) + return jsonData; + + Log.wtf("DebugLauncher", "next_node_times: " + next_node_times.toString()); + Log.wtf("DebugLauncher", "old jsonData: " + jsonData); + + String[] list_name_to_truncate = {"data", "dataSource"}; + for(String list_name: list_name_to_truncate){ + Log.wtf("DebugLauncher", "now truncating: " + list_name); + + // find fist "gen_responseSet": [ ... + Integer start = jsonData.indexOf(list_name); + if(start == -1){ // not found + continue; + } + Log.wtf("DebugLauncher", "found at: "+ start.toString()); + start = jsonData.indexOf('[', start) + 1; + + // find corresponding ] + Integer depth = 1, cur = start; + while (depth >= 1){ + if (jsonData.charAt(cur) == ']'){ + depth -= 1; + } else if (jsonData.charAt(cur) == '['){ + depth += 1; + } + cur += 1; + } + cur -= 1; // go back before [ + + // get start middle and end part of json + String newJsonData_middle = jsonData.substring(start, cur); + String newJsonData_start = jsonData.substring(0, start); + String newJsonData_end = jsonData.substring(cur); + + // truncate middle part + cur = 0; + Integer skip_times_ = this.next_node_times; + while(skip_times_ > 0){ + depth = 0; + + while (true){ + // Log.wtf("DebugLauncher", newJsonData_middle.substring(cur)); + if (newJsonData_middle.charAt(cur) == ',' && depth == 0){ + break; + } + + if (newJsonData_middle.charAt(cur) == '{' || + newJsonData_middle.charAt(cur) == '['){ + depth += 1; + } + + if (newJsonData_middle.charAt(cur) == '}' || + newJsonData_middle.charAt(cur) == ']'){ + depth -= 1; + } + cur += 1; + } + + skip_times_ -= 1; + cur += 1; // skip this comma + } + newJsonData_middle = newJsonData_middle.substring(cur); + + // apply modification to jsonData + Log.wtf("DebugLauncher", "newJsonData_start: " + newJsonData_start); + Log.wtf("DebugLauncher", "newJsonData_middle: " + newJsonData_middle); + Log.wtf("DebugLauncher", "newJsonData_end: " + newJsonData_end); + jsonData = newJsonData_start + newJsonData_middle + newJsonData_end; + } + + Log.wtf("DebugLauncher", "new jsonData: " + jsonData); + return jsonData; + } + +} diff --git a/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java b/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java index 5088a0b52..afa9860c6 100644 --- a/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java +++ b/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java @@ -730,6 +730,25 @@ else if (dataNameDescriptor.startsWith(TCONST.SOURCEFILE)) { } else if (dataNameDescriptor.startsWith("{")) { loadJSON(new JSONObject(dataNameDescriptor), null); + } else if (dataNameDescriptor.startsWith(TCONST.UNPACKAGED_ASSET)) { + + // This is for assets that haven't yet been completed (e.g. stories that need narrations) and exists for content creation + String storyFolder = dataNameDescriptor.substring(TCONST.UNPACKAGED_ASSET.length()).toLowerCase(); + + String[] levelval = storyFolder.split("_"); + + String levelFolder = levelval[0]; + + DATASOURCEPATH = TCONST.DOWNLOAD_PATH + "/"; + STORYSOURCEPATH = DATASOURCEPATH + levelFolder + "/" + storyFolder + "/"; + + AUDIOSOURCEPATH = TCONST.DOWNLOAD_PATH + "/" + levelFolder + "/" + storyFolder; + // Probably going to change this to put all the audio in one place + + configListenerLanguage(mMediaManager.getLanguageFeature(mTutor)); + mMediaManager.addSoundPackage(mTutor, MEDIA_STORY, new CMediaPackage(LANG_AUTO, AUDIOSOURCEPATH, LOCAL_STORY_AUDIO)); + + loadStory(STORYSOURCEPATH, "ASB_Data", TCONST.EXTERN); } else { throw (new Exception("BadDataSource")); @@ -1211,4 +1230,7 @@ public void clearAudioData() { public void startLine() { mViewManager.startLine(); } + + @Override + public void prevSentence() {mViewManager.prevSentence();} } diff --git a/app/src/tutor_xmatrices/StoryTransitions.en.csv b/app/src/tutor_xmatrices/StoryTransitions.en.csv index d14334c7d..3303aee76 100644 --- a/app/src/tutor_xmatrices/StoryTransitions.en.csv +++ b/app/src/tutor_xmatrices/StoryTransitions.en.csv @@ -1,2 +1,2 @@ -story.hear::1_1,story.echo::1_1,story.read::1_1,story.hear::1_2,story.echo::1_2,story.read::1_2,story.hear::1_3,story.echo::1_3,story.read::1_3,story.hear::1_11,story.echo::1_11,story.read::1_11,story.hear::1_12,story.echo::1_12,story.read::1_12,story.hear::1_13,story.echo::1_13,story.read::1_13,story.hear::1_14,story.echo::1_14,story.read::1_14,story.hear::1_15,story.echo::1_15,story.read::1_15,story.hear::1_16,story.echo::1_16,story.read::1_16,story.hear::1_17,story.echo::1_17,story.read::1_17,story.hear::1_18,story.echo::1_18,story.read::1_18,story.hear::1_19,story.echo::1_19,story.read::1_19,story.hear::1_20,story.echo::1_20,story.read::1_20,story.hear::1_21,story.echo::1_21,story.read::1_21,story.hear::1_22,story.echo::1_22,story.read::1_22,story.hear::1_23,story.echo::1_23,story.read::1_23,story.hear::1_24,story.echo::1_24,story.read::1_24,story.hear::1_25,story.echo::1_25,story.read::1_25,story.hear::1_26,story.echo::1_26,story.read::1_26,story.hear::1_27,story.echo::1_27,story.read::1_27,story.hear::1_28,story.echo::1_28,story.read::1_28,story.hear::1_29,story.echo::1_29,story.read::1_29,story.hear::1_30,story.echo::1_30,story.read::1_30,story.hear::1_31,story.echo::1_31,story.read::1_31,story.hear::1_32,story.echo::1_32,story.read::1_32,story.hear::1_34,story.echo::1_34,story.read::1_34,story.hear::1_35,story.echo::1_35,story.read::1_35,story.hear::1_36,story.echo::1_36,story.read::1_36,story.hear::1_37,story.echo::1_37,story.read::1_37 +story.hear::1_1,story.echo::1_1,story.read::1_1,story.hear::1_2,story.echo::1_2,story.read::1_2,story.hear::1 _3,story.echo::1_3,story.read::1_3,story.hear::1_11,story.echo::1_11,story.read::1_11,story.hear::1_12,story.echo::1_12,story.read::1_12,story.hear::1_13,story.echo::1_13,story.read::1_13,story.hear::1_14,story.echo::1_14,story.read::1_14,story.hear::1_15,story.echo::1_15,story.read::1_15,story.hear::1_16,story.echo::1_16,story.read::1_16,story.hear::1_17,story.echo::1_17,story.read::1_17,story.hear::1_18,story.echo::1_18,story.read::1_18,story.hear::1_19,story.echo::1_19,story.read::1_19,story.hear::1_20,story.echo::1_20,story.read::1_20,story.hear::1_21,story.echo::1_21,story.read::1_21,story.hear::1_22,story.echo::1_22,story.read::1_22,story.hear::1_23,story.echo::1_23,story.read::1_23,story.hear::1_24,story.echo::1_24,story.read::1_24,story.hear::1_25,story.echo::1_25,story.read::1_25,story.hear::1_26,story.echo::1_26,story.read::1_26,story.hear::1_27,story.echo::1_27,story.read::1_27,story.hear::1_28,story.echo::1_28,story.read::1_28,story.hear::1_29,story.echo::1_29,story.read::1_29,story.hear::1_30,story.echo::1_30,story.read::1_30,story.hear::1_31,story.echo::1_31,story.read::1_31,story.hear::1_32,story.echo::1_32,story.read::1_32,story.hear::1_34,story.echo::1_34,story.read::1_34,story.hear::1_35,story.echo::1_35,story.read::1_35,story.hear::1_36,story.echo::1_36,story.read::1_36,story.hear::1_37,story.echo::1_37,story.read::1_37 story.hear::2_1,story.echo::2_1,story.read::2_1,story.hear::2_2,story.echo::2_2,story.read::2_2,story.hear::2_3,story.echo::2_3,story.read::2_3,story.hear::2_4,story.echo::2_4,story.read::2_4,story.hear::2_5,story.echo::2_5,story.read::2_5,story.hear::2_6,story.echo::2_6,story.read::2_6,story.hear::2_7,story.echo::2_7,story.read::2_7,story.hear::2_8,story.echo::2_8,story.read::2_8,story.hear::2_9,story.echo::2_9,story.read::2_9,story.hear::2_10,story.echo::2_10,story.read::2_10,story.hear::2_11,story.echo::2_11,story.read::2_11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java index 27eea7222..ff37763f9 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java @@ -1,13 +1,17 @@ package edu.cmu.xprize.listener; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.ShortBuffer; import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; import java.util.Arrays; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; public class AudioDataStorage { @@ -20,10 +24,16 @@ public class AudioDataStorage { public static volatile short[] audioData = new short[MAX_LEN]; public static volatile int finIndex = 0; public static final int HEADER_LENGTH = 44; + public static List segmentation = new ArrayList(); + static JSONObject storyData; + static int currentSentence = 0; - // I initialize the array and add to it + public static void initStoryData(JSONObject JSONObj) { + storyData = JSONObj; + } - public static synchronized void addAudioData(short[] buffer) { + // I initialize the array and add to it + static synchronized void addAudioData(short[] buffer) { try { for (short s : buffer) { audioData[finIndex] = s; @@ -40,7 +50,11 @@ public static synchronized void clearAudioData() { finIndex = 0; } - public static void saveAudioData(String filepath) { + public static void saveAudioData(String fileName, String assetLocation) { + + String completeFilePath = assetLocation + "/" + fileName + ".wav"; + + // Write the audio to file try { // Remove trailing silence short[] trimmedAudio = null; @@ -54,31 +68,90 @@ public static void saveAudioData(String filepath) { } } - FileOutputStream os = new FileOutputStream(filepath + ".wav"); + + FileOutputStream os = new FileOutputStream(completeFilePath); int dataLen = trimmedAudio.length; ByteBuffer dataBuffer = ByteBuffer.allocate(dataLen * 2 /* the data */ + 44 /* The header */); dataBuffer.order(ByteOrder.LITTLE_ENDIAN); // I *think* little endian is what .wav uses, not sure about .mp3 - /* byte[] header = new byte[]{'R','I','F','F', + int sampleRate = 16000; + int channelNumber = 1; + long bitRate = sampleRate * channelNumber * 16; // sampleRate times number of Channels times the number of bits per sample + byte[] header = new byte[]{'R','I','F','F', (byte) (dataLen & 0xff), (byte) (byte) ((dataLen >> 8) & 0xff), (byte) ((dataLen >> 16) & 0xff),(byte) ((dataLen >> 24) & 0xff), - 'W','A','V','E','f','m','t',' ', (byte) 16, 0, 0, 1, 0, 1, 0, }; - dataBuffer.put(header); */ + 'W','A','V','E','f','m','t',' ', + (byte) 16, 0, 0, 1, 0, (byte) channelNumber, 0,(byte) (sampleRate & 0xff), (byte) ((sampleRate >> 8) & 0xff), (byte) ((sampleRate >> 16) & 0xff), (byte) ((sampleRate >> 24) & 0xff), + (byte) ((bitRate / 8) & 0xff), (byte) (((bitRate / 8) >> 8) & 0xff), (byte) (((bitRate / 8) >> 16) & 0xff), (byte) (((bitRate / 8) >> 24) & 0xff), (byte) ((channelNumber * 16) / 8), + 0, 16, 0, 'd', 'a', 't', 'a', (byte) (dataLen * 2 & 0xff), (byte) (((dataLen * 2) >> 8) & 0xff), (byte) (((dataLen * 2) >> 16) & 0xff), (byte) (((dataLen * 2) >> 24) & 0xff) + }; + dataBuffer.put(header); ShortBuffer dataBufferShort = dataBuffer.asShortBuffer(); - dataBufferShort.put(trimmedAudio); + dataBufferShort.put(trimmedAudio); // dataBuffer changes alongside dataBufferShort + FileChannel out = os.getChannel(); out.write(dataBuffer); out.close(); os.close(); - - } catch (IOException e) { + } catch (IOException | NullPointerException e) { e.printStackTrace(); // ? - } catch(NullPointerException e) { + } + + // write segmentation to file + try { + FileOutputStream os = new FileOutputStream(assetLocation + "/" + fileName + ".seg"); + StringBuilder segData = new StringBuilder(""); + + for(ListenerBase.HeardWord word: segmentation) { + segData.append(word.hypWord + "\t" + word.startTime + "\t" + word.startTime); + segData.append("\n"); + } + segData.deleteCharAt(segData.lastIndexOf("\n")); + + byte[] segBytes = segData.toString().getBytes(); + os.write(segBytes); + os.close(); + } catch(IOException | NullPointerException e) { e.printStackTrace(); - // Log empty audio } + + // Update Storydata.json + try { + boolean isSentence = true; + JSONObject sentenceData = storyData + .getJSONArray("data") + .getJSONObject(currentSentence) + .getJSONArray("text") + .getJSONArray(0) + .getJSONObject(0) + .getJSONArray("narration") + .getJSONObject(0); + + JSONArray segm = sentenceData.getJSONArray("segmentation"); + long finalEndTime = 0; + for(ListenerBase.HeardWord heardWord : segmentation) { + JSONObject segObj = new JSONObject(); + segObj.put("end", heardWord.endTime); + segObj.put("start", heardWord.startTime); + segObj.put("word", heardWord.hypWord); + segm.put(segObj); + finalEndTime = heardWord.endTime; + } + sentenceData.put("from", ""); // TODO: figure out what 'from' is supposed to represent + sentenceData.put("audio", completeFilePath); + sentenceData.put("until", finalEndTime); + sentenceData.put("utterance", fileName); + + } catch(JSONException e) { + + } + + segmentation.clear(); } + static void updateHypothesis(ListenerBase.HeardWord[] heardWords) { + segmentation.addAll(Arrays.asList(heardWords)); + } } diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerBase.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerBase.java index ff8e509c6..a46d7127e 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerBase.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerBase.java @@ -55,7 +55,7 @@ public class ListenerBase { static protected ListenerAssets assets; // created in init phase - protected String captureLabel = ""; // label for capture, logging files - protected boolean IS_LOGGING = false; + protected boolean IS_LOGGING = true; protected File configFile; // config file to use, null => default protected File modelsDir; // saved model directory diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerPLRT.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerPLRT.java index 497b9e6b6..4086b84f9 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerPLRT.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/ListenerPLRT.java @@ -480,6 +480,7 @@ private void processHypothesis(String[] asrWords, Boolean finalResult) { // log the partial hypothesis if(IS_LOGGING) logHyp(timestamp, TextUtils.join(" ", asrWords), segments, heardWords); + AudioDataStorage.updateHypothesis(heardWords); } } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java index 250eec66c..1b20fd959 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java @@ -1657,17 +1657,11 @@ public void saveToFile() { fileNameBuilder.append(word).append(" "); } fileNameBuilder.setLength(fileNameBuilder.length() - 1); - fileNameBuilder.append(".pcm"); String fileName = fileNameBuilder.toString(); - // Step 2. Get the story name - String fullPath = new StringBuilder(assetLocation) - .append("/") - .append(fileNameBuilder) - .toString(); - AudioDataStorage.saveAudioData(fullPath); + AudioDataStorage.saveAudioData(fileName, mAsset); } @Override @@ -1680,4 +1674,18 @@ public void startLine() { // Goes back to the beginning of the line seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, TCONST.ZERO); } + + @Override + public void prevSentence() { + // Not sure if it is zero-index or not. Right now treating the counting like it starts at 1 + if (mCurrLine > 2) { + mCurrLine--; + mCurrWord = 1; + } else if (mCurrPage > 2) { + mCurrPage--; + mCurrLine = 1; + mCurrWord = 1; + } + seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, mCurrWord); + } } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java b/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java index f7814ce53..84fb3372c 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java @@ -76,7 +76,9 @@ public interface ICRt_ViewManager extends ILoadableObject { public void execCommand(String _command, Object _target); - void saveToFile(); - void clearAudioData(); - void startLine(); + public void saveToFile(); + public void clearAudioData(); + public void startLine(); + + public void prevSentence(); } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java b/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java index 863dfdfb4..f084ec199 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java @@ -57,5 +57,5 @@ public interface IRtComponent { void saveToFile(); void clearAudioData(); void startLine(); - + void prevSentence(); } diff --git a/util/src/main/java/cmu/xprize/util/TCONST.java b/util/src/main/java/cmu/xprize/util/TCONST.java index 475692833..7f6b67cf9 100644 --- a/util/src/main/java/cmu/xprize/util/TCONST.java +++ b/util/src/main/java/cmu/xprize/util/TCONST.java @@ -205,6 +205,7 @@ public class TCONST { public static final String PUNC_STORY = "[punc]"; public static final String LOCAL_FILE = "[local_file]"; + public static final String UNPACKAGED_ASSET = "[unpackaged_asset]"; public static final String DOWNLOAD_PATH = "/sdcard/Download"; public static final String DOWNLOAD_RT_PATH = "/sdcard/Download/RoboTutor"; public static final String DOWNLOAD_RT_TUTOR = "/sdcard/Download/RoboTutor/assets"; From f2ce775fa4bd3220ab50a3c12f9f583172dbf901 Mon Sep 17 00:00:00 2001 From: chirag03k Date: Sat, 15 Aug 2020 19:28:54 -0700 Subject: [PATCH 09/45] Fixed issue with enabling narrate mode on startup --- app/build.gradle | 66 ++++---- .../tutors/story_reading/animator_graph.json | 8 +- .../robotutor/tutorengine/CDebugLauncher.java | 15 +- .../widgets/core/TRtComponent.java | 6 + comp_akira/build.gradle | 14 +- comp_ask/build.gradle | 12 +- comp_banner/build.gradle | 13 +- comp_bigmath/build.gradle | 16 +- comp_bubblepop/build.gradle | 9 + comp_clickmask/build.gradle | 8 +- comp_counting/build.gradle | 8 +- comp_counting2/build.gradle | 12 +- comp_debug/build.gradle | 14 +- comp_listener/build.gradle | 12 +- .../cmu/xprize/listener/AudioDataStorage.java | 141 +++++++++++++--- .../edu/cmu/xprize/listener/AudioWriter.java | 156 ++++++++++++++++++ .../cmu/xprize/listener/SpeechRecognizer.java | 4 +- comp_logging/build.gradle | 7 +- comp_ltkplus/build.gradle | 12 +- comp_math/build.gradle | 14 +- comp_nd/build.gradle | 14 +- comp_numberscale/build.gradle | 10 +- comp_picmatch/build.gradle | 12 +- comp_pointtap/build.gradle | 10 +- comp_questions/build.gradle | 16 +- comp_reading/build.gradle | 16 +- .../xprize/rt_component/CRt_Component.java | 3 + .../rt_component/CRt_ViewManagerASB.java | 43 ++++- .../rt_component/CRt_ViewManagerMari.java | 8 + .../xprize/rt_component/ICRt_ViewManager.java | 2 + .../cmu/xprize/rt_component/IRtComponent.java | 8 +- comp_scoreboard/build.gradle | 12 +- comp_session/build.gradle | 12 +- comp_spelling/build.gradle | 10 +- comp_writebox/build.gradle | 12 +- comp_writing/build.gradle | 16 +- fw_component/build.gradle | 10 +- mn_component/build.gradle | 12 +- nl_component/build.gradle | 14 +- resources/build.gradle | 6 +- sm_component/build.gradle | 12 +- util/build.gradle | 14 +- .../java/cmu/xprize/util/JSON_Helper.java | 2 +- 43 files changed, 573 insertions(+), 248 deletions(-) create mode 100644 comp_listener/src/main/java/edu/cmu/xprize/listener/AudioWriter.java diff --git a/app/build.gradle b/app/build.gradle index 6e921129b..2e2ab7dc1 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -45,9 +45,11 @@ android { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' } - debug { - debuggable true + /* + release_dbg { + debuggable true } + */ // This release includes the DEBUG selector and other various debug support features. // @@ -88,36 +90,36 @@ android { } dependencies { - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile 'com.android.support:percent:25.2.0' - compile 'com.google.code.gson:gson:2.8.0' - compile project(':comp_banner') - compile project(':comp_ltkplus') - compile project(':util') - compile project(':mn_component') - compile project(':comp_listener') - compile project(':comp_reading') - compile project(':comp_questions') - compile project(':sm_component') - compile project(':fw_component') - compile project(':nl_component') - compile project(':comp_math') - compile project(':comp_akira') - compile project(':comp_bubblepop') - compile project(':comp_writing') - compile project(':comp_pointtap') - compile project(path: ':comp_ask') - compile project(path: ':comp_session') - compile project(path: ':comp_debug') - compile project(':comp_logging') - compile project(path: ':comp_counting') - compile project(path: ':comp_counting2') - compile project(path: ':comp_numberscale') - compile project(path: ':comp_nd') - compile project(path: ':comp_picmatch') - compile project(path: ':comp_bigmath') - compile project(path: ':comp_spelling') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api 'com.android.support:percent:25.2.0' + api 'com.google.code.gson:gson:2.8.0' + api project(':comp_banner') + api project(':comp_ltkplus') + api project(':util') + api project(':mn_component') + api project(':comp_listener') + api project(':comp_reading') + api project(':comp_questions') + api project(':sm_component') + api project(':fw_component') + api project(':nl_component') + api project(':comp_math') + api project(':comp_akira') + api project(':comp_bubblepop') + api project(':comp_writing') + api project(':comp_pointtap') + api project(path: ':comp_ask') + api project(path: ':comp_session') + api project(path: ':comp_debug') + api project(':comp_logging') + api project(path: ':comp_counting') + api project(path: ':comp_counting2') + api project(path: ':comp_numberscale') + api project(path: ':comp_nd') + api project(path: ':comp_picmatch') + api project(path: ':comp_bigmath') + api project(path: ':comp_spelling') compile project(path: ':comp_intervention') } diff --git a/app/src/main/assets/tutors/story_reading/animator_graph.json b/app/src/main/assets/tutors/story_reading/animator_graph.json index b50fd3193..7f5e771b0 100644 --- a/app/src/main/assets/tutors/story_reading/animator_graph.json +++ b/app/src/main/assets/tutors/story_reading/animator_graph.json @@ -171,6 +171,7 @@ "mapname": "PLAYWRONG", "preexit": [], "edges": [ + {"constraint": "NARRATE_MODE", "edge": "NARRATE_WRONG"}, {"constraint": "", "edge": "LISTEN"} ] }, @@ -183,6 +184,7 @@ "mapname": "SPEAK_EVENT_MODULE", "preexit": [], "edges": [ + {"constraint": "NARRATE_MODE", "edge": "NARRATE_WRONG"}, {"constraint": "", "edge": "LISTEN"} ] }, @@ -244,6 +246,7 @@ "preenter": [], "preexit": [], "edges": [ + {"constraint": "NARRATE_MODE", "edge": "NARRATE_RIGHT"}, {"constraint": "", "edge": "LISTEN"} ] }, @@ -256,6 +259,7 @@ "preenter": [], "preexit": [], "edges": [ + {"constraint": "NARRATE_MODE", "edge": "NARRATE_RIGHT"}, {"constraint": "", "edge": "LISTEN"} ] }, @@ -317,7 +321,7 @@ "reuse": true, "COMMENT": "module that saves audio for NARRATE mode", "tracks" : [ - {"type": "COMMAND", "id": "SstoryReading", "method": "saveAudio", "parms": "none", "features": "" } + {"type": "COMMAND", "id": "SstoryReading", "method": "saveToFile", "parms": "none", "features": "" } ] }, @@ -326,7 +330,7 @@ "reuse": true, "COMMENT": "module that deletes current audio data", "tracks": [ - {"type": "COMMAND","id": "SstoryReading","method": "clearAudio","parms": "none","features": ""} + {"type": "COMMAND","id": "SstoryReading","method": "clearAudioData","parms": "none","features": ""} ] }, diff --git a/app/src/main/java/cmu/xprize/robotutor/tutorengine/CDebugLauncher.java b/app/src/main/java/cmu/xprize/robotutor/tutorengine/CDebugLauncher.java index 947987cba..dbd3beca3 100644 --- a/app/src/main/java/cmu/xprize/robotutor/tutorengine/CDebugLauncher.java +++ b/app/src/main/java/cmu/xprize/robotutor/tutorengine/CDebugLauncher.java @@ -1,6 +1,8 @@ package cmu.xprize.robotutor.tutorengine; +import android.os.Build; +import android.support.annotation.RequiresApi; import android.util.Log; import com.google.gson.Gson; @@ -10,8 +12,10 @@ import java.io.FileInputStream; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; // if /sdcard/Download/debug.json exists // bypass the activity selector and directly launch the tutor @@ -22,8 +26,6 @@ public class CDebugLauncher { String tutorId; String matrix; - // For content_creation_mode - String storyDataPath; public Boolean launchIfDebug() { try { @@ -49,12 +51,17 @@ public Boolean launchIfDebug() { this.dataSource = mResult.get("tutor_data"); this.tutorId = mResult.get("tutor_id"); this.matrix = mResult.get("skill1"); + //this.next_node_times = Integer.valueOf(mResult.get("next_node_times")); - this.storyDataPath = mResult.get("story_data_path"); return true; } catch (Exception e) { Log.wtf("CDebugLauncher", "/sdcard/Download/debug.json does not exist"); - + String stackString = ""; + for(StackTraceElement s : e.getStackTrace()) { + stackString += (s + "\n"); + } + Log.wtf("CDebugLauncher", stackString); + Log.wtf("CDebugLauncher", e.getClass().getName()); return false; } } diff --git a/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java b/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java index afa9860c6..5101ab94d 100644 --- a/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java +++ b/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java @@ -35,6 +35,8 @@ import cmu.xprize.comp_logging.ITutorLogger; import cmu.xprize.comp_logging.PerformanceLogItem; import cmu.xprize.robotutor.RoboTutor; +import cmu.xprize.robotutor.startup.configuration.Configuration; +import cmu.xprize.robotutor.tutorengine.CDebugLauncher; import cmu.xprize.robotutor.tutorengine.CMediaController; import cmu.xprize.robotutor.tutorengine.CMediaManager; import cmu.xprize.robotutor.tutorengine.CMediaPackage; @@ -465,6 +467,7 @@ public void logState(String logData) { extractFeatureContents(builder, _FeatureMap); RoboTutor.logManager.postTutorState(TUTOR_STATE_MSG, "target#reading_tutor," + logData + builder.toString()); + Log.d("TRt_Component", "Tutor is Listening"); } // ITutorLogger - End @@ -750,9 +753,12 @@ else if (dataNameDescriptor.startsWith(TCONST.SOURCEFILE)) { loadStory(STORYSOURCEPATH, "ASB_Data", TCONST.EXTERN); + } else { throw (new Exception("BadDataSource")); } + + enableNarrateMode(Configuration.getContentCreationMode(getContext())); } catch (Exception e) { CErrorManager.logEvent(TAG, "Invalid Data Source for : " + mTutor.getTutorName(), e, true); diff --git a/comp_akira/build.gradle b/comp_akira/build.gradle index 3f0804513..21a2378dc 100644 --- a/comp_akira/build.gradle +++ b/comp_akira/build.gradle @@ -30,11 +30,11 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - compile 'com.android.support:appcompat-v7:25.2.0' - compile 'com.android.support:percent:25.2.0' - testCompile 'junit:junit:4.12' - compile project(':util') - compile project(':comp_scoreboard') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + api 'com.android.support:appcompat-v7:25.2.0' + api 'com.android.support:percent:25.2.0' + testImplementation 'junit:junit:4.12' + api project(':util') + api project(':comp_scoreboard') + api project(':comp_logging') } diff --git a/comp_ask/build.gradle b/comp_ask/build.gradle index 6d32a0557..07c148552 100644 --- a/comp_ask/build.gradle +++ b/comp_ask/build.gradle @@ -20,13 +20,13 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') + api fileTree(include: ['*.jar'], dir: 'libs') androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile 'com.android.support:appcompat-v7:25.2.0' - testCompile 'junit:junit:4.12' - compile project(path: ':util') - compile project(':comp_logging') - compile project(':comp_debug') + api 'com.android.support:appcompat-v7:25.2.0' + testImplementation 'junit:junit:4.12' + api project(path: ':util') + api project(':comp_logging') + api project(':comp_debug') } diff --git a/comp_banner/build.gradle b/comp_banner/build.gradle index eee8978bd..96b8bce6c 100644 --- a/comp_banner/build.gradle +++ b/comp_banner/build.gradle @@ -29,11 +29,10 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile 'com.android.support:percent:25.2.0' - compile project(':util') - compile project(':comp_intervention') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api 'com.android.support:percent:25.2.0' + api project(':util') + api project(':comp_logging') } diff --git a/comp_bigmath/build.gradle b/comp_bigmath/build.gradle index 631a36e57..1b8b373c8 100644 --- a/comp_bigmath/build.gradle +++ b/comp_bigmath/build.gradle @@ -25,18 +25,18 @@ android { } dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) - compile 'com.android.support:appcompat-v7:26.1.0' - testCompile 'junit:junit:4.12' + api fileTree(dir: 'libs', include: ['*.jar']) + api 'com.android.support:appcompat-v7:26.1.0' + testImplementation 'junit:junit:4.12' androidTestCompile('com.android.support.test.espresso:espresso-core:3.0.2', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile 'com.android.support.constraint:constraint-layout:1.1.2' + api 'com.android.support.constraint:constraint-layout:1.1.2' - compile project(path: ':util') - compile project(path: ':comp_logging') - compile project(path: ':resources') - compile project(path: ':comp_writebox') + api project(path: ':util') + api project(path: ':comp_logging') + api project(path: ':resources') + api project(path: ':comp_writebox') } diff --git a/comp_bubblepop/build.gradle b/comp_bubblepop/build.gradle index 32b320cbd..ff2563cca 100644 --- a/comp_bubblepop/build.gradle +++ b/comp_bubblepop/build.gradle @@ -29,6 +29,7 @@ android { } dependencies { +<<<<<<< HEAD compile fileTree(include: ['*.jar'], dir: 'libs') testCompile 'junit:junit:4.12' compile 'com.android.support:appcompat-v7:25.2.0' @@ -36,4 +37,12 @@ dependencies { compile project(path: ':comp_intervention') compile project(':comp_logging') compile project(':comp_clickmask') +======= + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api project(path: ':util') + api project(':comp_logging') + api project(':comp_clickmask') +>>>>>>> eac62034... Fixed issue with enabling narrate mode on startup } diff --git a/comp_clickmask/build.gradle b/comp_clickmask/build.gradle index b9bb7073d..d96e67876 100644 --- a/comp_clickmask/build.gradle +++ b/comp_clickmask/build.gradle @@ -22,11 +22,11 @@ android { } dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) + api fileTree(dir: 'libs', include: ['*.jar']) androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile 'com.android.support:appcompat-v7:25.2.0' - testCompile 'junit:junit:4.12' - compile project(path: ':util') + api 'com.android.support:appcompat-v7:25.2.0' + testImplementation 'junit:junit:4.12' + api project(path: ':util') } diff --git a/comp_counting/build.gradle b/comp_counting/build.gradle index b9bb7073d..d96e67876 100644 --- a/comp_counting/build.gradle +++ b/comp_counting/build.gradle @@ -22,11 +22,11 @@ android { } dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) + api fileTree(dir: 'libs', include: ['*.jar']) androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile 'com.android.support:appcompat-v7:25.2.0' - testCompile 'junit:junit:4.12' - compile project(path: ':util') + api 'com.android.support:appcompat-v7:25.2.0' + testImplementation 'junit:junit:4.12' + api project(path: ':util') } diff --git a/comp_counting2/build.gradle b/comp_counting2/build.gradle index 790ea77be..5bddc140d 100644 --- a/comp_counting2/build.gradle +++ b/comp_counting2/build.gradle @@ -24,15 +24,15 @@ android { } dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) - compile 'com.android.support:appcompat-v7:25.2.0' - compile 'com.android.support:percent:25.2.0' + api fileTree(dir: 'libs', include: ['*.jar']) + api 'com.android.support:appcompat-v7:25.2.0' + api 'com.android.support:percent:25.2.0' - testCompile 'junit:junit:4.12' + testImplementation 'junit:junit:4.12' androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile project(path: ':util') + api project(path: ':util') - compile project(path: ':comp_ltkplus') + api project(path: ':comp_ltkplus') } diff --git a/comp_debug/build.gradle b/comp_debug/build.gradle index aa4fe4880..427c5ae3b 100644 --- a/comp_debug/build.gradle +++ b/comp_debug/build.gradle @@ -20,14 +20,14 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') + api fileTree(include: ['*.jar'], dir: 'libs') androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile 'com.android.support:appcompat-v7:25.2.0' - testCompile 'junit:junit:4.12' - compile 'com.android.support:percent:25.2.0' - compile project(path: ':util') - compile project(path: ':sm_component') - compile project(':comp_logging') + api 'com.android.support:appcompat-v7:25.2.0' + testImplementation 'junit:junit:4.12' + api 'com.android.support:percent:25.2.0' + api project(path: ':util') + api project(path: ':sm_component') + api project(':comp_logging') } diff --git a/comp_listener/build.gradle b/comp_listener/build.gradle index 26f85d3be..5b3e0fa2a 100644 --- a/comp_listener/build.gradle +++ b/comp_listener/build.gradle @@ -34,10 +34,10 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile project(':util') - compile files('libs/pocketsphinx-android-5prealpha-nolib.jar') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api project(':util') + api files('libs/pocketsphinx-android-5prealpha-nolib.jar') + api project(':comp_logging') } diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java index ff37763f9..986dec910 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java @@ -1,7 +1,11 @@ package edu.cmu.xprize.listener; +import android.util.Log; + +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.ShortBuffer; @@ -18,7 +22,7 @@ public class AudioDataStorage { /** * audioData stores the data * finIndex is used to add to audioData without overwriting existing data - * Volatile because it is accessed by the RecognizerThread but also by the _______ thread + * Volatile because it is a ccessed by the RecognizerThread but also by the _______ thread */ public static final int MAX_LEN = 160* 60 * 100; // can store 60 seconds worth of audio public static volatile short[] audioData = new short[MAX_LEN]; @@ -27,76 +31,154 @@ public class AudioDataStorage { public static List segmentation = new ArrayList(); static JSONObject storyData; static int currentSentence = 0; + static int sampleRate; + + static FileOutputStream outputStream; + static FileChannel outChannel; public static void initStoryData(JSONObject JSONObj) { storyData = JSONObj; } // I initialize the array and add to it - static synchronized void addAudioData(short[] buffer) { + static synchronized void addAudioData(int index, short[] buffer) { try { - for (short s : buffer) { - audioData[finIndex] = s; - finIndex++; + for(int i = 0; i < index; i++) { + audioData[finIndex + i] = buffer[i]; } + finIndex += index; + Log.d("AudioDataStorage", "Successfully stored Audio Data"); } catch (ArrayIndexOutOfBoundsException e) { // ?? // Should I save the audio file that we have and start writing to a new file? + Log.d("AudioDataStorage", "out of bounds. finIndex is " + finIndex); + } catch (Exception e) { + Log.d("AudioDataStorage", "Failed to capture audio"); } } public static synchronized void clearAudioData() { + Log.d("AudioDataStorage", "AudioData Cleared"); Arrays.fill(audioData, (short) 0); finIndex = 0; } public static void saveAudioData(String fileName, String assetLocation) { + Log.d("ADSSave", "attempting to save audiodata."); - String completeFilePath = assetLocation + "/" + fileName + ".wav"; + sampleRate = 16000; + String completeFilePath = assetLocation + fileName + "fail.wav"; + + Log.d("ADSSave", completeFilePath); // Write the audio to file try { // Remove trailing silence short[] trimmedAudio = null; - for(int x = audioData.length - 1; x > 0; x++) { + for(int x = MAX_LEN - 1; x > 0; x--) { if (audioData[x] != 0) { trimmedAudio = new short[x + 1]; - for(int y = 0; y> 8) & 0xff), (byte) ((dataLen >> 16) & 0xff),(byte) ((dataLen >> 24) & 0xff), 'W','A','V','E','f','m','t',' ', (byte) 16, 0, 0, 1, 0, (byte) channelNumber, 0,(byte) (sampleRate & 0xff), (byte) ((sampleRate >> 8) & 0xff), (byte) ((sampleRate >> 16) & 0xff), (byte) ((sampleRate >> 24) & 0xff), (byte) ((bitRate / 8) & 0xff), (byte) (((bitRate / 8) >> 8) & 0xff), (byte) (((bitRate / 8) >> 16) & 0xff), (byte) (((bitRate / 8) >> 24) & 0xff), (byte) ((channelNumber * 16) / 8), 0, 16, 0, 'd', 'a', 't', 'a', (byte) (dataLen * 2 & 0xff), (byte) (((dataLen * 2) >> 8) & 0xff), (byte) (((dataLen * 2) >> 16) & 0xff), (byte) (((dataLen * 2) >> 24) & 0xff) }; - dataBuffer.put(header); - ShortBuffer dataBufferShort = dataBuffer.asShortBuffer(); - dataBufferShort.put(trimmedAudio); // dataBuffer changes alongside dataBufferShort + */ + byte[] header = new byte[44]; + + header[0] = 'R'; + header[1] = 'I'; + header[2] = 'F'; + header[3] = 'F'; + header[4] = (byte) (dataLen & 0xff); + header[5] = (byte) ((dataLen >> 8) & 0xff); + header[6] = (byte) ((dataLen >> 16) & 0xff); + header[7] = (byte) ((dataLen >> 24) & 0xff); + header[8] = 'W'; + header[9] = 'A'; + header[10] = 'V'; + header[11] = 'E'; + header[12] = 'f'; + header[13] = 'm'; + header[14] = 't'; + header[15] = ' '; + header[16] = (byte) 16; + header[17] = 0; + header[18] = 0; + header[19] = 0; + header[20] = 1; + header[21] = 0; + header[22] = (byte) 1; + header[23] = 0; + header[24] = (byte) (sampleRate & 0xff); + header[25] = (byte) ((sampleRate >> 8) & 0xff); + header[26] = (byte) ((sampleRate >> 16) & 0xff); + header[27] = (byte) ((sampleRate >> 24) & 0xff); + header[28] = (byte) ((bitRate / 8) & 0xff); + header[29] = (byte) (((bitRate / 8) >> 8) & 0xff); + header[30] = (byte) (((bitRate / 8) >> 16) & 0xff); + header[31] = (byte) (((bitRate / 8) >> 24) & 0xff); + header[32] = (byte) ((channelNumber * 16) / 8); + header[33] = 0; + header[34] = 16; + header[35] = 0; + header[36] = 'd'; + header[37] = 'a'; + header[38] = 't'; + header[39] = 'a'; + header[40] = (byte) ((trimmedLen * 2) & 0xff); + header[41] = (byte) (((trimmedLen * 2) >> 8) & 0xff); + header[42] = (byte) (((trimmedLen * 2) >> 16) & 0xff); + header[43] = (byte) (((trimmedLen * 2) >> 24) & 0xff); + dataBuffer.put(header, 0, 44); + + Log.d("AudiDataStorageLog", "Just wrote header to file!"); + + for(short s : trimmedAudio) { + dataBuffer.putShort(s); + } + Log.d("AudioDataStorageLog", "Put " + trimmedAudio.length + " shorts into file!"); + //ShortBuffer dataBufferShort = dataBuffer.asShortBuffer(); + //dataBufferShort.put(trimmedAudio); // dataBuffer changes alongside dataBufferShort + dataBuffer.flip(); FileChannel out = os.getChannel(); out.write(dataBuffer); - out.close(); + os.flush(); os.close(); + + RandomAccessFile rafOut = new RandomAccessFile(completeFilePath, "rws"); + FileChannel rafChannel = rafOut.getChannel(); + int total = rafChannel.write(dataBuffer); + Log.d("AudioDataStorageLog", "Wrote out " + total + "bytes!"); + + rafChannel.close(); + rafOut.close(); } catch (IOException | NullPointerException e) { - e.printStackTrace(); // ? + Log.wtf("AudioDataStorage", "Failed to save recording to file!"); + Log.d("AudioRecordingFail", Log.getStackTraceString(e)); } // write segmentation to file @@ -104,8 +186,9 @@ public static void saveAudioData(String fileName, String assetLocation) { FileOutputStream os = new FileOutputStream(assetLocation + "/" + fileName + ".seg"); StringBuilder segData = new StringBuilder(""); + Log.d("Segprogress", "FileOutputStream Created at " + assetLocation + "/" + fileName + ".seg"); for(ListenerBase.HeardWord word: segmentation) { - segData.append(word.hypWord + "\t" + word.startTime + "\t" + word.startTime); + segData.append(word.hypWord + "\t" + word.startFrame + "\t" + word.endFrame); segData.append("\n"); } segData.deleteCharAt(segData.lastIndexOf("\n")); @@ -113,8 +196,10 @@ public static void saveAudioData(String fileName, String assetLocation) { byte[] segBytes = segData.toString().getBytes(); os.write(segBytes); os.close(); + } catch(IOException | NullPointerException e) { - e.printStackTrace(); + Log.wtf("AudioDataStorage", "Failed to write segmentation!"); + Log.d("SegmentationFail", Log.getStackTraceString(e)); } // Update Storydata.json @@ -145,13 +230,19 @@ public static void saveAudioData(String fileName, String assetLocation) { sentenceData.put("utterance", fileName); } catch(JSONException e) { - + Log.wtf("ADSStoryData", "Failed to update storyData!"); + Log.d("StoryDataFail", Log.getStackTraceString(e)); } - segmentation.clear(); } static void updateHypothesis(ListenerBase.HeardWord[] heardWords) { + Log.d("AudioDataStorageHyp", "Hypothesis Updated"); + segmentation.clear(); segmentation.addAll(Arrays.asList(heardWords)); } + + static void setSampleRate(int samplerate) { + // samplerate is always 16000 from my experience + } } diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioWriter.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioWriter.java new file mode 100644 index 000000000..1a65bac45 --- /dev/null +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioWriter.java @@ -0,0 +1,156 @@ +package edu.cmu.xprize.listener; + +import android.media.AudioRecord; +import android.util.Log; + +import java.io.BufferedOutputStream; +import java.io.DataOutputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; + +public class AudioWriter { + + public static volatile short[] audioData = new short[160 * 60 * 100]; + public static volatile OutputStream outputStream; + public static volatile FileChannel fileChannel; + public static volatile BufferedOutputStream bufferedOutputStream; + public static volatile DataOutputStream dataOutputStream; + + public static String audioFileName; + public static String audioAssetLocation; + public static String completeFilePath; + + public static int dataLen; + + // AudioWriter is updated with a new filepath + public static void initializePath(String fileName, String assetLocation) { + dataLen = 0; + + + try { + outputStream.close(); + } catch (Exception ignored) {} + + try { + bufferedOutputStream.close(); + } catch (Exception ignored) {} + + try { + dataOutputStream.close(); + } catch (Exception ignored) {} + + + try { + addHeader(new RandomAccessFile(completeFilePath, "rws")); + } catch (Exception e) { + Log.d("AudioWriterHeaderFail", Log.getStackTraceString(e)); + } + + audioFileName = fileName; + audioAssetLocation = assetLocation; + completeFilePath = assetLocation + fileName + ".wav"; + + try { + outputStream = new FileOutputStream(completeFilePath); + bufferedOutputStream = new BufferedOutputStream(outputStream); + dataOutputStream = new DataOutputStream(bufferedOutputStream); + + for(int i = 0; i < 44; i++) { + dataOutputStream.writeByte(0); + } + } catch (IOException e) { + Log.wtf("AudioWriterFail", Log.getStackTraceString(e)); + } + } + + public static void addAudio(int noOfShorts, short[] dataBuffer) { + try { + ByteBuffer bytes = ByteBuffer.allocate(noOfShorts * 2); + bytes.order(ByteOrder.LITTLE_ENDIAN); + + for(int i = 0; i < noOfShorts; i++) { + bytes.putShort(dataBuffer[i]); + } + + dataOutputStream.write(bytes.array()); + dataOutputStream.flush(); + dataLen += noOfShorts; + Log.d("AudioWriter", "Just wrote " + noOfShorts + " shorts to the audio file!"); + } catch (IOException e) { + Log.wtf("AudioWriterFail", Log.getStackTraceString(e)); + } + } + + public static void destroyContent() { + initializePath(audioFileName, audioAssetLocation); + } + + public static void addHeader(RandomAccessFile raf) throws IOException { + long dataLen = raf.length() - 8; + int sampleRate = 16000; + int channelNumber = 1; + int bitRate = sampleRate * channelNumber * 16; + long trimmedLen = dataLen - 36; + + byte[] header = new byte[44]; + + header[0] = 'R'; + header[1] = 'I'; + header[2] = 'F'; + header[3] = 'F'; + header[4] = (byte) (dataLen & 0xff); + header[5] = (byte) ((dataLen >> 8) & 0xff); + header[6] = (byte) ((dataLen >> 16) & 0xff); + header[7] = (byte) ((dataLen >> 24) & 0xff); + header[8] = 'W'; + header[9] = 'A'; + header[10] = 'V'; + header[11] = 'E'; + header[12] = 'f'; + header[13] = 'm'; + header[14] = 't'; + header[15] = ' '; + header[16] = (byte) 16; + header[17] = 0; + header[18] = 0; + header[19] = 0; + header[20] = 1; + header[21] = 0; + header[22] = (byte) 1; + header[23] = 0; + header[24] = (byte) (sampleRate & 0xff); + header[25] = (byte) ((sampleRate >> 8) & 0xff); + header[26] = (byte) ((sampleRate >> 16) & 0xff); + header[27] = (byte) ((sampleRate >> 24) & 0xff); + header[28] = (byte) ((bitRate / 8) & 0xff); + header[29] = (byte) (((bitRate / 8) >> 8) & 0xff); + header[30] = (byte) (((bitRate / 8) >> 16) & 0xff); + header[31] = (byte) (((bitRate / 8) >> 24) & 0xff); + header[32] = (byte) ((channelNumber * 16) / 8); + header[33] = 0; + header[34] = 16; + header[35] = 0; + header[36] = 'd'; + header[37] = 'a'; + header[38] = 't'; + header[39] = 'a'; + header[40] = (byte) ((trimmedLen * 2) & 0xff); + header[41] = (byte) (((trimmedLen * 2) >> 8) & 0xff); + header[42] = (byte) (((trimmedLen * 2) >> 16) & 0xff); + header[43] = (byte) (((trimmedLen * 2) >> 24) & 0xff); + + raf.seek(0); + + raf.write(header, 0, 44); + + raf.seek(raf.length()); + raf.close(); + } + +} diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java index a552f5e2f..f657aeb7b 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/SpeechRecognizer.java @@ -461,6 +461,7 @@ public void run() { Log.d("ASR", "AudioRecorder Create Failed: " + e); } isRunningRecognizer = true; + AudioDataStorage.setSampleRate(sampleRate); // Collect audio samples continuously while not paused and until the // Thread is killed. This allow UI/UX activity while the listener is still @@ -577,7 +578,8 @@ public void run() { decoder.processRaw(buffer, nread, false, false); //Log.d("ASR", "Time in processRaw: " + (System.currentTimeMillis() - ASRTimer)); - AudioDataStorage.addAudioData(buffer); + AudioDataStorage.addAudioData(nread, buffer); + AudioWriter.addAudio(nread, buffer); nSamples += nread; diff --git a/comp_logging/build.gradle b/comp_logging/build.gradle index adbb57b35..fbf4fe84f 100644 --- a/comp_logging/build.gradle +++ b/comp_logging/build.gradle @@ -22,11 +22,16 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') + api fileTree(include: ['*.jar'], dir: 'libs') androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', { exclude group: 'com.android.support', module: 'support-annotations' }) +<<<<<<< HEAD compile 'com.android.support:appcompat-v7:25.2.0' compile 'com.google.code.gson:gson:2.8.0' testCompile 'junit:junit:4.12' +======= + api 'com.android.support:appcompat-v7:25.2.0' + testImplementation 'junit:junit:4.12' +>>>>>>> eac62034... Fixed issue with enabling narrate mode on startup } diff --git a/comp_ltkplus/build.gradle b/comp_ltkplus/build.gradle index 9c7e7d559..ff63ef4b6 100644 --- a/comp_ltkplus/build.gradle +++ b/comp_ltkplus/build.gradle @@ -34,10 +34,10 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile 'com.android.support:percent:25.2.0' - compile project(':util') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api 'com.android.support:percent:25.2.0' + api project(':util') + api project(':comp_logging') } diff --git a/comp_math/build.gradle b/comp_math/build.gradle index 6794867cb..617607edc 100644 --- a/comp_math/build.gradle +++ b/comp_math/build.gradle @@ -29,11 +29,11 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile project(path: ':util') - compile project(':comp_logging') - compile project(path: ':comp_writebox') - compile 'com.android.support.constraint:constraint-layout:1.1.2' + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api project(path: ':util') + api project(':comp_logging') + api project(path: ':comp_writebox') + api 'com.android.support.constraint:constraint-layout:1.1.2' } diff --git a/comp_nd/build.gradle b/comp_nd/build.gradle index 8d828e921..d9c1cd2ec 100644 --- a/comp_nd/build.gradle +++ b/comp_nd/build.gradle @@ -24,15 +24,15 @@ android { } dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) - compile 'com.android.support:appcompat-v7:26.1.0' - testCompile 'junit:junit:4.12' + api fileTree(dir: 'libs', include: ['*.jar']) + api 'com.android.support:appcompat-v7:26.1.0' + testImplementation 'junit:junit:4.12' androidTestCompile('com.android.support.test.espresso:espresso-core:3.0.2', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile project(path: ':util') - compile project (':resources') - compile 'com.android.support.constraint:constraint-layout:1.1.2' + api project(path: ':util') + api project (':resources') + api 'com.android.support.constraint:constraint-layout:1.1.2' - compile 'com.google.android:flexbox:1.0.0' + api 'com.google.android:flexbox:1.0.0' } diff --git a/comp_numberscale/build.gradle b/comp_numberscale/build.gradle index 7aceea4d2..a16374d27 100644 --- a/comp_numberscale/build.gradle +++ b/comp_numberscale/build.gradle @@ -22,10 +22,10 @@ android { } dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) - compile 'com.android.support:appcompat-v7:25.2.0' - testCompile 'junit:junit:4.12' - compile project(path: ':util') - compile project(':comp_logging') + api fileTree(dir: 'libs', include: ['*.jar']) + api 'com.android.support:appcompat-v7:25.2.0' + testImplementation 'junit:junit:4.12' + api project(path: ':util') + api project(':comp_logging') } diff --git a/comp_picmatch/build.gradle b/comp_picmatch/build.gradle index b702ad93d..db51611a0 100644 --- a/comp_picmatch/build.gradle +++ b/comp_picmatch/build.gradle @@ -21,14 +21,14 @@ android { } dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) - compile 'com.android.support:appcompat-v7:26.1.0' - compile 'com.android.support.constraint:constraint-layout:1.1.2' - testCompile 'junit:junit:4.12' + api fileTree(dir: 'libs', include: ['*.jar']) + api 'com.android.support:appcompat-v7:26.1.0' + api 'com.android.support.constraint:constraint-layout:1.1.2' + testImplementation 'junit:junit:4.12' androidTestCompile('com.android.support.test.espresso:espresso-core:3.0.2', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile project(path: ':util') - compile project(':comp_logging') + api project(path: ':util') + api project(':comp_logging') } diff --git a/comp_pointtap/build.gradle b/comp_pointtap/build.gradle index a0f21b10a..4299e0817 100644 --- a/comp_pointtap/build.gradle +++ b/comp_pointtap/build.gradle @@ -29,9 +29,9 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile project(path: ':util') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api project(path: ':util') + api project(':comp_logging') } diff --git a/comp_questions/build.gradle b/comp_questions/build.gradle index 1a6edd726..180b11e96 100644 --- a/comp_questions/build.gradle +++ b/comp_questions/build.gradle @@ -25,12 +25,12 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile 'com.android.support:percent:25.2.0' - compile project(':comp_listener') - compile project(':util') - compile project(':comp_ask') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api 'com.android.support:percent:25.2.0' + api project(':comp_listener') + api project(':util') + api project(':comp_ask') + api project(':comp_logging') } diff --git a/comp_reading/build.gradle b/comp_reading/build.gradle index ff04fa8b7..724d0ecdb 100644 --- a/comp_reading/build.gradle +++ b/comp_reading/build.gradle @@ -29,12 +29,12 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile 'com.android.support:percent:25.2.0' - compile project(':comp_listener') - compile project(':util') - compile project(':comp_ask') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api 'com.android.support:percent:25.2.0' + api project(':comp_listener') + api project(':util') + api project(':comp_ask') + api project(':comp_logging') } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_Component.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_Component.java index 3709d336c..9d854d9a9 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_Component.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_Component.java @@ -603,6 +603,9 @@ public void loadStory(String EXTERNPATH, String viewType, String assetLocation, } + public void enableNarrateMode(boolean isNarrateMode) { + mViewManager.enableNarrateMode(isNarrateMode); + } /** * TODO: this currently only supports extern assets - need to allow for internal assets diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java index 1b20fd959..d008a16d2 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerASB.java @@ -46,13 +46,13 @@ import cmu.xprize.util.JSON_Helper; import cmu.xprize.util.TCONST; import edu.cmu.xprize.listener.AudioDataStorage; +import edu.cmu.xprize.listener.AudioWriter; import edu.cmu.xprize.listener.ListenerBase; import static cmu.xprize.util.TCONST.FTR_USER_READ; import static cmu.xprize.util.TCONST.FTR_USER_READING; import static cmu.xprize.util.TCONST.QGRAPH_MSG; - /** * This view manager provides student UX for the African Story Book format used * in the CMU XPrize submission @@ -218,14 +218,18 @@ public void initStory(IVManListener owner, String assetPath, String location) { //TODO: CHECK mParent.animatePageFlip(true,mCurrViewIndex); - // Turns on narrate mode if it is specified in Config.json - if (mContext.getSharedPreferences("ROBOTUTOR CONFIGURATION", Context.MODE_PRIVATE) - .getBoolean("CONTENT_CREATION_MODE", false)) { + feedSentence(); + + } + + // Narrate mode gets activated here + public void enableNarrateMode(boolean isNarrateMode) { + if (isNarrateMode) { mParent.setFeature(TCONST.FTR_NARRATE_MODE, TCONST.ADD_FEATURE); + Log.d("CRT_ViewManagerASB", "Narrate Mode has been activated through setNarrateMode()"); } } - /** * NOTE: we reset mCurrWord - last parm in seekToStoryPosition * @@ -253,6 +257,8 @@ public void startStory() { storyBooting = false; speakOrListen(); } + + Log.d("InitStory", "Story has initialized"); } @@ -1066,6 +1072,8 @@ public void nextPage() { // Actually do the page animation // mParent.animatePageFlip(true, mCurrViewIndex); + + feedSentence(); } @Override public void prevPage() { @@ -1121,6 +1129,8 @@ public void nextPara() { if (mCurrPara < mParaCount-1) { incPara(TCONST.INCR); } + + feedSentence(); } @Override @@ -1165,6 +1175,8 @@ public void nextLine() { if (mCurrLine < mLineCount-1) { incLine(TCONST.INCR); } + + feedSentence(); } @Override public void prevLine() { @@ -1651,6 +1663,7 @@ public void loadJSON(JSONObject jsonData, IScope scope) { @Override public void saveToFile() { + Log.d("CRt_ saveToFile", "Preparing to save audio data"); // Step 1. get sentence text StringBuilder fileNameBuilder = new StringBuilder(); for (String word : wordsToSpeak) { // This uses the wordsToDisplay String[] because it contains punctuation @@ -1660,13 +1673,15 @@ public void saveToFile() { String fileName = fileNameBuilder.toString(); - + Log.d("CRt_ saveToFile", "Telling AudioDataStorage to being writing file"); AudioDataStorage.saveAudioData(fileName, mAsset); + } @Override public void clearAudioData() { AudioDataStorage.clearAudioData(); + AudioWriter.destroyContent(); } @Override @@ -1688,4 +1703,20 @@ public void prevSentence() { } seekToStoryPosition(mCurrPage, mCurrPara, mCurrLine, mCurrWord); } + + public void feedSentence() { + // Step 1. get sentence text + StringBuilder fileNameBuilder = new StringBuilder(); + for (String word : wordsToSpeak) { // This uses the wordsToDisplay String[] because it contains punctuation + fileNameBuilder.append(word).append(" "); + } + fileNameBuilder.setLength(fileNameBuilder.length() - 1); + + String fileName = fileNameBuilder.toString(); + + Log.d("CRt_ saveToFile", "Telling AudioDataStorage to being writing file"); + AudioWriter.initializePath(fileName, mAsset); + } + + } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerMari.java b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerMari.java index f68479c0a..028391fa9 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerMari.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/CRt_ViewManagerMari.java @@ -609,6 +609,14 @@ public void loadJSON(JSONObject jsonData, IScope scope) { System.out.println(sb.toString()); sentences = new ArrayList(Arrays.asList(sb.toString().split("\\."))); } + + public void prevSentence() { + + } + + public void enableNarrateMode(boolean isNarrateMode) { + + } } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java b/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java index 84fb3372c..222b58f99 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/ICRt_ViewManager.java @@ -81,4 +81,6 @@ public interface ICRt_ViewManager extends ILoadableObject { public void startLine(); public void prevSentence(); + + public void enableNarrateMode(boolean isNarrateMode); } diff --git a/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java b/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java index f084ec199..a6ef1c84b 100644 --- a/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java +++ b/comp_reading/src/main/java/cmu/xprize/rt_component/IRtComponent.java @@ -54,8 +54,8 @@ public interface IRtComponent { public void continueListening(); - void saveToFile(); - void clearAudioData(); - void startLine(); - void prevSentence(); + public void saveToFile(); + public void clearAudioData(); + public void startLine(); + public void prevSentence(); } diff --git a/comp_scoreboard/build.gradle b/comp_scoreboard/build.gradle index 81e0f5727..823bc1ff9 100644 --- a/comp_scoreboard/build.gradle +++ b/comp_scoreboard/build.gradle @@ -29,10 +29,10 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile 'com.android.support:percent:25.2.0' - compile project(':util') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api 'com.android.support:percent:25.2.0' + api project(':util') + api project(':comp_logging') } diff --git a/comp_session/build.gradle b/comp_session/build.gradle index adb4f92ce..47ab9dda1 100644 --- a/comp_session/build.gradle +++ b/comp_session/build.gradle @@ -20,13 +20,13 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') + api fileTree(include: ['*.jar'], dir: 'libs') androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile 'com.android.support:appcompat-v7:25.2.0' - testCompile 'junit:junit:4.12' - compile project(path: ':comp_ask') - compile project(':comp_debug') - compile project(':comp_logging') + api 'com.android.support:appcompat-v7:25.2.0' + testImplementation 'junit:junit:4.12' + api project(path: ':comp_ask') + api project(':comp_debug') + api project(':comp_logging') } diff --git a/comp_spelling/build.gradle b/comp_spelling/build.gradle index 4e43db336..e0618a28b 100644 --- a/comp_spelling/build.gradle +++ b/comp_spelling/build.gradle @@ -24,14 +24,14 @@ android { } dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) - compile 'com.android.support:appcompat-v7:26.1.0' - compile 'com.android.support.constraint:constraint-layout:1.1.2' + api fileTree(dir: 'libs', include: ['*.jar']) + api 'com.android.support:appcompat-v7:26.1.0' + api 'com.android.support.constraint:constraint-layout:1.1.2' - testCompile 'junit:junit:4.12' + testImplementation 'junit:junit:4.12' androidTestCompile('com.android.support.test.espresso:espresso-core:3.0.2', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile project(path: ':util') + api project(path: ':util') } diff --git a/comp_writebox/build.gradle b/comp_writebox/build.gradle index 0095e475e..be8a0c1b4 100644 --- a/comp_writebox/build.gradle +++ b/comp_writebox/build.gradle @@ -21,14 +21,14 @@ android { } dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) - compile 'com.android.support:appcompat-v7:26.1.0' - testCompile 'junit:junit:4.12' + api fileTree(dir: 'libs', include: ['*.jar']) + api 'com.android.support:appcompat-v7:26.1.0' + testImplementation 'junit:junit:4.12' androidTestCompile('com.android.support.test.espresso:espresso-core:3.0.2', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile project(path: ':util') - compile project(':comp_ltkplus') - compile project(':comp_logging') + api project(path: ':util') + api project(':comp_ltkplus') + api project(':comp_logging') } diff --git a/comp_writing/build.gradle b/comp_writing/build.gradle index a9e84b65b..df585b40c 100644 --- a/comp_writing/build.gradle +++ b/comp_writing/build.gradle @@ -29,12 +29,12 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile 'com.android.support:percent:25.2.0' - compile project(path: ':util') - compile project(':comp_ltkplus') - compile project(path: ':comp_pointtap') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api 'com.android.support:percent:25.2.0' + api project(path: ':util') + api project(':comp_ltkplus') + api project(path: ':comp_pointtap') + api project(':comp_logging') } diff --git a/fw_component/build.gradle b/fw_component/build.gradle index b2bff7b54..a8a1cd2cc 100644 --- a/fw_component/build.gradle +++ b/fw_component/build.gradle @@ -29,9 +29,9 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile project(':comp_ltkplus') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api project(':comp_ltkplus') + api project(':comp_logging') } diff --git a/mn_component/build.gradle b/mn_component/build.gradle index 91ea34665..36eed5f59 100644 --- a/mn_component/build.gradle +++ b/mn_component/build.gradle @@ -29,10 +29,10 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile 'com.android.support:percent:25.2.0' - compile project(':util') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api 'com.android.support:percent:25.2.0' + api project(':util') + api project(':comp_logging') } diff --git a/nl_component/build.gradle b/nl_component/build.gradle index 8feb84b0b..62447bb0c 100644 --- a/nl_component/build.gradle +++ b/nl_component/build.gradle @@ -29,11 +29,11 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile project(':util') - compile project(':comp_listener') - compile project(':fw_component') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api project(':util') + api project(':comp_listener') + api project(':fw_component') + api project(':comp_logging') } diff --git a/resources/build.gradle b/resources/build.gradle index c2df18eb0..f80c7fe38 100644 --- a/resources/build.gradle +++ b/resources/build.gradle @@ -23,9 +23,9 @@ android { } dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) - compile 'com.android.support:appcompat-v7:26.1.0' - testCompile 'junit:junit:4.12' + api fileTree(dir: 'libs', include: ['*.jar']) + api 'com.android.support:appcompat-v7:26.1.0' + testImplementation 'junit:junit:4.12' androidTestCompile('com.android.support.test.espresso:espresso-core:3.0.2', { exclude group: 'com.android.support', module: 'support-annotations' }) diff --git a/sm_component/build.gradle b/sm_component/build.gradle index 91ea34665..36eed5f59 100644 --- a/sm_component/build.gradle +++ b/sm_component/build.gradle @@ -29,10 +29,10 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - compile 'com.android.support:appcompat-v7:25.2.0' - compile 'com.android.support:percent:25.2.0' - compile project(':util') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + api 'com.android.support:appcompat-v7:25.2.0' + api 'com.android.support:percent:25.2.0' + api project(':util') + api project(':comp_logging') } diff --git a/util/build.gradle b/util/build.gradle index bdcbc4439..4a6f38a3a 100644 --- a/util/build.gradle +++ b/util/build.gradle @@ -33,11 +33,11 @@ android { } dependencies { - compile fileTree(include: ['*.jar'], dir: 'libs') - testCompile 'junit:junit:4.12' - testCompile 'org.json:json:20140107' - compile 'com.android.support:appcompat-v7:25.2.0' - compile 'com.android.support:percent:25.2.0' - compile files('libs/datashop-logging.jar') - compile project(':comp_logging') + api fileTree(include: ['*.jar'], dir: 'libs') + testImplementation 'junit:junit:4.12' + testImplementation 'org.json:json:20140107' + api 'com.android.support:appcompat-v7:25.2.0' + api 'com.android.support:percent:25.2.0' + api files('libs/datashop-logging.jar') + api project(':comp_logging') } diff --git a/util/src/main/java/cmu/xprize/util/JSON_Helper.java b/util/src/main/java/cmu/xprize/util/JSON_Helper.java index fd4c1308d..1c128b43c 100644 --- a/util/src/main/java/cmu/xprize/util/JSON_Helper.java +++ b/util/src/main/java/cmu/xprize/util/JSON_Helper.java @@ -279,7 +279,7 @@ else if (fieldClass.equals(HashMap.class)) { // You can have a global hash RH type where you defince the Right hand type once // This permits native types - i.e. arrays etc on the RH side. - // Note that type info gets lost in the compile so it is not available for introspection + // Note that type info gets lost in the api so it is not available for introspection // Check for global type Class elemClass = null; From cd4a8c569b879642d8c27c5dddb2d78d3055b7d6 Mon Sep 17 00:00:00 2001 From: chirag03k Date: Wed, 19 Aug 2020 12:39:48 -0700 Subject: [PATCH 10/45] Mean mode - TODO: fix bug where the buttons are not found by the ViewManager (returns null, causing a NullPointerException to be thrown) --- .../widgets/core/TRtComponent.java | 5 + app/src/main/res/layout/story_reading.xml | 43 +++++- .../cmu/xprize/listener/AudioDataStorage.java | 126 ++---------------- .../edu/cmu/xprize/listener/AudioWriter.java | 47 ++++++- .../src/main/res/layout/asb_evenpage.xml | 49 ++++++- .../xprize/rt_component/CRt_Component.java | 5 +- .../rt_component/CRt_ViewManagerASB.java | 54 ++++++++ .../rt_component/CRt_ViewManagerMari.java | 4 + .../xprize/rt_component/ICRt_ViewManager.java | 2 + .../cmu/xprize/rt_component/IRtComponent.java | 1 + .../src/main/res/layout/asb_oddpage.xml | 48 ++++++- .../src/main/res/layout/rt__component.xml | 1 + 12 files changed, 261 insertions(+), 124 deletions(-) diff --git a/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java b/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java index 5101ab94d..cc7887a32 100644 --- a/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java +++ b/app/src/main/java/cmu/xprize/robotutor/tutorengine/widgets/core/TRtComponent.java @@ -23,6 +23,7 @@ import android.util.Log; import android.view.View; import android.widget.Button; +import android.widget.LinearLayout; import org.json.JSONObject; @@ -34,6 +35,7 @@ import cmu.xprize.comp_logging.ITutorLogger; import cmu.xprize.comp_logging.PerformanceLogItem; +import cmu.xprize.robotutor.R; import cmu.xprize.robotutor.RoboTutor; import cmu.xprize.robotutor.startup.configuration.Configuration; import cmu.xprize.robotutor.tutorengine.CDebugLauncher; @@ -1239,4 +1241,7 @@ public void startLine() { @Override public void prevSentence() {mViewManager.prevSentence();} + + @Override + public void skipSentence() {mViewManager.skipSentence(); } } diff --git a/app/src/main/res/layout/story_reading.xml b/app/src/main/res/layout/story_reading.xml index 7031ccecc..6aec51ab8 100644 --- a/app/src/main/res/layout/story_reading.xml +++ b/app/src/main/res/layout/story_reading.xml @@ -2,6 +2,7 @@ @@ -18,11 +19,51 @@ android:textColor="#FFF" android:text="This is the Banner Area."/> + + android:layout_height="match_parent" + > + + + diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java index 986dec910..73ee31e23 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioDataStorage.java @@ -36,8 +36,12 @@ public class AudioDataStorage { static FileOutputStream outputStream; static FileChannel outChannel; - public static void initStoryData(JSONObject JSONObj) { - storyData = JSONObj; + public static void initStoryData(String jsonData) { + try { + storyData = new JSONObject(jsonData); + } catch (Exception e) { + Log.wtf("AudioDataStorage", "Could not load storydata"); + } } // I initialize the array and add to it @@ -68,118 +72,9 @@ public static void saveAudioData(String fileName, String assetLocation) { sampleRate = 16000; - String completeFilePath = assetLocation + fileName + "fail.wav"; + String completeFilePath = assetLocation + fileName + ".wav"; Log.d("ADSSave", completeFilePath); - // Write the audio to file - try { - // Remove trailing silence - short[] trimmedAudio = null; - for(int x = MAX_LEN - 1; x > 0; x--) { - if (audioData[x] != 0) { - trimmedAudio = new short[x + 1]; - System.arraycopy(audioData, 0, trimmedAudio, 0, x); - break; - } - } - - - FileOutputStream os = new FileOutputStream(completeFilePath); - int trimmedLen = trimmedAudio.length; - int dataLen = (trimmedLen * 2) + 36; - ByteBuffer dataBuffer = ByteBuffer.allocate(trimmedLen * 2 /* the data */ + 36 /* The header */); - - dataBuffer.order(ByteOrder.BIG_ENDIAN); // Audio is recorded in Big Endian - - // Sample rate is determined by the recognizer (through some external file but I don't know what) - // CHANNEL_IN_MONO (1 channel input) - - int channelNumber = 1; - long bitRate = sampleRate * channelNumber * 16; // sampleRate times number of Channels times the number of bits per sample (16) - /*byte[] header = new byte[]{'R','I','F','F', - (byte) (dataLen & 0xff), (byte) (byte) ((dataLen >> 8) & 0xff), (byte) ((dataLen >> 16) & 0xff),(byte) ((dataLen >> 24) & 0xff), - 'W','A','V','E','f','m','t',' ', - (byte) 16, 0, 0, 1, 0, (byte) channelNumber, 0,(byte) (sampleRate & 0xff), (byte) ((sampleRate >> 8) & 0xff), (byte) ((sampleRate >> 16) & 0xff), (byte) ((sampleRate >> 24) & 0xff), - (byte) ((bitRate / 8) & 0xff), (byte) (((bitRate / 8) >> 8) & 0xff), (byte) (((bitRate / 8) >> 16) & 0xff), (byte) (((bitRate / 8) >> 24) & 0xff), (byte) ((channelNumber * 16) / 8), - 0, 16, 0, 'd', 'a', 't', 'a', (byte) (dataLen * 2 & 0xff), (byte) (((dataLen * 2) >> 8) & 0xff), (byte) (((dataLen * 2) >> 16) & 0xff), (byte) (((dataLen * 2) >> 24) & 0xff) - }; - - */ - byte[] header = new byte[44]; - - header[0] = 'R'; - header[1] = 'I'; - header[2] = 'F'; - header[3] = 'F'; - header[4] = (byte) (dataLen & 0xff); - header[5] = (byte) ((dataLen >> 8) & 0xff); - header[6] = (byte) ((dataLen >> 16) & 0xff); - header[7] = (byte) ((dataLen >> 24) & 0xff); - header[8] = 'W'; - header[9] = 'A'; - header[10] = 'V'; - header[11] = 'E'; - header[12] = 'f'; - header[13] = 'm'; - header[14] = 't'; - header[15] = ' '; - header[16] = (byte) 16; - header[17] = 0; - header[18] = 0; - header[19] = 0; - header[20] = 1; - header[21] = 0; - header[22] = (byte) 1; - header[23] = 0; - header[24] = (byte) (sampleRate & 0xff); - header[25] = (byte) ((sampleRate >> 8) & 0xff); - header[26] = (byte) ((sampleRate >> 16) & 0xff); - header[27] = (byte) ((sampleRate >> 24) & 0xff); - header[28] = (byte) ((bitRate / 8) & 0xff); - header[29] = (byte) (((bitRate / 8) >> 8) & 0xff); - header[30] = (byte) (((bitRate / 8) >> 16) & 0xff); - header[31] = (byte) (((bitRate / 8) >> 24) & 0xff); - header[32] = (byte) ((channelNumber * 16) / 8); - header[33] = 0; - header[34] = 16; - header[35] = 0; - header[36] = 'd'; - header[37] = 'a'; - header[38] = 't'; - header[39] = 'a'; - header[40] = (byte) ((trimmedLen * 2) & 0xff); - header[41] = (byte) (((trimmedLen * 2) >> 8) & 0xff); - header[42] = (byte) (((trimmedLen * 2) >> 16) & 0xff); - header[43] = (byte) (((trimmedLen * 2) >> 24) & 0xff); - dataBuffer.put(header, 0, 44); - - Log.d("AudiDataStorageLog", "Just wrote header to file!"); - - for(short s : trimmedAudio) { - dataBuffer.putShort(s); - } - Log.d("AudioDataStorageLog", "Put " + trimmedAudio.length + " shorts into file!"); - - //ShortBuffer dataBufferShort = dataBuffer.asShortBuffer(); - //dataBufferShort.put(trimmedAudio); // dataBuffer changes alongside dataBufferShort - - dataBuffer.flip(); - FileChannel out = os.getChannel(); - out.write(dataBuffer); - os.flush(); - os.close(); - - RandomAccessFile rafOut = new RandomAccessFile(completeFilePath, "rws"); - FileChannel rafChannel = rafOut.getChannel(); - int total = rafChannel.write(dataBuffer); - Log.d("AudioDataStorageLog", "Wrote out " + total + "bytes!"); - - rafChannel.close(); - rafOut.close(); - } catch (IOException | NullPointerException e) { - Log.wtf("AudioDataStorage", "Failed to save recording to file!"); - Log.d("AudioRecordingFail", Log.getStackTraceString(e)); - } // write segmentation to file try { @@ -224,14 +119,19 @@ public static void saveAudioData(String fileName, String assetLocation) { segm.put(segObj); finalEndTime = heardWord.endTime; } - sentenceData.put("from", ""); // TODO: figure out what 'from' is supposed to represent + sentenceData.put("from", segmentation.get(0).startTime); // TODO: figure out what 'from' is supposed to represent sentenceData.put("audio", completeFilePath); sentenceData.put("until", finalEndTime); sentenceData.put("utterance", fileName); + FileOutputStream outJson = new FileOutputStream(assetLocation + "storydata.json"); + outJson.write(storyData.toString().getBytes()); + outJson.close(); } catch(JSONException e) { Log.wtf("ADSStoryData", "Failed to update storyData!"); Log.d("StoryDataFail", Log.getStackTraceString(e)); + } catch(IOException e) { + Log.d("ADSStoryDataFail", "Was not able to open file"); } } diff --git a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioWriter.java b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioWriter.java index 1a65bac45..19764a5c9 100644 --- a/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioWriter.java +++ b/comp_listener/src/main/java/edu/cmu/xprize/listener/AudioWriter.java @@ -3,8 +3,11 @@ import android.media.AudioRecord; import android.util.Log; +import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -13,6 +16,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; +import java.nio.file.Files; public class AudioWriter { @@ -28,11 +32,9 @@ public class AudioWriter { public static int dataLen; - // AudioWriter is updated with a new filepath - public static void initializePath(String fileName, String assetLocation) { - dataLen = 0; - + static byte[] savedFileData; + public static void closeStreams() { try { outputStream.close(); } catch (Exception ignored) {} @@ -44,17 +46,36 @@ public static void initializePath(String fileName, String assetLocation) { try { dataOutputStream.close(); } catch (Exception ignored) {} + } + // AudioWriter is updated with a new filepath + public static void initializePath(String fileName, String assetLocation) { + dataLen = 0; + // Close up old file + closeStreams(); + //Reopen to add header to the beginning of the file try { addHeader(new RandomAccessFile(completeFilePath, "rws")); } catch (Exception e) { Log.d("AudioWriterHeaderFail", Log.getStackTraceString(e)); } - audioFileName = fileName; + // Prepare new file + audioFileName = fileName.toLowerCase().replace(" ", "_"); audioAssetLocation = assetLocation; - completeFilePath = assetLocation + fileName + ".wav"; + completeFilePath = assetLocation + audioFileName + ".wav"; + + // Just in case the new file is already existing, save its contents + try { + File inFile = new File(completeFilePath); + FileInputStream input = new FileInputStream(inFile); + savedFileData = new byte[(int) inFile.length()]; + BufferedInputStream bis =new BufferedInputStream(input); + bis.read(savedFileData,0,savedFileData.length); + bis.close(); + input.close(); + } catch (IOException ignored) {} try { outputStream = new FileOutputStream(completeFilePath); @@ -153,4 +174,18 @@ public static void addHeader(RandomAccessFile raf) throws IOException { raf.close(); } + // When the sentence is not being recorded, put the original file contents back + public static void abortOperation() { + try { + closeStreams(); + + // rewrite old contents of the file + FileOutputStream output = new FileOutputStream(completeFilePath); + output.write(savedFileData); + } catch (IOException ignored) { + + } catch (NullPointerException ignored) {// this is thrown if file doesn't exist + } + + } } diff --git a/comp_questions/src/main/res/layout/asb_evenpage.xml b/comp_questions/src/main/res/layout/asb_evenpage.xml index 54774b8de..7b03e07cb 100644 --- a/comp_questions/src/main/res/layout/asb_evenpage.xml +++ b/comp_questions/src/main/res/layout/asb_evenpage.xml @@ -7,6 +7,21 @@ android:layout_height="match_parent" android:background="#FFF"> + + + android:layout_marginRight="30dp" > + + + +