CP3108 SR AI-powered marking (#3126)

* Add Grading comment selector basic UI

* Connect to backend

* Content changes not found but are showing up as changed

* Hide generate comment button if no LLM prompt

* Input field for API key during Course creation

* Add input field for enabling LLM Grading

* Disable comment selector if llm grading not enabled

* Added LLM API Key input and integration with backend

* Add to the course editing panel for llm grading and api key

* Update localStorage.ts

Fix redux state not saving issue

* Add save chosen and final comment feature

Comment selector can now handle selecting multiple suggestions

* Add Spinner when generating comments

* Update to show API Key as password field

* Fix format

* Revert "Content changes not found but are showing up as changed"

This reverts commit 780a269.

* Edit frontend to load previous AI comments by default

* Moves api key, api url, llm model and course prompt to course level

* Add tooltip about formatting instructions

* style

* Add view "composed prompt" in the grading editor view

* Remove un-used 'save comments'

* typo

* Add copy prompt to clipboard

* Fix typo

* Fix bug preventing avengers from generating ai comments

* Fix TS errors

* Fix more TS errors!

* fix some style issues

* Update src/pages/academy/grading/subcomponents/GradingEditor.tsx

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/commons/dropdown/DropdownCreateCourse.tsx

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Work on comments

* Small style fix

* Abstract styles to scss modules

* Remove llmApiKey from any kind of storage on FE

* Re-format ai_comments to reference answer_id instead

* Merge all prompts into :prompts

---------

Co-authored-by: tkaixiang <tkaixiang@gmail.com>

Author: EugeneOYZ1203n

src/commons/application/actions/__tests__/SessionActions.test.ts

@@ -605,6 +605,8 @@ test('updateGrading generates correct action object', async () => {
   const grading: GradingQuery = {
     answers: [
       {
+        id: 0,
+        prompts: [],
         question: await vi.importMock('../../../../features/grading/GradingTypes'),
         student: {
           name: 'test student',
@@ -620,9 +622,12 @@ test('updateGrading generates correct action object', async () => {
             id: 100
           },
           gradedAt: '2019-08-16T13:26:32+00:00'
-        }
+        },
+        autogradingResults: [],
+        autoGradingStatus: 'N/A'
       }
     ],
+    enable_llm_grading: false,
     assessment: {
       coverPicture: 'https://i.imgur.com/dR7zBPI.jpeg',
       id: 1,

src/commons/application/reducers/__tests__/SessionReducer.test.ts

@@ -402,8 +402,13 @@ test('UPDATE_ASSESSMENT_OVERVIEWS works correctly in updating assessment overvie
 
 // Test data for UPDATE_GRADING
 const gradingTest1: GradingQuery = {
+  enable_llm_grading: false,
   answers: [
     {
+      id: 0,
+      prompts: [],
+      autogradingResults: [],
+      autoGradingStatus: 'N/A',
       question: await vi.importMock('../../../../features/grading/GradingTypes'),
       student: {
         name: 'test student',
@@ -430,8 +435,13 @@ const gradingTest1: GradingQuery = {
 };
 
 const gradingTest2: GradingQuery = {
+  enable_llm_grading: false,
   answers: [
     {
+      id: 0,
+      prompts: [],
+      autogradingResults: [],
+      autoGradingStatus: 'N/A',
       question: await vi.importMock('../../../../features/grading/GradingTypes'),
       student: {
         name: 'another test student',

src/commons/application/types/SessionTypes.ts

@@ -39,6 +39,10 @@ export type SessionState = {
   readonly enableAchievements?: boolean;
   readonly enableSourcecast?: boolean;
   readonly enableStories?: boolean;
+  readonly enableLlmGrading?: boolean;
+  readonly llmModel?: string;
+  readonly llmApiUrl?: string;
+  readonly llmCourseLevelPrompt?: string;
   readonly enableOverallLeaderboard?: boolean;
   readonly enableContestLeaderboard?: boolean;
   readonly topLeaderboardDisplay?: number;
@@ -109,6 +113,7 @@ export type CourseConfiguration = {
   enableAchievements: boolean;
   enableSourcecast: boolean;
   enableStories: boolean;
+  enableLlmGrading?: boolean;
   enableOverallLeaderboard: boolean;
   enableContestLeaderboard: boolean;
   topLeaderboardDisplay: number;
@@ -117,6 +122,9 @@ export type CourseConfiguration = {
   sourceVariant: Variant;
   moduleHelpText: string;
   assetsPrefix: string;
+  llmModel?: string;
+  llmApiUrl?: string;
+  llmCourseLevelPrompt?: string;
 };
 
 export type AdminPanelCourseRegistration = {
@@ -128,4 +136,4 @@ export type AdminPanelCourseRegistration = {
   group?: string;
 };
 
-export type UpdateCourseConfiguration = Partial<CourseConfiguration>;
+export type UpdateCourseConfiguration = Partial<CourseConfiguration & { llmApiKey: string }>;

src/commons/assessment/AssessmentTypes.ts

@@ -143,6 +143,11 @@ export interface IContestVotingQuestion extends BaseQuestion {
   type: 'voting';
 }
 
+export type LLMPrompt = {
+  role: string;
+  content: string;
+};
+
 export type BaseQuestion = {
   answer: string | number | ContestEntry[] | null;
   comments?: string;

src/commons/dropdown/DropdownCreateCourse.tsx

@@ -40,9 +40,11 @@ const DropdownCreateCourse: React.FC<Props> = props => {
     enableAchievements: true,
     enableSourcecast: true,
     enableStories: false,
+    enableLlmGrading: false,
     sourceChapter: Chapter.SOURCE_1,
     sourceVariant: Variant.DEFAULT,
-    moduleHelpText: ''
+    moduleHelpText: '',
+    llmApiKey: ''
   });
 
   const [courseHelpTextSelectedTab, setCourseHelpTextSelectedTab] =
@@ -222,7 +224,8 @@ const DropdownCreateCourse: React.FC<Props> = props => {
                 })
               }
             />
-
+          </div>
+          <div>
             <Switch
               checked={courseConfig.enableStories}
               inline
@@ -234,6 +237,18 @@ const DropdownCreateCourse: React.FC<Props> = props => {
                 })
               }
             />
+
+            <Switch
+              checked={courseConfig.enableLlmGrading}
+              inline
+              label="Enable LLM Grading"
+              onChange={e =>
+                setCourseConfig({
+                  ...courseConfig,
+                  enableLlmGrading: (e.target as HTMLInputElement).checked
+                })
+              }
+            />
           </div>
         </div>
         <div>
@@ -273,6 +288,24 @@ const DropdownCreateCourse: React.FC<Props> = props => {
               fill
             />
           </FormGroup>
+          <FormGroup
+            helperText="API Key for LLM endpoint. This key will be encrypted and will not be retrievable on the frontend after."
+            label={'LLM API Key'}
+            labelInfo="(optional)"
+            labelFor="llmApiKey"
+          >
+            <InputGroup
+              id="llmApiKey"
+              type="password"
+              value={courseConfig.llmApiKey}
+              onChange={e =>
+                setCourseConfig({
+                  ...courseConfig,
+                  llmApiKey: e.target.value
+                })
+              }
+            />
+          </FormGroup>
         </div>
         <div className="create-course-button-container">
           <Button text="Create Course" onClick={submitHandler} />

src/commons/mocks/BackendMocks.ts

@@ -343,7 +343,11 @@ export function* mockBackendSaga(): SagaIterator {
       return gradingQuestion;
     });
     yield put(
-      actions.updateGrading(submissionId, { answers: newGrading, assessment: grading.assessment })
+      actions.updateGrading(submissionId, {
+        answers: newGrading,
+        assessment: grading.assessment,
+        enable_llm_grading: false
+      })
     );
     yield call(showSuccessMessage, 'Submitted!', 1000);
   };

src/commons/mocks/GradingMocks.ts

@@ -128,6 +128,10 @@ export const mockTestcases: Testcase[] = [
 
 export const mockGradingAnswer: GradingAnswer = [
   {
+    id: 0,
+    prompts: [],
+    autogradingResults: [],
+    autoGradingStatus: 'N/A',
     question: {
       answer: `function remainder(n, d) {
   return (n - d) < 0 ? n : remainder(n - d, d);
@@ -244,6 +248,10 @@ _italics_
     }
   },
   {
+    id: 1,
+    prompts: [],
+    autogradingResults: [],
+    autoGradingStatus: 'N/A',
     question: {
       prepend: '',
       postpend: '',
@@ -336,6 +344,10 @@ New message from **Avenger**!
     }
   },
   {
+    id: 2,
+    prompts: [],
+    autogradingResults: [],
+    autoGradingStatus: 'N/A',
     question: {
       // C is the answer
       prepend: '',
@@ -434,7 +446,8 @@ Starring: Source Academy`,
 
 export const mockGradingQuery: GradingQuery = {
   answers: mockGradingAnswer,
-  assessment: mockGradingAssessment
+  assessment: mockGradingAssessment,
+  enable_llm_grading: false
 };
 
 /**

src/commons/sagas/BackendSaga.ts

@@ -637,7 +637,11 @@ function* sendGrade(
   });
 
   yield put(
-    actions.updateGrading(submissionId, { answers: newGrading, assessment: grading.assessment })
+    actions.updateGrading(submissionId, {
+      answers: newGrading,
+      assessment: grading.assessment,
+      enable_llm_grading: false
+    })
   );
 }
 

src/commons/sagas/RequestsSaga.ts

@@ -1030,8 +1030,9 @@ export const getGrading = async (
 
   const gradingResult = await resp.json();
   const grading: GradingAnswer = gradingResult.answers.map((gradingQuestion: any) => {
-    const { student, question, grade, team } = gradingQuestion;
+    const { id, student, question, grade, team } = gradingQuestion;
     const result = {
+      id,
       question: {
         answer: question.answer,
         lastModifiedAt: question.lastModifiedAt,
@@ -1054,7 +1055,9 @@ export const getGrading = async (
         xp: grade.xp,
         xpAdjustment: grade.xpAdjustment,
         comments: grade.comments
-      }
+      },
+      ai_comments: gradingQuestion.ai_comments?.response?.split('|||') || [],
+      prompts: gradingQuestion.prompts
     } as GradingQuestion;
 
     if (gradingQuestion.grade.grader !== null) {
@@ -1064,7 +1067,11 @@ export const getGrading = async (
     return result;
   });
 
-  return { answers: grading, assessment: gradingResult.assessment };
+  return {
+    enable_llm_grading: gradingResult.enable_llm_grading,
+    answers: grading,
+    assessment: gradingResult.assessment
+  };
 };
 
 /**
@@ -1548,6 +1555,36 @@ export const removeAssessmentConfig = async (
   return resp;
 };
 
+/**
+ * POST /courses/{courseId}/admin/generate-comments/{answer_id}/
+ */
+export const postGenerateComments = async (
+  tokens: Tokens,
+  answer_id: number
+): Promise<{ comments: string[] } | null> => {
+  const resp = await request(`${courseId()}/admin/generate-comments/${answer_id}`, 'POST', {
+    ...tokens
+  });
+  if (!resp || !resp.ok) {
+    return null;
+  }
+
+  return await resp.json();
+};
+
+export const saveFinalComment = async (
+  tokens: Tokens,
+  answer_id: number,
+  comment: string
+): Promise<Response | null> => {
+  const resp = await request(`${courseId()}/admin/save-final-comment/${answer_id}`, 'POST', {
+    body: { comment: comment },
+    ...tokens
+  });
+
+  return resp;
+};
+
 /**
  * GET /courses/{courseId}/admin/users
  */

src/features/grading/GradingTypes.ts

@@ -4,6 +4,7 @@ import {
   AssessmentStatus,
   AssessmentType,
   AutogradingResult,
+  LLMPrompt,
   MCQChoice,
   ProgressStatus,
   Question,
@@ -99,6 +100,7 @@ export type GradingAssessment = {
 export type GradingQuery = {
   answers: GradingAnswer;
   assessment: GradingAssessment;
+  enable_llm_grading: boolean | null;
 };
 
 export type GradingSubmissionTableProps = {
@@ -168,6 +170,7 @@ export type IGradingTableProperties = {
  * particular question in a submission.
  */
 export type GradingQuestion = {
+  id: number;
   question: AnsweredQuestion;
   team?: Array<{
     username: any;
@@ -189,6 +192,10 @@ export type GradingQuestion = {
     };
     gradedAt?: string;
   };
+  autogradingResults: AutogradingResult[];
+  autoGradingStatus: string;
+  ai_comments?: string[];
+  prompts: LLMPrompt[];
 };
 
 /**