Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
ede6582
Add alternate state logic for LLM question parts
sjd210 Oct 16, 2025
1bfd46e
Add function to extract attempts by question ID
sjd210 Oct 21, 2025
2a4b1bf
Read marksAwarded value for LLMFreeText responses
sjd210 Oct 21, 2025
983de47
Change question correctness threshold to maxMarks
sjd210 Oct 21, 2025
501149d
Clean up experimental code
sjd210 Oct 21, 2025
c76f5f0
Adapt retrieval function response type as argument
sjd210 Oct 21, 2025
02d68a4
Merge branch 'main' into improvement/count-llm-marks-seperately
sjd210 Oct 22, 2025
7a6e2e3
Migrate database to add mark to question_attempts
sjd210 Oct 22, 2025
92203a6
Convert database migration query to split by dates
sjd210 Oct 22, 2025
9230a55
Update tests to new LLMFreeText correct definition
sjd210 Oct 22, 2025
a02f051
Update postgres functions to process marks field
sjd210 Oct 22, 2025
f2c4fd4
Add marks to test question_attempts table
sjd210 Oct 22, 2025
2700c48
Add marks to lightweight response
sjd210 Oct 22, 2025
0b5809d
Add marks to lightweight responses in tests
sjd210 Oct 22, 2025
248a000
Allow QuestionValidationResponse to derive marks
sjd210 Oct 22, 2025
696ecdf
Also add marks field to quiz_question_attempts
sjd210 Oct 23, 2025
c982579
Remove question ID extraction code
sjd210 Oct 23, 2025
d4e0cd9
Augment game items with new marks field
sjd210 Oct 23, 2025
36a60e7
Update migration script dates split ranges
sjd210 Oct 23, 2025
13a1077
Restructure update_marks function as a scalar
sjd210 Oct 23, 2025
a62e73f
Move game item augmentation to another branch
sjd210 Oct 23, 2025
1874716
Add marks to QuestionValidationResponseDTO
sjd210 Oct 23, 2025
733d777
Correct syntax error
sjd210 Oct 23, 2025
4cd2ea1
Simplify conditional
sjd210 Oct 23, 2025
351d514
Replace marksAwarded with marks in LLM response
sjd210 Oct 23, 2025
b750792
Allow quiz question attempt to register with marks
sjd210 Oct 23, 2025
05b8e3c
Cleanup unnecessary changes
sjd210 Oct 23, 2025
ecd6cf4
Replace placeholder database access code
sjd210 Oct 27, 2025
00db082
Remove unused imports
sjd210 Oct 27, 2025
9f6d09a
Reorder marks column to be after correct
sjd210 Nov 5, 2025
7b16212
Fully replace marksAwarded field in LLM response
sjd210 Nov 5, 2025
08263ed
Reset correctness criteria on LLM-marked questions
sjd210 Nov 5, 2025
bedca79
Correct SQL table representation
sjd210 Nov 5, 2025
43a51be
Reset LLM question test correctness criteria
sjd210 Nov 6, 2025
3ae21ff
Replace space with tab
sjd210 Nov 6, 2025
7541eba
Explicitly cast null marks to 0
sjd210 Nov 6, 2025
07c9d40
Replace another space with tab to fix test
sjd210 Nov 6, 2025
108d55a
Explicitly ignore "marksAwarded" field in Jackson
sjd210 Nov 7, 2025
accaa39
Update script to add but not update columns first
sjd210 Nov 7, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ public GameManager(final GitContentManager contentManager,
* list of question categories (i.e. problem_solving, book) to include in filtered results
* @param boardOwner
* The user that should be marked as the creator of the gameBoard.
* @return a gameboard if possible that satisifies the conditions provided by the parameters. Will return null if no
* @return a gameboard if possible that satisfies the conditions provided by the parameters. Will return null if no
* questions can be provided.
* @throws SegueDatabaseException
* - if there is an error contacting the database.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ public PgQuizQuestionAttemptPersistenceManager(final PostgresSqlDb database, fin
@Override
public void registerQuestionAttempt(Long quizAttemptId, QuestionValidationResponse questionResponse) throws SegueDatabaseException {

String query = "INSERT INTO quiz_question_attempts(quiz_attempt_id, question_id, question_attempt, correct, \"timestamp\")" +
" VALUES (?, ?, ?::text::jsonb, ?, ?);";
String query = "INSERT INTO quiz_question_attempts(quiz_attempt_id, question_id, question_attempt, correct, marks, \"timestamp\")" +
" VALUES (?, ?, ?::text::jsonb, ?, ?, ?);";
try (Connection conn = database.getDatabaseConnection();
PreparedStatement pst = conn.prepareStatement(query, Statement.RETURN_GENERATED_KEYS);
) {
Expand All @@ -73,7 +73,14 @@ public void registerQuestionAttempt(Long quizAttemptId, QuestionValidationRespon
} else {
pst.setNull(4, Types.BOOLEAN);
}
pst.setTimestamp(5, new java.sql.Timestamp(questionResponse.getDateAttempted().getTime()));

if (questionResponse.getMarks() != null) {
pst.setInt(5, questionResponse.getMarks());
} else {
pst.setInt(5, java.sql.Types.NULL);
}

pst.setTimestamp(6, new java.sql.Timestamp(questionResponse.getDateAttempted().getTime()));

if (pst.executeUpdate() == 0) {
throw new SegueDatabaseException("Unable to save quiz question attempt.");
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package uk.ac.cam.cl.dtg.isaac.dos;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import uk.ac.cam.cl.dtg.isaac.dos.content.Choice;
import uk.ac.cam.cl.dtg.isaac.dos.content.Content;
import uk.ac.cam.cl.dtg.isaac.dos.content.DTOMapping;
Expand All @@ -9,9 +10,9 @@
import java.util.Date;
import java.util.List;

@JsonIgnoreProperties({ "marksAwarded" })
@DTOMapping(LLMFreeTextQuestionValidationResponseDTO.class)
public class LLMFreeTextQuestionValidationResponse extends QuestionValidationResponse {
private Integer marksAwarded;
private List<LLMFreeTextMarkSchemeEntry> markBreakdown;

public LLMFreeTextQuestionValidationResponse() {
Expand All @@ -23,13 +24,6 @@ public LLMFreeTextQuestionValidationResponse(final String questionId, final Choi
super(questionId, answer, correct, explanation, dateAttempted);
}

public Integer getMarksAwarded() {
return marksAwarded;
}
public void setMarksAwarded(Integer marksAwarded) {
this.marksAwarded = marksAwarded;
}

public List<LLMFreeTextMarkSchemeEntry> getMarkBreakdown() {
return markBreakdown;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ public class LightweightQuestionValidationResponse {
private String questionId;
private Boolean correct;
private Date dateAttempted;
private Integer marks;

/**
* Default Constructor for mappers.
Expand All @@ -25,13 +26,34 @@ public LightweightQuestionValidationResponse() {
* -
* @param correct
* -
* @param marks
* -
* @param dateAttempted
* -
*/
public LightweightQuestionValidationResponse(final String questionId, final Boolean correct,
final Integer marks, final Date dateAttempted) {
this.questionId = questionId;
this.correct = correct;
this.marks = marks;
this.dateAttempted = dateAttempted;
}

/**
* Constructor without specifying marks (instead derived from 'correct')
*
* @param questionId
* -
* @param correct
* -
* @param dateAttempted
* -
*/
public LightweightQuestionValidationResponse(final String questionId, final Boolean correct,
final Date dateAttempted) {
this.questionId = questionId;
this.correct = correct;
this.marks = (correct != null && correct) ? 1 : 0;
this.dateAttempted = dateAttempted;
}

Expand Down Expand Up @@ -73,6 +95,25 @@ public final void setCorrect(final Boolean correct) {
this.correct = correct;
}

/**
* Gets the marks.
*
* @return the marks
*/
public Integer getMarks() {
return marks;
}

/**
* Sets the marks.
*
* @param marks
* the marks to set
*/
public void setMarks(final Integer marks) {
this.marks = marks;
}

/**
* Gets the dateAttempted.
*
Expand All @@ -95,6 +136,6 @@ public void setDateAttempted(final Date dateAttempted) {
@Override
public String toString() {
return "QuestionValidationResponse [questionId=" + questionId + ", correct=" + correct +
", dateAttempted=" + dateAttempted + "]";
", marks=" + marks + ", dateAttempted=" + dateAttempted + "]";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,29 @@ public QuestionValidationResponse() {
* -
* @param correct
* -
* @param marks
* -
* @param explanation
* -
* @param dateAttempted
* -
*/
public QuestionValidationResponse(final String questionId, final Choice answer, final Boolean correct,
final Integer marks, final Content explanation, final Date dateAttempted) {
super(questionId, correct, marks, dateAttempted);
this.answer = answer;
this.explanation = explanation;
}

/**
* Constructor without specifying marks (instead derived from 'correct')
*
* @param questionId
* -
* @param answer
* -
* @param correct
* -
* @param explanation
* -
* @param dateAttempted
Expand Down Expand Up @@ -101,7 +124,7 @@ public final void setExplanation(final Content explanation) {
public String toString() {
return "QuestionValidationResponse [questionId=" + super.getQuestionId() + ", answer=" + answer +
", correct=" + super.isCorrect() + ", explanation=" + explanation +
", dateAttempted=" + super.getDateAttempted() + "]";
", marks=" + super.getMarks() + ", dateAttempted=" + super.getDateAttempted() + "]";
}

}
Original file line number Diff line number Diff line change
@@ -1,23 +1,17 @@
package uk.ac.cam.cl.dtg.isaac.dto;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import uk.ac.cam.cl.dtg.isaac.dto.content.LLMFreeTextMarkSchemeEntryDTO;

import java.util.List;

@JsonIgnoreProperties({ "marksAwarded" })
public class LLMFreeTextQuestionValidationResponseDTO extends QuestionValidationResponseDTO {
private Integer marksAwarded;
private List<LLMFreeTextMarkSchemeEntryDTO> markBreakdown;

public LLMFreeTextQuestionValidationResponseDTO() {
}

public Integer getMarksAwarded() {
return marksAwarded;
}
public void setMarksAwarded(Integer marksAwarded) {
this.marksAwarded = marksAwarded;
}

public List<LLMFreeTextMarkSchemeEntryDTO> getMarkBreakdown() {
return markBreakdown;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

import java.util.Date;

import uk.ac.cam.cl.dtg.isaac.dos.content.Choice;
import uk.ac.cam.cl.dtg.isaac.dos.content.Content;
import uk.ac.cam.cl.dtg.isaac.dto.content.ChoiceDTO;
import uk.ac.cam.cl.dtg.isaac.dto.content.ContentDTO;

Expand All @@ -28,6 +30,7 @@ public class QuestionValidationResponseDTO {
private String questionId;
private ChoiceDTO answer;
private Boolean correct;
private Integer marks;
private ContentDTO explanation;
private Date dateAttempted;

Expand All @@ -47,20 +50,48 @@ public QuestionValidationResponseDTO() {
* -
* @param correct
* -
* @param marks
* -
* @param explanation
* -
* @param dateAttempted
* -
*/
public QuestionValidationResponseDTO(final String questionId, final ChoiceDTO answer, final Boolean correct,
final Integer marks, final ContentDTO explanation, final Date dateAttempted) {
this.questionId = questionId;
this.answer = answer;
this.correct = correct;
this.marks = marks;
this.explanation = explanation;
this.dateAttempted = dateAttempted;
}

/**
* Constructor without specifying marks (instead derived from 'correct')
*
* @param questionId
* -
* @param answer
* -
* @param correct
* -
* @param explanation
* -
* @param dateAttempted
* -
*/
public QuestionValidationResponseDTO(final String questionId, final ChoiceDTO answer, final Boolean correct,
final ContentDTO explanation, final Date dateAttempted) {
final ContentDTO explanation, final Date dateAttempted) {
this.questionId = questionId;
this.answer = answer;
this.correct = correct;
this.marks = (correct != null && correct) ? 1 : 0;
this.explanation = explanation;
this.dateAttempted = dateAttempted;
}


/**
* Gets the questionId.
*
Expand Down Expand Up @@ -118,6 +149,25 @@ public final void setCorrect(final Boolean correct) {
this.correct = correct;
}

/**
* Gets the marks.
*
* @return the marks
*/
public Integer getMarks() {
return marks;
}

/**
* Sets the marks.
*
* @param marks
* the marks to set
*/
public void setMarks(final Integer marks) {
this.marks = marks;
}

/**
* Gets the explanation.
*
Expand Down Expand Up @@ -159,6 +209,6 @@ public void setDateAttempted(final Date dateAttempted) {
@Override
public String toString() {
return "QuestionValidationResponseDTO [questionId=" + questionId + ", answer=" + answer + ", correct="
+ correct + ", explanation=" + explanation + ", dateAttempted=" + dateAttempted + "]";
+ correct + ", marks=" + marks + ", explanation=" + explanation + ", dateAttempted=" + dateAttempted + "]";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,7 @@ private int evaluateMarkTotal(final IsaacLLMFreeTextQuestion question, final Map
* @param question the question being marked so that we can return the mark scheme.
* @param answer the user's attempt at the question.
* @param awardedMarks the marks awarded for each field in the mark scheme according to the LLM response.
* @param markTotal the calculated mark value based on which individual marks were awarded
* @return a response to the user's attempt at the question.
*/
private LLMFreeTextQuestionValidationResponse generateQuestionValidationResponse(
Expand All @@ -294,7 +295,7 @@ private LLMFreeTextQuestionValidationResponse generateQuestionValidationResponse

LLMFreeTextQuestionValidationResponse validationResponse = new LLMFreeTextQuestionValidationResponse(
question.getId(), answer, isConsideredCorrect, null, new Date());
validationResponse.setMarksAwarded(markTotal);
validationResponse.setMarks(markTotal);
validationResponse.setMarkBreakdown(markBreakdown);
return validationResponse;
}
Expand Down
23 changes: 18 additions & 5 deletions src/main/java/uk/ac/cam/cl/dtg/isaac/quiz/PgQuestionAttempts.java
Original file line number Diff line number Diff line change
Expand Up @@ -186,8 +186,8 @@ public Map<String, Map<String, List<QuestionValidationResponse>>> getAnonymousQu
public void registerQuestionAttempt(final Long userId, final String questionPageId, final String fullQuestionId,
final QuestionValidationResponse questionAttempt) throws SegueDatabaseException {

String query = "INSERT INTO question_attempts(user_id, page_id, question_id, question_attempt, correct, \"timestamp\")"
+ " VALUES (?, ?, ?, ?::text::jsonb, ?, ?);";
String query = "INSERT INTO question_attempts(user_id, page_id, question_id, question_attempt, correct, marks, \"timestamp\")"
+ " VALUES (?, ?, ?, ?::text::jsonb, ?, ?, ?);";
try (Connection conn = database.getDatabaseConnection();
PreparedStatement pst = conn.prepareStatement(query, Statement.RETURN_GENERATED_KEYS);
) {
Expand All @@ -201,7 +201,14 @@ public void registerQuestionAttempt(final Long userId, final String questionPage
} else {
pst.setNull(5, java.sql.Types.NULL);
}
pst.setTimestamp(6, new java.sql.Timestamp(questionAttempt.getDateAttempted().getTime()));

if (questionAttempt.getMarks() != null) {
pst.setInt(6, questionAttempt.getMarks());
} else {
pst.setInt(6, java.sql.Types.NULL);
}

pst.setTimestamp(7, new java.sql.Timestamp(questionAttempt.getDateAttempted().getTime()));

if (pst.executeUpdate() == 0) {
throw new SegueDatabaseException("Unable to save question attempt.");
Expand Down Expand Up @@ -260,7 +267,7 @@ public Map<Long, Map<String, Map<String, List<LightweightQuestionValidationRespo
return Collections.emptyMap();
}

String query = "SELECT id, user_id, question_id, correct, timestamp FROM question_attempts"
String query = "SELECT id, user_id, question_id, correct, marks, timestamp FROM question_attempts"
+ " WHERE user_id = ANY(?) ORDER BY \"timestamp\" ASC";

Map<Long, Map<String, Map<String, List<LightweightQuestionValidationResponse>>>> mapToReturn
Expand Down Expand Up @@ -307,7 +314,7 @@ public Map<String, Map<String, List<LightweightQuestionValidationResponse>>> get
= userIds.stream().collect(Collectors.toMap(Function.identity(), k -> Maps.newHashMap()));;

try (Connection conn = database.getDatabaseConnection()) {
String query = "SELECT id, user_id, question_id, correct, timestamp FROM question_attempts"
String query = "SELECT id, user_id, question_id, correct, marks, timestamp FROM question_attempts"
+ " WHERE user_id = ANY(?) AND page_id = ANY(?)"
+ " ORDER BY \"timestamp\" ASC";

Expand Down Expand Up @@ -441,6 +448,12 @@ private LightweightQuestionValidationResponse resultsToLightweightValidationResp
partialQuestionAttempt.setCorrect(results.getBoolean("correct"));
partialQuestionAttempt.setQuestionId(results.getString("question_id"));
partialQuestionAttempt.setDateAttempted(results.getTimestamp("timestamp"));
int marks = results.getInt("marks");
if (results.wasNull()) {
partialQuestionAttempt.setMarks(0);
} else {
partialQuestionAttempt.setMarks(marks);
}

return partialQuestionAttempt;
}
Expand Down
Loading
Loading