Skip to content

use parameterized sqlite for insertRowIntoDB #385

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 6 commits into
base: master
Choose a base branch
from
Draft

use parameterized sqlite for insertRowIntoDB #385

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
5423509
use parameterized sqlite for `instertRowIntoDB`
jspjutNV Jun 16, 2022
f7decaf
Merge branch 'master' into ParameterizedSQL
jspjutNV Aug 31, 2022
0953f9e
WIP toward insert rows
jspjutNV Aug 31, 2022
6335b4a
Fix max variables by splitting into groups
jspjutNV Aug 31, 2022
fc0dc14
Remove old string sanitizing
jspjutNV Aug 31, 2022
d8000c4
Fix to update session on actually complete
bboudaoud-nv Aug 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 38 additions & 38 deletions source/Logger.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,19 +67,19 @@ void FPSciLogger::initResultsFile(const String& filename,
// Add the session info to the sessions table
m_openTimeStr = genUniqueTimestamp();
RowEntry sessValues = {
"'" + sessConfig->id + "'",
"'" + m_openTimeStr + "'",
"'" + m_openTimeStr + "'",
"'" + subjectID + "'",
"'" + description + "'",
sessConfig->id,
m_openTimeStr,
m_openTimeStr,
subjectID,
description,
"false",
"0"
};

// Create any table to do lookup here
Any a = sessConfig->toAny(true);
// Add the looked up values
for (String name : sessConfig->logger.sessParamsToLog) { sessValues.append("'" + a[name].unparse() + "'"); }
for (String name : sessConfig->logger.sessParamsToLog) { sessValues.append(a[name].unparse()); }
// add header row
insertRowIntoDB(m_db, "Sessions", sessValues);

Expand All @@ -102,10 +102,10 @@ void FPSciLogger::createExperimentsTable(const String& expConfigFilename) {

// Update row
RowEntry expRow = {
"'" + expConfig.description + "'",
"'" + genUniqueTimestamp() + "'",
"'" + format("0x%x", hash) + "'",
"'" + readWholeFile(expConfigFilename) + "'"
expConfig.description,
genUniqueTimestamp(),
format("0x%x", hash),
readWholeFile(expConfigFilename)
};
insertRowIntoDB(m_db, "Experiments", expRow);
}
Expand All @@ -123,7 +123,7 @@ void FPSciLogger::createSessionsTable(const shared_ptr<SessionConfig>& sessConfi
{ "trials_complete", "integer" }
};
// add any user-specified parameters as headers
for (String name : sessConfig->logger.sessParamsToLog) { sessColumns.append({ "'" + name + "'", "text", "NOT NULL" }); }
for (String name : sessConfig->logger.sessParamsToLog) { sessColumns.append({ name, "text", "NOT NULL" }); }
createTableInDB(m_db, "Sessions", sessColumns); // no need of Primary Key for this table.
}

Expand Down Expand Up @@ -168,9 +168,9 @@ void FPSciLogger::logTargetTypes(const Array<shared_ptr<TargetConfig>>& targets)
const String type = (config->destinations.size() > 0) ? "waypoint" : "parametrized";
const String modelName = config->modelSpec["filename"];
const RowEntry targetTypeRow = {
"'" + config->id + "'",
"'" + type + "'",
"'" + config->destSpace + "'",
config->id,
type,
config->destSpace,
String(std::to_string(config->size[0])),
String(std::to_string(config->size[1])),
config->symmetricEccH ? "true" : "false",
Expand All @@ -184,7 +184,7 @@ void FPSciLogger::logTargetTypes(const Array<shared_ptr<TargetConfig>>& targets)
String(std::to_string(config->motionChangePeriod[0])),
String(std::to_string(config->motionChangePeriod[1])),
config->jumpEnabled ? "true" : "false",
"'" + modelName + "'"
modelName
};
rows.append(targetTypeRow);
}
Expand All @@ -206,9 +206,9 @@ void FPSciLogger::createTargetsTable() {

void FPSciLogger::addTarget(const String& name, const shared_ptr<TargetConfig>& config, const String& spawnTime, const float& size, const Point2& spawnEcc) {
const RowEntry targetValues = {
"'" + name + "'",
"'" + config->id + "'",
"'" + spawnTime + "'",
name,
config->id,
spawnTime,
String(std::to_string(size)),
String(std::to_string(spawnEcc.x)),
String(std::to_string(spawnEcc.y)),
Expand Down Expand Up @@ -251,9 +251,9 @@ void FPSciLogger::recordTargetLocations(const Array<TargetLocation>& locations)
for (const auto& loc : locations) {
String stateStr = presentationStateToString(loc.state);
Array<String> targetTrajectoryValues = {
"'" + FPSciLogger::formatFileTime(loc.time) + "'",
"'" + loc.name + "'",
"'" + stateStr + "'",
FPSciLogger::formatFileTime(loc.time),
loc.name,
stateStr,
String(std::to_string(loc.position.x)),
String(std::to_string(loc.position.y)),
String(std::to_string(loc.position.z)),
Expand Down Expand Up @@ -294,15 +294,15 @@ void FPSciLogger::recordPlayerActions(const Array<PlayerAction>& actions) {
}

Array<String> playerActionValues = {
"'" + FPSciLogger::formatFileTime(action.time) + "'",
FPSciLogger::formatFileTime(action.time),
String(std::to_string(action.viewDirection.x)),
String(std::to_string(action.viewDirection.y)),
String(std::to_string(action.position.x)),
String(std::to_string(action.position.y)),
String(std::to_string(action.position.z)),
"'" + stateStr + "'",
"'" + actionStr + "'",
"'" + action.targetName + "'",
stateStr,
actionStr,
action.targetName,
};
rows.append(playerActionValues);
}
Expand All @@ -323,7 +323,7 @@ void FPSciLogger::recordFrameInfo(const Array<FrameInfo>& frameInfo) {
Array<RowEntry> rows;
for (FrameInfo info : frameInfo) {
Array<String> frameValues = {
"'" + FPSciLogger::formatFileTime(info.time) + "'",
FPSciLogger::formatFileTime(info.time),
//String(std::to_string(info.idt)),
String(std::to_string(info.sdt))
};
Expand Down Expand Up @@ -355,13 +355,13 @@ void FPSciLogger::addQuestion(Question q, String session, const shared_ptr<Dialo
orderStr = Any(dynamic_pointer_cast<SelectionDialog>(dialog)->options()).unparse();
}
RowEntry rowContents = {
"'" + time + "'",
"'" + session + "'",
"'" + q.prompt + "'",
"'" + optStr + "'",
"'" + keyStr + "'",
"'" + orderStr + "'",
"'" + q.result + "'"
time,
session,
q.prompt,
optStr,
keyStr,
orderStr,
q.result
};
logQuestionResult(rowContents);
}
Expand Down Expand Up @@ -400,17 +400,17 @@ void FPSciLogger::logUserConfig(const UserConfig& user, const String& sessId, co
const String time = genUniqueTimestamp();

RowEntry row = {
"'" + user.id + "'",
"'" + sessId + "'",
"'" + time + "'",
user.id,
sessId,
time,
String(std::to_string(cmp360)),
String(std::to_string(user.mouseDegPerMm)),
String(std::to_string(user.mouseDPI)),
String(std::to_string(user.reticle.index)),
String(std::to_string(user.reticle.scale[0])),
String(std::to_string(user.reticle.scale[1])),
"'" + user.reticle.color[0].toString() + "'",
"'" + user.reticle.color[1].toString() + "'",
user.reticle.color[0].toString(),
user.reticle.color[1].toString(),
String(std::to_string(user.reticle.changeTimeS)),
String(std::to_string(user.turnScale.x)),
String(std::to_string(userYTurnScale)),
Expand Down
12 changes: 6 additions & 6 deletions source/Session.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,7 @@ void Session::processResponse()
if (notNull(logger)) {
int totalTrials = 0;
for (int tCount : m_completedTrials) { totalTrials += tCount; }
logger->updateSessionEntry((m_remainingTrials[m_currTrialIdx] == 0), totalTrials); // Update session entry in database
logger->updateSessionEntry(false, totalTrials); // Update session entry in database
}

// Check for whether all targets have been destroyed
Expand Down Expand Up @@ -480,7 +480,7 @@ void Session::updatePresentationState()
if (notNull(logger) && m_config->logger.enable) {
int totalTrials = 0;
for (int tCount : m_completedTrials) { totalTrials += tCount; }
logger->updateSessionEntry((m_remainingTrials[m_currTrialIdx] == 0), totalTrials); // Update session entry in database
logger->updateSessionEntry(true, totalTrials); // Update session entry in database
}
if (m_config->logger.enable) {
endLogging();
Expand Down Expand Up @@ -588,12 +588,12 @@ void Session::recordTrialResponse(int destroyedTargets, int totalTargets)
if (m_config->logger.logTrialResponse) {
// Trials table. Record trial start time, end time, and task completion time.
FPSciLogger::TrialValues trialValues = {
"'" + m_config->id + "'",
m_config->id,
String(std::to_string(m_currTrialIdx)),
String(std::to_string(m_completedTrials[m_currTrialIdx])),
format("'Block %d'", m_currBlock),
"'" + m_taskStartTime + "'",
"'" + m_taskEndTime + "'",
format("Block %d", m_currBlock),
m_taskStartTime,
m_taskEndTime,
String(std::to_string(m_pretrialDuration)),
String(std::to_string(m_taskExecutionTime)),
String(std::to_string(destroyedTargets)),
Expand Down
132 changes: 106 additions & 26 deletions source/sqlHelpers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,56 +27,136 @@ bool createTableInDB(sqlite3* db, const String tableName, const Array<Array<Stri
return ret == SQLITE_OK;
}

bool insertRowIntoDB(sqlite3* db, const String tableName, const Array<String>& values, const String colNames) {
bool insertRowIntoDB(sqlite3* db, const String tableName, const Array<String>& values) {
if (values.length() == 0) {
logPrintf("Warning insert row with empty values ignored!\n");
return false; // Don't attempt to insert for empty values
return false;
}
// Quotes must be added around text-type values (eg. "addQuotes(expVersion)")
// Note that ID does not need to be provided unless PRIMARY KEY is set.
String insertC = "INSERT INTO " + tableName + colNames + " VALUES(";

// ? is a variable that will be bound later with a `sqlite3_bind_XXXX` function
// see https://www2.sqlite.org/draft/c3ref/bind_blob.html for details
String insertC = "INSERT INTO " + tableName + " VALUES(";
for (int i = 0; i < values.size(); i++) {
insertC += values[i];
insertC += "?";
if(i < values.size() - 1) insertC += ",";
}
insertC += ");";

// prepare
sqlite3_stmt* res;
int ret = sqlite3_prepare_v2(db, insertC.c_str(), -1, &res, 0);
if (ret != SQLITE_OK)
{
logPrintf("Error preparing INSERT INTO statement (%s): %s\n", insertC, sqlite3_errmsg(db));
return ret == SQLITE_OK;
}
// bind values
for (int i = 0; i < values.size(); i++) {
// All values sent to this function are explicitly Strings and get stored as text in the database
sqlite3_bind_text(res, i + 1, values[i].c_str(), -1, SQLITE_TRANSIENT);
}
ret = sqlite3_step(res);
if (ret != SQLITE_DONE)
{
logPrintf("Error in INSERT (%s) with VALUE including %s!\n", insertC, values[0]);
}
// clean up the sqlite3_stmt
ret = sqlite3_finalize(res);
//logPrintf("Inserting row into %s table w/ SQL query:%s\n\n", tableName.c_str(), insertC.c_str());
char* errmsg;
int ret = sqlite3_exec(db, insertC.c_str(), 0, 0, &errmsg);
if (ret != SQLITE_OK) {
logPrintf("Error in INSERT INTO statement (%s): %s\n", insertC, errmsg);
logPrintf("Error in INSERT INTO statement (%s): %s\n", insertC, sqlite3_errmsg(db));
}
return ret == SQLITE_OK;
}

bool insertRowsIntoDB(sqlite3* db, const String tableName, const Array<Array<String>>& value_vector, const String colNames) {
if (value_vector.length() == 0) {
logPrintf("Warning insert rows with empty row value array ignored!\n");
return false; // Don't insert for empty value vector (creates an error)
}
// Quotes must be added around text-type values
// Note that ID does not need to be provided unless PRIMARY KEY is set
String insertC = "INSERT INTO " + tableName + colNames + " VALUES";
for (int i = 0; i < value_vector.size(); ++i) {
/** Helper function that takes in a start and end row and only inserts that given range.
Assumes the range is valid and below the SQLITE_LIMIT_VARIABLE_NUMBER */
bool groupInsertRows(sqlite3* db, const String tableName, const Array<Array<String>>& value_vector, const int start_row, const int end_row) {

// ? is a variable that will be bound later with a `sqlite3_bind_XXXX` function
// see https://www2.sqlite.org/draft/c3ref/bind_blob.html for details
String insertC = "INSERT INTO " + tableName + " VALUES";
for (int i = start_row; i < end_row; ++i) {
insertC += "(";
for (int j = 0; j < value_vector[i].size(); j++) {
insertC += value_vector[i][j];
insertC += "?";
if (j < value_vector[i].size() - 1) insertC += ",";
}
insertC += ")";
if (i < value_vector.size() - 1) { // We have more rows coming after this row.
insertC += ",";
if (i < end_row - 1) {
// We have more rows coming after this row.
insertC += ",";
}
else { // The last row of this insert operation. Terminate it with a semi-colon.
else {
// The last row of this insert operation. Terminate it with a semi-colon (which is optional).
insertC += ";";
}
}
//logPrintf("Inserting rows into %s table with SQL query:%s\n\n", tableName.c_str(), insertC.c_str());
char* errmsg;
int ret = sqlite3_exec(db, insertC.c_str(), 0, 0, &errmsg);
logPrintf("insertRowsIntoDB: %s\n\n", insertC);


// prepare
sqlite3_stmt* res;
int ret = sqlite3_prepare_v2(db, insertC.c_str(), -1, &res, 0);
if (ret != SQLITE_OK)
{
logPrintf("Error preparing INSERT INTO statement (%s): %s\n", insertC, sqlite3_errmsg(db));
return ret == SQLITE_OK;
}
// bind values
for (int i = start_row; i < end_row; ++i) {
for (int j = 0; j < value_vector[i].size(); j++) {
sqlite3_bind_text(res, i * value_vector[i].size() + j + 1, value_vector[i][j].c_str(), -1, SQLITE_TRANSIENT);
}
}
ret = sqlite3_step(res);
if (ret != SQLITE_DONE)
{
logPrintf("Error in INSERT (%s) with VALUE including %s!\n", insertC, value_vector[0][0]);
}
// clean up the sqlite3_stmt
ret = sqlite3_finalize(res);
//logPrintf("Inserting row into %s table w/ SQL query:%s\n\n", tableName.c_str(), insertC.c_str());
if (ret != SQLITE_OK) {
logPrintf("Error in INSERT INTO statement (%s): %s\n", insertC, errmsg);
logPrintf("Error in INSERT INTO statement (%s): %s\n", insertC, sqlite3_errmsg(db));
}
return ret == SQLITE_OK;
}

bool insertRowsIntoDB(sqlite3* db, const String tableName, const Array<Array<String>>& value_vector) {
if (value_vector.length() == 0) {
logPrintf("Warning insert rows with empty row value array ignored!\n");
return false;
}

// Figure out how many insert groups we need
// SQLITE_LIMIT_VARIABLE_NUMBER indicates the max number of variables in the compiled sqlite version
int max_variables = sqlite3_limit(db, SQLITE_LIMIT_VARIABLE_NUMBER, -1);
int num_rows_per_group = max_variables / value_vector[0].size();
//logPrintf("insertRows %d total in %d rows with %d groups of %d per INSERT\n",
// value_vector.size() * value_vector[0].size(),
// value_vector.size(),
// num_rows_per_group,
// value_vector[0].size()
// );

int ret = 0; // sqlite return value
int start_row = 0;
bool ret_val = true;
while (start_row < value_vector.size()) {
int end_row = min(start_row + num_rows_per_group, value_vector.size());
//logPrintf("inserting rows %d to %d\n", start_row, end_row);

bool success = groupInsertRows(db, tableName, value_vector, start_row, end_row);

if (!success) {
logPrintf("Failed group insert!\n");
ret_val = false;
}

start_row += num_rows_per_group;
}

return ret_val;
}

5 changes: 3 additions & 2 deletions source/sqlHelpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,6 @@


bool createTableInDB(sqlite3* db, const String tableName, const Array<Array<String>>& columns);
bool insertRowIntoDB(sqlite3* db, const String tableName, const Array<String>& values, const String colNames = "");
bool insertRowsIntoDB(sqlite3* db, const String tableName, const Array<Array<String>>& valueVector, const String colNames = "");
/**All values sent to this function are stored in the database as Strings.*/
bool insertRowIntoDB(sqlite3* db, const String tableName, const Array<String>& values);
bool insertRowsIntoDB(sqlite3* db, const String tableName, const Array<Array<String>>& valueVector);