From 61ea4318ddeff44a538d3bd68f3ad46c6a13d73b Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Sat, 8 Feb 2025 15:40:48 +1100 Subject: [PATCH 01/41] Initial commit of DatabaseIngestor class. --- engine/CSVParser.cc | 142 ------------------------------------- engine/CSVParser.h | 145 ++++++++++++++++++++++++++++++++++++++ model/databaseIngestor.cc | 116 ++++++++++++++++++++++++++++++ model/databaseIngestor.h | 45 ++++++++++++ 4 files changed, 306 insertions(+), 142 deletions(-) create mode 100644 model/databaseIngestor.cc create mode 100644 model/databaseIngestor.h diff --git a/engine/CSVParser.cc b/engine/CSVParser.cc index 2dafae9e6..297800159 100644 --- a/engine/CSVParser.cc +++ b/engine/CSVParser.cc @@ -42,150 +42,8 @@ using namespace std; #include #include -namespace escapedListSeparator -{ - // pinched from boost::escape_list_separator, and modified to not throw - template ::traits_type > - class EscapedListSeparator { - - private: - typedef std::basic_string string_type; - struct char_eq { - Char e_; - char_eq(Char e):e_(e) { } - bool operator()(Char c) { - return Traits::eq(e_,c); - } - }; - string_type escape_; - string_type c_; - string_type quote_; - bool last_; - - bool is_escape(Char e) { - const char_eq f(e); - return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end(); - } - bool is_c(Char e) { - const char_eq f(e); - return std::find_if(c_.begin(),c_.end(),f)!=c_.end(); - } - bool is_quote(Char e) { - const char_eq f(e); - return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end(); - } - template - void do_escape(iterator& next,iterator end,Token& tok) { - if (++next >= end) - // don't throw, but pass on verbatim - tok+=escape_.front(); - if (Traits::eq(*next,'n')) { - tok+='\n'; - return; - } - if (is_quote(*next)) { - tok+=*next; - return; - } - if (is_c(*next)) { - tok+=*next; - return; - } - if (is_escape(*next)) { - tok+=*next; - return; - } - // don't throw, but pass on verbatim - tok+=escape_.front()+*next; - } - - public: - - explicit EscapedListSeparator(Char e = '\\', - Char c = ',',Char q = '\"') - : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { } - - EscapedListSeparator(string_type e, string_type c, string_type q) - : escape_(e), c_(c), quote_(q), last_(false) { } - - void reset() {last_=false;} - - template - bool operator()(InputIterator& next,InputIterator end,Token& tok) { - bool bInQuote = false; - tok = Token(); - - if (next >= end) { - next=end; // reset next in case it has adavanced beyond - if (last_) { - last_ = false; - return true; - } - return false; - } - last_ = false; - while (next < end) { - if (is_escape(*next)) { - do_escape(next,end,tok); - } - else if (is_c(*next)) { - if (!bInQuote) { - // If we are not in quote, then we are done - ++next; - // The last character was a c, that means there is - // 1 more blank field - last_ = true; - return true; - } - tok+=*next; - } - else if (is_quote(*next)) { - bInQuote=!bInQuote; - } - else { - tok += *next; - } - ++next; - } - return true; - } - }; -} -using Parser=escapedListSeparator::EscapedListSeparator; - typedef boost::tokenizer Tokenizer; -struct SpaceSeparatorParser -{ - char escape, quote; - SpaceSeparatorParser(char escape='\\', char sep=' ', char quote='"'): - escape(escape), quote(quote) {} - template - bool operator()(I& next, I end, std::string& tok) - { - tok.clear(); - bool quoted=false; - while (next!=end) - { - if (*next==escape) - tok+=*(++next); - else if (*next==quote) - quoted=!quoted; - else if (!quoted && isspace(*next)) - { - while (isspace(*next)) ++next; - return true; - } - else - tok+=*next; - ++next; - } - return !tok.empty(); - } - void reset() {} -}; - namespace { /// An any with cached hash diff --git a/engine/CSVParser.h b/engine/CSVParser.h index 6e34a79dd..0f4236ee8 100644 --- a/engine/CSVParser.h +++ b/engine/CSVParser.h @@ -125,6 +125,151 @@ namespace minsky /// replace doubled quotes with escaped quotes void escapeDoubledQuotes(std::string&,const DataSpec&); + + namespace escapedListSeparator +{ + // pinched from boost::escape_list_separator, and modified to not throw + template ::traits_type > + class EscapedListSeparator { + + private: + typedef std::basic_string string_type; + struct char_eq { + Char e_; + char_eq(Char e):e_(e) { } + bool operator()(Char c) { + return Traits::eq(e_,c); + } + }; + string_type escape_; + string_type c_; + string_type quote_; + bool last_; + + bool is_escape(Char e) { + const char_eq f(e); + return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end(); + } + bool is_c(Char e) { + const char_eq f(e); + return std::find_if(c_.begin(),c_.end(),f)!=c_.end(); + } + bool is_quote(Char e) { + const char_eq f(e); + return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end(); + } + template + void do_escape(iterator& next,iterator end,Token& tok) { + if (++next >= end) + // don't throw, but pass on verbatim + tok+=escape_.front(); + if (Traits::eq(*next,'n')) { + tok+='\n'; + return; + } + if (is_quote(*next)) { + tok+=*next; + return; + } + if (is_c(*next)) { + tok+=*next; + return; + } + if (is_escape(*next)) { + tok+=*next; + return; + } + // don't throw, but pass on verbatim + tok+=escape_.front()+*next; + } + + public: + + explicit EscapedListSeparator(Char e = '\\', + Char c = ',',Char q = '\"') + : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { } + + EscapedListSeparator(string_type e, string_type c, string_type q) + : escape_(e), c_(c), quote_(q), last_(false) { } + + void reset() {last_=false;} + + template + bool operator()(InputIterator& next,InputIterator end,Token& tok) { + bool bInQuote = false; + tok = Token(); + + if (next >= end) { + next=end; // reset next in case it has adavanced beyond + if (last_) { + last_ = false; + return true; + } + return false; + } + last_ = false; + while (next < end) { + if (is_escape(*next)) { + do_escape(next,end,tok); + } + else if (is_c(*next)) { + if (!bInQuote) { + // If we are not in quote, then we are done + ++next; + // The last character was a c, that means there is + // 1 more blank field + last_ = true; + return true; + } + tok+=*next; + } + else if (is_quote(*next)) { + bInQuote=!bInQuote; + } + else { + tok += *next; + } + ++next; + } + return true; + } + }; +} +using Parser=escapedListSeparator::EscapedListSeparator; + +struct SpaceSeparatorParser +{ + char escape, quote; + SpaceSeparatorParser(char escape='\\', char sep=' ', char quote='"'): + escape(escape), quote(quote) {} + template + bool operator()(I& next, I end, std::string& tok) + { + tok.clear(); + bool quoted=false; + while (next!=end) + { + if (*next==escape) + tok+=*(++next); + else if (*next==quote) + quoted=!quoted; + else if (!quoted && isspace(*next)) + { + while (isspace(*next)) ++next; + return true; + } + else + tok+=*next; + ++next; + } + return !tok.empty(); + } + void reset() {} +}; + + + } #include "CSVParser.cd" diff --git a/model/databaseIngestor.cc b/model/databaseIngestor.cc new file mode 100644 index 000000000..f2ceba78f --- /dev/null +++ b/model/databaseIngestor.cc @@ -0,0 +1,116 @@ +/* + @copyright Steve Keen 2025 + @author Russell Standish + This file is part of Minsky. + + Minsky is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Minsky is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Minsky. If not, see . +*/ + +#include "databaseIngestor.h" +using soci::use; +using soci::transaction; + +namespace minsky +{ + vector parseRow(const string& line, char separator) + { + if (separator==' ') + { + SpaceSeparatorParser csvParser; + const boost::tokenizer tok(line.begin(),line.end(), csvParser); + return {tok.begin(), tok.end();} + } + Parser csvParser; + const boost::tokenizer tok(line.begin(),line.end(), csvParser); + return {tok.begin(), tok.end();} + } + + void DatabaseIngestor::createTable + (const std::vector& filenames, const DataSpecSchema& spec) + { + session<<"drop "+table+" if exists"; + // for now, load time data as strings - TODO handle date-time parsing + string def="create table "+table+" ("; + for (size_t i=0; i DatabaseIngestor::load + (soci::statement& stmt, const std::vector& filenames, + const DataSpecSchema& spec) + { + Tokeniser csvParser; + vector cells; + // prepare the insertion string based on spec + auto statement=(session.prepare<<"upsert into "+table+" ...",use(cells)); + for (auto f: filenames) + { + ifstream input(f); + size_t row=0; + string line; + for (; getWholeLine(input,line,spec) && row tok(line.begin(),line.end(), csvParser); + cells.assign(tok.begin(), tok.end()); + stmt.execute(true); + if (row%100==0) + { + session.commit(); + session.begin(); + } + } + transaction.commit(); + } + } + + void DatabaseIngestor::importFromCSV + (const std::vector& filenames, const DataSpecSchema& spec) + { + if (!session.is_connected()) session.reconnect(); + if (!session.is_connected()) return; + + // TODO check if table exists, and call createTable if not + // select * from table limit 1 - and check for exception thrown? + + + if (spec.separator==' ') + load(statement,filenames,spec); + else + load(statement,filenames,spec); + } + +} diff --git a/model/databaseIngestor.h b/model/databaseIngestor.h new file mode 100644 index 000000000..7ca8af825 --- /dev/null +++ b/model/databaseIngestor.h @@ -0,0 +1,45 @@ +/* + @copyright Steve Keen 2025 + @author Russell Standish + This file is part of Minsky. + + Minsky is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Minsky is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Minsky. If not, see . +*/ + +#ifndef DATABSE_INGESTOR_H +#define DATABSE_INGESTOR_H + +#include "CSVParser.h" + +#include +#include + +namespace minsky +{ + class DatabaseIngestor + { + soci::session session; + template DatabaseIngestor::load + (const std::vector& filenames, const DataSpecSchema& spec); + public: + void connect(const std::string& dbType, const std::string& connection) + {session.open(dbType,connection);} + std::string table; // table name to use + /// import CSV files, using \a spec + void importFromCSV(const std::vector& filenames, + const DataSpecSchema& spec); + }; +} + +#endif From 1e1a6b503f33516027ba59d2ac93f9e8af361bb4 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Sun, 9 Feb 2025 17:58:19 +1100 Subject: [PATCH 02/41] Trying to build database ingestor. --- Makefile | 2 +- ecolab | 2 +- engine/CSVParser.h | 14 +++++++++----- model/databaseIngestor.cc | 17 ++++++++++++----- model/databaseIngestor.h | 15 +++++++++++---- model/minsky.h | 2 ++ 6 files changed, 36 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index f9ffad59c..bb63b050e 100644 --- a/Makefile +++ b/Makefile @@ -144,7 +144,7 @@ PREFIX=/usr/local # custom one that picks up its scripts from a relative library # directory MODLINK=$(LIBMODS:%=$(ECOLAB_HOME)/lib/%) -MODEL_OBJS=autoLayout.o cairoItems.o canvas.o CSVDialog.o dataOp.o equationDisplay.o godleyIcon.o godleyTable.o godleyTableWindow.o grid.o group.o item.o intOp.o lasso.o lock.o minsky.o operation.o operationRS.o operationRS1.o operationRS2.o phillipsDiagram.o plotWidget.o port.o pubTab.o ravelWrap.o renderNativeWindow.o selection.o sheet.o SVGItem.o switchIcon.o userFunction.o userFunction_units.o variableInstanceList.o variable.o variablePane.o windowInformation.o wire.o +MODEL_OBJS=autoLayout.o cairoItems.o canvas.o CSVDialog.o databaseIngestor.o dataOp.o equationDisplay.o godleyIcon.o godleyTable.o godleyTableWindow.o grid.o group.o item.o intOp.o lasso.o lock.o minsky.o operation.o operationRS.o operationRS1.o operationRS2.o phillipsDiagram.o plotWidget.o port.o pubTab.o ravelWrap.o renderNativeWindow.o selection.o sheet.o SVGItem.o switchIcon.o userFunction.o userFunction_units.o variableInstanceList.o variable.o variablePane.o windowInformation.o wire.o ENGINE_OBJS=coverage.o clipboard.o derivative.o equationDisplayRender.o equations.o evalGodley.o evalOp.o flowCoef.o \ godleyExport.o latexMarkup.o valueId.o variableValue.o node_latex.o node_matlab.o CSVParser.o \ minskyTensorOps.o mdlReader.o saver.o rungeKutta.o diff --git a/ecolab b/ecolab index c4ddda0a5..35989cd51 160000 --- a/ecolab +++ b/ecolab @@ -1 +1 @@ -Subproject commit c4ddda0a585723d10e3e05ebc192bb51b77df61f +Subproject commit 35989cd51365470d38b4a6931a853a5e37846566 diff --git a/engine/CSVParser.h b/engine/CSVParser.h index 0f4236ee8..ff948e9b0 100644 --- a/engine/CSVParser.h +++ b/engine/CSVParser.h @@ -126,6 +126,9 @@ namespace minsky /// replace doubled quotes with escaped quotes void escapeDoubledQuotes(std::string&,const DataSpec&); + /// get complete line from input, allowing for quoted linefeed + bool getWholeLine(std::istream& input, std::string& line, const DataSpec& spec); + namespace escapedListSeparator { // pinched from boost::escape_list_separator, and modified to not throw @@ -186,11 +189,12 @@ namespace minsky public: - explicit EscapedListSeparator(Char e = '\\', - Char c = ',',Char q = '\"') - : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { } - - EscapedListSeparator(string_type e, string_type c, string_type q) + explicit EscapedListSeparator(Char e = '\\') + : escape_(1,e), c_(1,','), quote_(1,'\"'), last_(false) { } + EscapedListSeparator(Char e, Char c,Char q = '\"') + : escape_(1,'\\'), c_(1,','), quote_(1,q), last_(false) { } + EscapedListSeparator(EscapedListSeparator::string_type e, EscapedListSeparator::string_type c, + EscapedListSeparator::string_type q) : escape_(e), c_(c), quote_(q), last_(false) { } void reset() {last_=false;} diff --git a/model/databaseIngestor.cc b/model/databaseIngestor.cc index f2ceba78f..81cb1e0f6 100644 --- a/model/databaseIngestor.cc +++ b/model/databaseIngestor.cc @@ -18,8 +18,13 @@ */ #include "databaseIngestor.h" +#include "databaseIngestor.rcd" +#include "minsky_epilogue.h" + +using civita::Dimension; using soci::use; using soci::transaction; +using namespace std; namespace minsky { @@ -29,15 +34,15 @@ namespace minsky { SpaceSeparatorParser csvParser; const boost::tokenizer tok(line.begin(),line.end(), csvParser); - return {tok.begin(), tok.end();} + return {tok.begin(), tok.end()}; } Parser csvParser; const boost::tokenizer tok(line.begin(),line.end(), csvParser); - return {tok.begin(), tok.end();} + return {tok.begin(), tok.end()}; } void DatabaseIngestor::createTable - (const std::vector& filenames, const DataSpecSchema& spec) + (const std::vector& filenames, const DataSpec& spec) { session<<"drop "+table+" if exists"; // for now, load time data as strings - TODO handle date-time parsing @@ -57,8 +62,9 @@ namespace minsky } if (!filenames.empty()) { - ifstream input(filenames.begin()); - for (; getWholeLine(input,line,spec) && row DatabaseIngestor::load (soci::statement& stmt, const std::vector& filenames, const DataSpecSchema& spec) diff --git a/model/databaseIngestor.h b/model/databaseIngestor.h index 7ca8af825..4473266f4 100644 --- a/model/databaseIngestor.h +++ b/model/databaseIngestor.h @@ -22,6 +22,7 @@ #include "CSVParser.h" +#include #include #include @@ -30,16 +31,22 @@ namespace minsky class DatabaseIngestor { soci::session session; - template DatabaseIngestor::load - (const std::vector& filenames, const DataSpecSchema& spec); + template void load(const std::vector&, const DataSpec&); + CLASSDESC_ACCESS(DatabaseIngestor); public: + ///< connect to a database. Consult SOCI documentation for meaning of the two parameters void connect(const std::string& dbType, const std::string& connection) {session.open(dbType,connection);} - std::string table; // table name to use + std::string table; ///< table name to use + /// create an empty table satisfying \a filenames and \a spec + void createTable + (const std::vector& filenames, const DataSpec& spec); /// import CSV files, using \a spec void importFromCSV(const std::vector& filenames, - const DataSpecSchema& spec); + const DataSpec& spec); }; } +#include "databaseIngestor.cd" +#include "databaseIngestor.xcd" #endif diff --git a/model/minsky.h b/model/minsky.h index c4ec3cb9e..8bfa47ca7 100644 --- a/model/minsky.h +++ b/model/minsky.h @@ -26,6 +26,7 @@ #include "cairoItems.h" #include "canvas.h" #include "clipboard.h" +#include "databaseIngestor.h" #include "dimension.h" #include "evalOp.h" #include "equationDisplay.h" @@ -154,6 +155,7 @@ namespace minsky FontDisplay fontSampler; PhillipsDiagram phillipsDiagram; std::vector publicationTabs; + DatabaseIngestor databaseIngestor; void addNewPublicationTab(const std::string& name) {publicationTabs.emplace_back(name);} void addCanvasItemToPublicationTab(size_t i) { From 90e7c894ac2a5f2c89315323069a3bb34800b201 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Mon, 10 Feb 2025 11:06:33 +1100 Subject: [PATCH 03/41] Compiles now. --- Makefile | 2 +- RESTService/typescriptAPI.cc | 1 + engine/CSVParser.cc | 4 +-- engine/CSVParser.h | 11 +++--- gui-js/libs/shared/src/lib/backend/minsky.ts | 12 +++++++ model/databaseIngestor.cc | 35 +++++++++++--------- model/databaseIngestor.h | 2 +- 7 files changed, 43 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index bb63b050e..548fdd4f2 100644 --- a/Makefile +++ b/Makefile @@ -307,7 +307,7 @@ endif LIBS+= -LRavelCAPI -lravelCAPI -LRavelCAPI/civita -lcivita \ -lboost_system$(BOOST_EXT) -lboost_regex$(BOOST_EXT) \ -lboost_date_time$(BOOST_EXT) -lboost_program_options$(BOOST_EXT) \ - -lboost_filesystem$(BOOST_EXT) -lboost_thread$(BOOST_EXT) -lgsl -lgslcblas -lssl -lcrypto + -lboost_filesystem$(BOOST_EXT) -lboost_thread$(BOOST_EXT) -lsoci_core -lgsl -lgslcblas -lssl -lcrypto ifdef MXE LIBS+=-lcrypt32 -lbcrypt -lshcore diff --git a/RESTService/typescriptAPI.cc b/RESTService/typescriptAPI.cc index dd5a0592d..f2ffcd275 100644 --- a/RESTService/typescriptAPI.cc +++ b/RESTService/typescriptAPI.cc @@ -12,6 +12,7 @@ #include "CSVDialog.tcd" #include "CSVParser.tcd" #include "constMap.tcd" +#include "databaseIngestor.tcd" #include "dataSpecSchema.tcd" #include "dataOp.h" #include "dataOp.tcd" diff --git a/engine/CSVParser.cc b/engine/CSVParser.cc index 297800159..439ce91fe 100644 --- a/engine/CSVParser.cc +++ b/engine/CSVParser.cc @@ -439,7 +439,7 @@ namespace minsky } // gets a line, accounting for quoted newlines - bool getWholeLine(istream& input, string& line, const DataSpec& spec) + bool getWholeLine(istream& input, string& line, const DataSpecSchema& spec) { line.clear(); bool r=getline(input,line).good(); @@ -460,7 +460,7 @@ namespace minsky return r || !line.empty(); } - void escapeDoubledQuotes(std::string& line,const DataSpec& spec) + void escapeDoubledQuotes(std::string& line,const DataSpecSchema& spec) { // replace doubled quotes with escape quote for (size_t i=1; i::string_type e, +// typename EscapedListSeparator::string_type c, +// EscapedListSeparator::string_type q) +// : escape_(e), c_(c), quote_(q), last_(false) { } void reset() {last_=false;} diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index 72c8be43c..edeaa34d1 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -608,6 +608,16 @@ export class DataSpecSchema extends CppClass { async separator(...args: number[]): Promise {return this.$callMethod('separator',...args);} } +export class DatabaseIngestor extends CppClass { + constructor(prefix: string){ + super(prefix); + } + async connect(a1: string,a2: string): Promise {return this.$callMethod('connect',a1,a2);} + async createTable(a1: string[],a2: DataSpec): Promise {return this.$callMethod('createTable',a1,a2);} + async importFromCSV(a1: string[],a2: DataSpec): Promise {return this.$callMethod('importFromCSV',a1,a2);} + async table(...args: string[]): Promise {return this.$callMethod('table',...args);} +} + export class EngNotation extends CppClass { constructor(prefix: string){ super(prefix); @@ -1281,6 +1291,7 @@ export class Lock extends Item { export class Minsky extends CppClass { canvas: Canvas; conversions: civita__Conversions; + databaseIngestor: DatabaseIngestor; dimensions: Map; equationDisplay: EquationDisplay; evalGodley: EvalGodley; @@ -1302,6 +1313,7 @@ export class Minsky extends CppClass { super(prefix); this.canvas=new Canvas(this.$prefix()+'.canvas'); this.conversions=new civita__Conversions(this.$prefix()+'.conversions'); + this.databaseIngestor=new DatabaseIngestor(this.$prefix()+'.databaseIngestor'); this.dimensions=new Map(this.$prefix()+'.dimensions',civita__Dimension); this.equationDisplay=new EquationDisplay(this.$prefix()+'.equationDisplay'); this.evalGodley=new EvalGodley(this.$prefix()+'.evalGodley'); diff --git a/model/databaseIngestor.cc b/model/databaseIngestor.cc index 81cb1e0f6..1e770c5f3 100644 --- a/model/databaseIngestor.cc +++ b/model/databaseIngestor.cc @@ -64,48 +64,51 @@ namespace minsky { ifstream input(filenames.front()); string line; - for (; getWholeLine(input,line,spec) && row DatabaseIngestor::load - (soci::statement& stmt, const std::vector& filenames, - const DataSpecSchema& spec) + template void DatabaseIngestor::load + (const std::vector& filenames, + const DataSpec& spec) { Tokeniser csvParser; vector cells; // prepare the insertion string based on spec - auto statement=(session.prepare<<"upsert into "+table+" ...",use(cells)); + soci::statement statement=(session.prepare<<"upsert into "+table+" ...",use(cells)); for (auto f: filenames) { ifstream input(f); size_t row=0; string line; - for (; getWholeLine(input,line,spec) && row tok(line.begin(),line.end(), csvParser); cells.assign(tok.begin(), tok.end()); - stmt.execute(true); + statement.execute(true); if (row%100==0) { session.commit(); session.begin(); } } - transaction.commit(); + tr.commit(); } } void DatabaseIngestor::importFromCSV - (const std::vector& filenames, const DataSpecSchema& spec) + (const std::vector& filenames, const DataSpec& spec) { if (!session.is_connected()) session.reconnect(); if (!session.is_connected()) return; @@ -115,9 +118,11 @@ namespace minsky if (spec.separator==' ') - load(statement,filenames,spec); + load(filenames,spec); else - load(statement,filenames,spec); + load(filenames,spec); } } + +CLASSDESC_ACCESS_EXPLICIT_INSTANTIATION(minsky::DatabaseIngestor); diff --git a/model/databaseIngestor.h b/model/databaseIngestor.h index 4473266f4..d54c99373 100644 --- a/model/databaseIngestor.h +++ b/model/databaseIngestor.h @@ -30,7 +30,7 @@ namespace minsky { class DatabaseIngestor { - soci::session session; + classdesc::Exclude session; template void load(const std::vector&, const DataSpec&); CLASSDESC_ACCESS(DatabaseIngestor); public: From 511e680a28034a623bd1c7afb608ce546bc21e73 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Mon, 10 Feb 2025 11:33:16 +1100 Subject: [PATCH 04/41] Add soci_core to libs in test directory. --- test/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index 2be928751..03830d2fc 100755 --- a/test/Makefile +++ b/test/Makefile @@ -17,7 +17,7 @@ FLAGS+=-std=c++20 -DJSON_PACK_NO_FALL_THROUGH_TO_STREAMING -DUSE_UNROLLED -DCLA -Wno-unknown-warning-option -Wno-unused-local-typedefs -Wno-unused-command-line-argument -I../model -I../engine -I../schema LIBS+=-L../RavelCAPI -lravelCAPI -L../RavelCAPI/civita -lcivita -lboost_system -lboost_thread \ -lboost_regex -lboost_date_time -lboost_filesystem -lclipboard -lxcb -lX11 \ - -lUnitTest++ -lgsl -lssl -lcrypto -lgslcblas -lxml2 -ltiff -ldl + -lUnitTest++ -lsoci_core -lgsl -lssl -lcrypto -lgslcblas -lxml2 -ltiff -ldl ifdef MXE else From 6212fb88d5063c426a62d1ef6209b757c2a9a7e8 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Mon, 10 Feb 2025 18:22:31 +1100 Subject: [PATCH 05/41] Regression tests now passing - except for known issues in DEBUG builds. --- engine/CSVParser.h | 2 +- model/databaseIngestor.cc | 25 +++++++++++++++---------- model/databaseIngestor.h | 13 ++++++------- model/minsky.cc | 1 + model/minsky.h | 2 +- test/testCSVParser.cc | 5 +---- 6 files changed, 25 insertions(+), 23 deletions(-) diff --git a/engine/CSVParser.h b/engine/CSVParser.h index 4fb5daf63..e0a08f5f3 100644 --- a/engine/CSVParser.h +++ b/engine/CSVParser.h @@ -192,7 +192,7 @@ namespace minsky explicit EscapedListSeparator(Char e = '\\') : escape_(1,e), c_(1,','), quote_(1,'\"'), last_(false) { } EscapedListSeparator(Char e, Char c,Char q = '\"') - : escape_(1,'\\'), c_(1,','), quote_(1,q), last_(false) { } + : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { } // EscapedListSeparator(typename EscapedListSeparator::string_type e, // typename EscapedListSeparator::string_type c, // EscapedListSeparator::string_type q) diff --git a/model/databaseIngestor.cc b/model/databaseIngestor.cc index 1e770c5f3..2b3e2850c 100644 --- a/model/databaseIngestor.cc +++ b/model/databaseIngestor.cc @@ -41,10 +41,14 @@ namespace minsky return {tok.begin(), tok.end()}; } + void DatabaseIngestor::connect(const string& dbType, const string& connection) + {session=make_shared(dbType,connection);} + void DatabaseIngestor::createTable (const std::vector& filenames, const DataSpec& spec) { - session<<"drop "+table+" if exists"; + if (!session) return; + *session<<"drop "+table+" if exists"; // for now, load time data as strings - TODO handle date-time parsing string def="create table "+table+" ("; for (size_t i=0; i void DatabaseIngestor::load - (const std::vector& filenames, - const DataSpec& spec) + (const std::vector& filenames, const DataSpec& spec) { + if (!session) return; Tokeniser csvParser; vector cells; // prepare the insertion string based on spec - soci::statement statement=(session.prepare<<"upsert into "+table+" ...",use(cells)); + soci::statement statement=(session->prepare<<"upsert into "+table+" ...",use(cells)); for (auto f: filenames) { ifstream input(f); size_t row=0; string line; for (; getWholeLine(input,line,spec) && row tok(line.begin(),line.end(), csvParser); @@ -99,8 +103,8 @@ namespace minsky statement.execute(true); if (row%100==0) { - session.commit(); - session.begin(); + session->commit(); + session->begin(); } } tr.commit(); @@ -110,8 +114,9 @@ namespace minsky void DatabaseIngestor::importFromCSV (const std::vector& filenames, const DataSpec& spec) { - if (!session.is_connected()) session.reconnect(); - if (!session.is_connected()) return; + if (!session) return; + if (!session->is_connected()) session->reconnect(); + if (!session->is_connected()) return; // TODO check if table exists, and call createTable if not // select * from table limit 1 - and check for exception thrown? diff --git a/model/databaseIngestor.h b/model/databaseIngestor.h index d54c99373..6e36e0f1b 100644 --- a/model/databaseIngestor.h +++ b/model/databaseIngestor.h @@ -30,20 +30,19 @@ namespace minsky { class DatabaseIngestor { - classdesc::Exclude session; + classdesc::Exclude> session; template void load(const std::vector&, const DataSpec&); CLASSDESC_ACCESS(DatabaseIngestor); public: - ///< connect to a database. Consult SOCI documentation for meaning of the two parameters - void connect(const std::string& dbType, const std::string& connection) - {session.open(dbType,connection);} std::string table; ///< table name to use - /// create an empty table satisfying \a filenames and \a spec + ///< connect to a database. Consult SOCI documentation for meaning of the two parameters + void connect(const std::string& dbType, const std::string& connection); + /// create an empty table satisfying \a filenames and \a spec. drops any preexisting table void createTable (const std::vector& filenames, const DataSpec& spec); /// import CSV files, using \a spec - void importFromCSV(const std::vector& filenames, - const DataSpec& spec); + void importFromCSV + (const std::vector& filenames, const DataSpec& spec); }; } diff --git a/model/minsky.cc b/model/minsky.cc index e45c06f3e..b800afbef 100644 --- a/model/minsky.cc +++ b/model/minsky.cc @@ -204,6 +204,7 @@ namespace minsky VariablePtr Minsky::definingVar(const string& valueId) const { + if (!model) return {}; return dynamic_pointer_cast (model->findAny(&Group::items, [&](const ItemPtr& x) { auto v=x->variableCast(); diff --git a/model/minsky.h b/model/minsky.h index 8bfa47ca7..71a256b97 100644 --- a/model/minsky.h +++ b/model/minsky.h @@ -155,7 +155,6 @@ namespace minsky FontDisplay fontSampler; PhillipsDiagram phillipsDiagram; std::vector publicationTabs; - DatabaseIngestor databaseIngestor; void addNewPublicationTab(const std::string& name) {publicationTabs.emplace_back(name);} void addCanvasItemToPublicationTab(size_t i) { @@ -256,6 +255,7 @@ namespace minsky GroupPtr model{new Group}; Canvas canvas{model}; + DatabaseIngestor databaseIngestor; void clearAllMaps(bool clearHistory); void clearAllMaps() {clearAllMaps(true);} diff --git a/test/testCSVParser.cc b/test/testCSVParser.cc index bca41fc3d..c495d7565 100644 --- a/test/testCSVParser.cc +++ b/test/testCSVParser.cc @@ -554,10 +554,7 @@ SUITE(CSVParser) spec.guessFromStream(f); } VariableValue newV(VariableType::flow); - { - ifstream f("tmp.csv"); - loadValueFromCSVFile(newV,f,spec); - } + loadValueFromCSVFile(newV,{"tmp.csv"},spec); CHECK(newV.hypercube().xvectors==v.hypercube().xvectors); CHECK_EQUAL(v.size(), newV.size()); From 3e98abd6a2e856fc08856d697ac662a1e0fc4222 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Wed, 12 Feb 2025 15:43:59 +1100 Subject: [PATCH 06/41] Loading citibike database example now works. --- RESTService/pyminsky.cc | 2 ++ loadDb.py | 22 +++++++++++++ model/databaseIngestor.cc | 68 +++++++++++++++++++++++++++------------ 3 files changed, 71 insertions(+), 21 deletions(-) create mode 100644 loadDb.py diff --git a/RESTService/pyminsky.cc b/RESTService/pyminsky.cc index e0ef790ff..da61071e1 100644 --- a/RESTService/pyminsky.cc +++ b/RESTService/pyminsky.cc @@ -97,6 +97,8 @@ namespace pyminsky } CLASSDESC_ADD_FUNCTION(findObject); CLASSDESC_ADD_FUNCTION(findVariable); + using minsky::DataSpec; + CLASSDESC_DECLARE_TYPE(DataSpec); } CLASSDESC_PYTHON_MODULE(pyminsky); diff --git a/loadDb.py b/loadDb.py new file mode 100644 index 000000000..9a26b3b73 --- /dev/null +++ b/loadDb.py @@ -0,0 +1,22 @@ +import sys +import json +from pyminsky import minsky, DataSpec +minsky.databaseIngestor.connect("sqlite3","db=citibike.sqlite") +# sys.argv[0] is this script name +filenames=sys.argv[1:] + +# set up spec for Citibike +spec=DataSpec() +spec.setDataArea(1,14) +spec.dataCols([0,11,14]) +spec.dimensionCols([1,4,8,12,13]) +spec.dimensionNames(["tripduration","starttime","stoptime","start station id","start station name","start station latitude","start station longitude","end station id","end station name","end station latitude","end station longitude","bikeid","usertype","birth year","gender"]) +spec.dimensions(15*[{"type":"string","units":""}]) +spec.duplicateKeyAction("av") +spec.numCols(15) + +#'{"counter":false,"dataColOffset":14,"dataCols":[0,11],"dataRowOffset":1,"decSeparator":".","dimensionCols":[1,4,8,12,13,14],"dimensionNames":["tripduration","starttime","stoptime","start station id","start station name","start station latitude","start station longitude","end station id","end station name","end station latitude","end station longitude","bikeid","usertype","birth year","gender"],"dimensions":[{"type":"value","units":""},{"type":"time","units":""},{"type":"time","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""}],"dontFail":false,"duplicateKeyAction":"av","escape":"\0","headerRow":0,"horizontalDimName":"?","horizontalDimension":{"type":"string","units":""},"maxColumn":1000,"mergeDelimiters":false,"missingValue":NaN,"numCols":15,"quote":"\"","separator":","})' + +minsky.databaseIngestor.table("citibike") +minsky.databaseIngestor.createTable(filenames,spec()) +minsky.databaseIngestor.importFromCSV(filenames,spec()) diff --git a/model/databaseIngestor.cc b/model/databaseIngestor.cc index 2b3e2850c..796a7befd 100644 --- a/model/databaseIngestor.cc +++ b/model/databaseIngestor.cc @@ -47,23 +47,25 @@ namespace minsky void DatabaseIngestor::createTable (const std::vector& filenames, const DataSpec& spec) { - if (!session) return; - *session<<"drop "+table+" if exists"; + if (!session.get()) return; + *session<<"drop table if exists "+table; // for now, load time data as strings - TODO handle date-time parsing string def="create table "+table+" ("; - for (size_t i=0; i void DatabaseIngestor::load (const std::vector& filenames, const DataSpec& spec) { - if (!session) return; Tokeniser csvParser; - vector cells; + set insertCols; + // compute complement of dimensionCols union dataCols + for (unsigned i=0; i cells(insertCols.size()); + // prepare the insertion string based on spec - soci::statement statement=(session->prepare<<"upsert into "+table+" ...",use(cells)); + string insertStatement="insert into "+table+"("; + for (size_t i=0; iprepare< tok(line.begin(),line.end(), csvParser); - cells.assign(tok.begin(), tok.end()); + unsigned col=0, cell=0; + for (auto t: tok) + if (insertCols.contains(col++)) + cells[cell++]=t; statement.execute(true); - if (row%100==0) + if (row%1000==0) { session->commit(); session->begin(); + cout<& filenames, const DataSpec& spec) { - if (!session) return; + if (!session.get()) return; if (!session->is_connected()) session->reconnect(); if (!session->is_connected()) return; From d2bf6316584a1eba20a3e0242167eb98ba4de129 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Thu, 13 Feb 2025 14:36:29 +1100 Subject: [PATCH 07/41] Handle time axis processing. --- loadDb.py | 3 +++ model/databaseIngestor.cc | 32 ++++++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/loadDb.py b/loadDb.py index 9a26b3b73..5975f3a51 100644 --- a/loadDb.py +++ b/loadDb.py @@ -12,6 +12,9 @@ spec.dimensionCols([1,4,8,12,13]) spec.dimensionNames(["tripduration","starttime","stoptime","start station id","start station name","start station latitude","start station longitude","end station id","end station name","end station latitude","end station longitude","bikeid","usertype","birth year","gender"]) spec.dimensions(15*[{"type":"string","units":""}]) +spec.dimensions[1]({"type":"time","units":""}) +spec.dimensions[11]({"type":"value","units":""}) +spec.dimensions[14]({"type":"value","units":""}) spec.duplicateKeyAction("av") spec.numCols(15) diff --git a/model/databaseIngestor.cc b/model/databaseIngestor.cc index 796a7befd..53eccc94d 100644 --- a/model/databaseIngestor.cc +++ b/model/databaseIngestor.cc @@ -57,12 +57,18 @@ namespace minsky { def+=(i? ", '":"'")+spec.dimensionNames[i]+"'"; if (spec.dimensionCols.contains(i)) - { - if (spec.dimensions[i].type==Dimension::value) + switch (spec.dimensions[i].type) + { + case Dimension::value: def+=" double"; - else - def+=" char(255)"; // TODO - better string type? - } + break; + case Dimension::string: + def+=" varchar(255)"; + break; + case Dimension::time: + def+=" timestamp with timezone"; + break; + } else def+=" double"; } @@ -122,9 +128,19 @@ namespace minsky bytesRead+=line.size(); const boost::tokenizer tok(line.begin(),line.end(), csvParser); unsigned col=0, cell=0; - for (auto t: tok) - if (insertCols.contains(col++)) - cells[cell++]=t; + for (auto& t: tok) + { + if (insertCols.contains(col)) + { + if (spec.dimensions[col].type==Dimension::time) + // reformat input as ISO standard + cells[cell++]=str(anyVal(spec.dimensions[col],t)); + else + cells[cell++]=t; + } + ++col; + } + statement.execute(true); if (row%1000==0) { From fef673b7b0f9878ba34a8fde4a88cf60f2236ec7 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Fri, 14 Feb 2025 12:28:21 +1100 Subject: [PATCH 08/41] Sample code for determining column types in database. --- loadDb.py | 5 +++-- model/databaseIngestor.cc | 28 +++++++++++++++++++++++++--- model/databaseIngestor.h | 1 + 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/loadDb.py b/loadDb.py index 5975f3a51..f9672129d 100644 --- a/loadDb.py +++ b/loadDb.py @@ -1,7 +1,8 @@ import sys import json from pyminsky import minsky, DataSpec -minsky.databaseIngestor.connect("sqlite3","db=citibike.sqlite") +#minsky.databaseIngestor.connect("sqlite3","db=citibike.sqlite") +minsky.databaseIngestor.connect("sqlite3","db=foo.sqlite") # sys.argv[0] is this script name filenames=sys.argv[1:] @@ -22,4 +23,4 @@ minsky.databaseIngestor.table("citibike") minsky.databaseIngestor.createTable(filenames,spec()) -minsky.databaseIngestor.importFromCSV(filenames,spec()) +#minsky.databaseIngestor.importFromCSV(filenames,spec()) diff --git a/model/databaseIngestor.cc b/model/databaseIngestor.cc index 53eccc94d..46a20854b 100644 --- a/model/databaseIngestor.cc +++ b/model/databaseIngestor.cc @@ -42,7 +42,10 @@ namespace minsky } void DatabaseIngestor::connect(const string& dbType, const string& connection) - {session=make_shared(dbType,connection);} + { + session=make_shared(dbType,connection); + this->dbType=dbType; + } void DatabaseIngestor::createTable (const std::vector& filenames, const DataSpec& spec) @@ -66,7 +69,10 @@ namespace minsky def+=" varchar(255)"; break; case Dimension::time: - def+=" timestamp with timezone"; + if (dbType=="sqlite3") // sqlite backend returns an integer type, not time type. + def+=" datetime"; + else + def+=" "+session->get_backend()->create_column_type(soci::dt_date,0,0); break; } else @@ -86,6 +92,23 @@ namespace minsky } def+=")"; *session<prepare<<"select * from "+table+" limit 1",soci::into(result))); +// s.execute(false); +// soci::data_type type; +// std::string name; +// auto backEnd=s.get_backend(); +// auto numCols=backEnd->prepare_for_describe(); +// for (int i=0; idescribe_column(i,type,name); +// std::cout<prepare<> session; + std::string dbType; //stash the database type here template void load(const std::vector&, const DataSpec&); CLASSDESC_ACCESS(DatabaseIngestor); public: From 52f6ecc19b252b7e159826e3b07cdc276be05702 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Wed, 16 Apr 2025 09:52:04 +1000 Subject: [PATCH 09/41] CSVParser stuff moved into RavelCAPI DatabaseIngestor stuff moved into Ravel. --- RESTService/typescriptAPI.cc | 1 + RavelCAPI | 2 +- ecolab | 2 +- engine/CSVParser.cc | 49 ++--------- engine/CSVParser.h | 152 ----------------------------------- engine/saver.cc | 1 + model/databaseIngestor.cc | 152 ----------------------------------- model/databaseIngestor.h | 9 +-- model/minsky.cc | 1 + schema/dataSpecSchema.h | 5 +- schema/schema3.cc | 1 + test/testSaver.cc | 1 + 12 files changed, 17 insertions(+), 359 deletions(-) diff --git a/RESTService/typescriptAPI.cc b/RESTService/typescriptAPI.cc index f2ffcd275..394a7e646 100644 --- a/RESTService/typescriptAPI.cc +++ b/RESTService/typescriptAPI.cc @@ -11,6 +11,7 @@ #include "canvas.tcd" #include "CSVDialog.tcd" #include "CSVParser.tcd" +#include "CSVTools.tcd" #include "constMap.tcd" #include "databaseIngestor.tcd" #include "dataSpecSchema.tcd" diff --git a/RavelCAPI b/RavelCAPI index c81ede8a3..df03eed0e 160000 --- a/RavelCAPI +++ b/RavelCAPI @@ -1 +1 @@ -Subproject commit c81ede8a38ef12babba7b14342cc10b29eb0f271 +Subproject commit df03eed0e955fbe37ac2bce6cf399e525cfa6cfd diff --git a/ecolab b/ecolab index 35989cd51..16ad37dc6 160000 --- a/ecolab +++ b/ecolab @@ -1 +1 @@ -Subproject commit 35989cd51365470d38b4a6931a853a5e37846566 +Subproject commit 16ad37dc6add4d96cff9408c48ac484259789995 diff --git a/engine/CSVParser.cc b/engine/CSVParser.cc index 439ce91fe..39e9214b5 100644 --- a/engine/CSVParser.cc +++ b/engine/CSVParser.cc @@ -20,6 +20,7 @@ #include "minsky.h" #include "CSVParser.h" +#include "CSVTools.rcd" #include "CSVParser.rcd" #include "dataSpecSchema.rcd" #include "dimension.rcd" @@ -36,13 +37,16 @@ using namespace minsky; using namespace std; +using ravel::Parser; +using ravel::SpaceSeparatorParser; +using ravel::getWholeLine; #include #include #include #include -typedef boost::tokenizer Tokenizer; +//typedef boost::tokenizer Tokenizer; namespace { @@ -431,49 +435,6 @@ void DataSpec::populateFromRavelMetadata(const std::string& metadata, const stri namespace minsky { - // handle DOS files with '\r' '\n' line terminators - void chomp(string& buf) - { - if (!buf.empty() && buf.back()=='\r') - buf.erase(buf.size()-1); - } - - // gets a line, accounting for quoted newlines - bool getWholeLine(istream& input, string& line, const DataSpecSchema& spec) - { - line.clear(); - bool r=getline(input,line).good(); - chomp(line); - while (r) - { - int quoteCount=0; - for (auto i: line) - if (i==spec.quote) - ++quoteCount; - if (quoteCount%2==0) break; // data line correctly terminated - string buf; - r=getline(input,buf).good(); // read next line and append - chomp(buf); - line+=buf; - } - escapeDoubledQuotes(line,spec); - return r || !line.empty(); - } - - void escapeDoubledQuotes(std::string& line,const DataSpecSchema& spec) - { - // replace doubled quotes with escape quote - for (size_t i=1; i1 && - ((line[i-2]!=spec.quote && line[i-2]!=spec.escape && - (line[i-2]!=spec.separator || i==line.size()-1|| line[i+1]!=spec.quote)) // deal with ,'' - || // deal with "" middle or end - (line[i-2]==spec.quote && (i==2 || line[i-3]==spec.separator || line[i-3]==spec.escape)))))) // deal with leading """ - line[i-1]=spec.escape; - } - /// handle reporting errors in loadValueFromCSVFileT when loading files struct OnError { diff --git a/engine/CSVParser.h b/engine/CSVParser.h index e0a08f5f3..7a6789172 100644 --- a/engine/CSVParser.h +++ b/engine/CSVParser.h @@ -123,158 +123,6 @@ namespace minsky /// load a variableValue from a stream according to data spec void loadValueFromCSVFile(VariableValue&, std::istream& input, const DataSpec&); - /// replace doubled quotes with escaped quotes - void escapeDoubledQuotes(std::string&,const DataSpecSchema&); - - /// get complete line from input, allowing for quoted linefeed - bool getWholeLine(std::istream& input, std::string& line, const DataSpecSchema& spec); - - namespace escapedListSeparator -{ - // pinched from boost::escape_list_separator, and modified to not throw - template ::traits_type > - class EscapedListSeparator { - - private: - typedef std::basic_string string_type; - struct char_eq { - Char e_; - char_eq(Char e):e_(e) { } - bool operator()(Char c) { - return Traits::eq(e_,c); - } - }; - string_type escape_; - string_type c_; - string_type quote_; - bool last_; - - bool is_escape(Char e) { - const char_eq f(e); - return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end(); - } - bool is_c(Char e) { - const char_eq f(e); - return std::find_if(c_.begin(),c_.end(),f)!=c_.end(); - } - bool is_quote(Char e) { - const char_eq f(e); - return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end(); - } - template - void do_escape(iterator& next,iterator end,Token& tok) { - if (++next >= end) - // don't throw, but pass on verbatim - tok+=escape_.front(); - if (Traits::eq(*next,'n')) { - tok+='\n'; - return; - } - if (is_quote(*next)) { - tok+=*next; - return; - } - if (is_c(*next)) { - tok+=*next; - return; - } - if (is_escape(*next)) { - tok+=*next; - return; - } - // don't throw, but pass on verbatim - tok+=escape_.front()+*next; - } - - public: - - explicit EscapedListSeparator(Char e = '\\') - : escape_(1,e), c_(1,','), quote_(1,'\"'), last_(false) { } - EscapedListSeparator(Char e, Char c,Char q = '\"') - : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { } -// EscapedListSeparator(typename EscapedListSeparator::string_type e, -// typename EscapedListSeparator::string_type c, -// EscapedListSeparator::string_type q) -// : escape_(e), c_(c), quote_(q), last_(false) { } - - void reset() {last_=false;} - - template - bool operator()(InputIterator& next,InputIterator end,Token& tok) { - bool bInQuote = false; - tok = Token(); - - if (next >= end) { - next=end; // reset next in case it has adavanced beyond - if (last_) { - last_ = false; - return true; - } - return false; - } - last_ = false; - while (next < end) { - if (is_escape(*next)) { - do_escape(next,end,tok); - } - else if (is_c(*next)) { - if (!bInQuote) { - // If we are not in quote, then we are done - ++next; - // The last character was a c, that means there is - // 1 more blank field - last_ = true; - return true; - } - tok+=*next; - } - else if (is_quote(*next)) { - bInQuote=!bInQuote; - } - else { - tok += *next; - } - ++next; - } - return true; - } - }; -} -using Parser=escapedListSeparator::EscapedListSeparator; - -struct SpaceSeparatorParser -{ - char escape, quote; - SpaceSeparatorParser(char escape='\\', char sep=' ', char quote='"'): - escape(escape), quote(quote) {} - template - bool operator()(I& next, I end, std::string& tok) - { - tok.clear(); - bool quoted=false; - while (next!=end) - { - if (*next==escape) - tok+=*(++next); - else if (*next==quote) - quoted=!quoted; - else if (!quoted && isspace(*next)) - { - while (isspace(*next)) ++next; - return true; - } - else - tok+=*next; - ++next; - } - return !tok.empty(); - } - void reset() {} -}; - - - } #include "CSVParser.cd" diff --git a/engine/saver.cc b/engine/saver.cc index 7c926fa7c..8e15c0886 100644 --- a/engine/saver.cc +++ b/engine/saver.cc @@ -19,6 +19,7 @@ #include "saver.h" #include "schema3.h" +#include "CSVTools.xcd" #include "minsky_epilogue.h" namespace minsky diff --git a/model/databaseIngestor.cc b/model/databaseIngestor.cc index 46a20854b..8d8cd3443 100644 --- a/model/databaseIngestor.cc +++ b/model/databaseIngestor.cc @@ -21,175 +21,23 @@ #include "databaseIngestor.rcd" #include "minsky_epilogue.h" -using civita::Dimension; -using soci::use; -using soci::transaction; using namespace std; namespace minsky { - vector parseRow(const string& line, char separator) - { - if (separator==' ') - { - SpaceSeparatorParser csvParser; - const boost::tokenizer tok(line.begin(),line.end(), csvParser); - return {tok.begin(), tok.end()}; - } - Parser csvParser; - const boost::tokenizer tok(line.begin(),line.end(), csvParser); - return {tok.begin(), tok.end()}; - } - void DatabaseIngestor::connect(const string& dbType, const string& connection) { - session=make_shared(dbType,connection); - this->dbType=dbType; } void DatabaseIngestor::createTable (const std::vector& filenames, const DataSpec& spec) { - if (!session.get()) return; - *session<<"drop table if exists "+table; - // for now, load time data as strings - TODO handle date-time parsing - string def="create table "+table+" ("; - for (size_t i=0; iget_backend()->create_column_type(soci::dt_date,0,0); - break; - } - else - def+=" double"; - } - if (!filenames.empty()) - { - ifstream input(filenames.front()); - string line; - for (size_t row=0; getWholeLine(input,line,spec) && rowprepare<<"select * from "+table+" limit 1",soci::into(result))); -// s.execute(false); -// soci::data_type type; -// std::string name; -// auto backEnd=s.get_backend(); -// auto numCols=backEnd->prepare_for_describe(); -// for (int i=0; idescribe_column(i,type,name); -// std::cout< void DatabaseIngestor::load - (const std::vector& filenames, const DataSpec& spec) - { - Tokeniser csvParser; - set insertCols; - // compute complement of dimensionCols union dataCols - for (unsigned i=0; i cells(insertCols.size()); - - // prepare the insertion string based on spec - string insertStatement="insert into "+table+"("; - for (size_t i=0; iprepare< tok(line.begin(),line.end(), csvParser); - unsigned col=0, cell=0; - for (auto& t: tok) - { - if (insertCols.contains(col)) - { - if (spec.dimensions[col].type==Dimension::time) - // reformat input as ISO standard - cells[cell++]=str(anyVal(spec.dimensions[col],t)); - else - cells[cell++]=t; - } - ++col; - } - - statement.execute(true); - if (row%1000==0) - { - session->commit(); - session->begin(); - cout<& filenames, const DataSpec& spec) { - if (!session.get()) return; - if (!session->is_connected()) session->reconnect(); - if (!session->is_connected()) return; - - // TODO check if table exists, and call createTable if not - // select * from table limit 1 - and check for exception thrown? - - - if (spec.separator==' ') - load(filenames,spec); - else - load(filenames,spec); } } diff --git a/model/databaseIngestor.h b/model/databaseIngestor.h index 68b74d3fd..9859a30ea 100644 --- a/model/databaseIngestor.h +++ b/model/databaseIngestor.h @@ -17,12 +17,12 @@ along with Minsky. If not, see . */ -#ifndef DATABSE_INGESTOR_H -#define DATABSE_INGESTOR_H +#ifndef DATABASE_INGESTOR_H +#define DATABASE_INGESTOR_H #include "CSVParser.h" +#include -#include #include #include @@ -30,9 +30,6 @@ namespace minsky { class DatabaseIngestor { - classdesc::Exclude> session; - std::string dbType; //stash the database type here - template void load(const std::vector&, const DataSpec&); CLASSDESC_ACCESS(DatabaseIngestor); public: std::string table; ///< table name to use diff --git a/model/minsky.cc b/model/minsky.cc index b800afbef..d1c9632e4 100644 --- a/model/minsky.cc +++ b/model/minsky.cc @@ -35,6 +35,7 @@ #include "minskyVersion.h" +#include "CSVTools.xcd" #include "fontDisplay.rcd" #include "minsky.rcd" #include "minsky.xcd" diff --git a/schema/dataSpecSchema.h b/schema/dataSpecSchema.h index 07cad917d..80089dbb3 100644 --- a/schema/dataSpecSchema.h +++ b/schema/dataSpecSchema.h @@ -19,20 +19,19 @@ #ifndef DATASPECSCHEMA_H #define DATASPECSCHEMA_H +#include "CSVTools.h" #include #include #include namespace minsky { - struct DataSpecSchema + struct DataSpecSchema: public ravel::CSVSpec { // these fields are only used for persistence. Need to be handled specially within schema code std::size_t dataRowOffset, dataColOffset; std::size_t numCols=0; ///< number of columns in CSV. Must be > dataColOffset - // NB escape character might be backslash ('\\'), but not usually used in CSV files, so set to nul. - char separator=',', quote='"', escape='\0', decSeparator='.'; bool mergeDelimiters=false; bool counter=false; ///< count data items, not read their values bool dontFail=false; ///< do not throw an error on corrupt data diff --git a/schema/schema3.cc b/schema/schema3.cc index 824b4342b..52bb0fc13 100644 --- a/schema/schema3.cc +++ b/schema/schema3.cc @@ -18,6 +18,7 @@ */ #include "dataOp.h" #include "schema3.h" +#include "CSVTools.xcd" #include "sheet.h" #include "userFunction.h" #include "minsky_epilogue.h" diff --git a/test/testSaver.cc b/test/testSaver.cc index 01675d913..a01869e5c 100644 --- a/test/testSaver.cc +++ b/test/testSaver.cc @@ -19,6 +19,7 @@ #include "saver.h" #include "schema3.h" +#include "CSVTools.xcd" #include "minsky_epilogue.h" #include #include From 665066bf2cae57fe34b6f9a5868c98df3606d667 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Wed, 16 Apr 2025 11:29:50 +1000 Subject: [PATCH 10/41] Trim test/Makefile. --- test/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index 03830d2fc..98f2b73db 100755 --- a/test/Makefile +++ b/test/Makefile @@ -11,7 +11,7 @@ VPATH= .. ../schema ../model ../engine ../RESTService ../RavelCAPI/civita ../Rav UNITTESTOBJS=main.o testCSVParser.o testCanvas.o testDerivative.o testExpressionWalker.o testGrid.o testLatexToPango.o testLockGroup.o testMdl.o testMinsky.o testModel.o testPannableTab.o testPhillips.o testPlotWidget.o testPubTab.o testSaver.o testStr.o testTensorOps.o testUnits.o testUserFunction.o testVariable.o testVariablePane.o testXVector.o ticket-1461.o -MINSKYOBJS=localMinsky.o ../libminsky.a +MINSKYOBJS=localMinsky.o ../libminsky.a FLAGS:=-I.. -I../RESTService -I../RavelCAPI/civita -I../RavelCAPI $(FLAGS) FLAGS+=-std=c++20 -DJSON_PACK_NO_FALL_THROUGH_TO_STREAMING -DUSE_UNROLLED -DCLASSDESC_ARITIES=0xf \ -Wno-unknown-warning-option -Wno-unused-local-typedefs -Wno-unused-command-line-argument -I../model -I../engine -I../schema From 6809ca7d10f2f84c8c3fb5e3dbd8c59cbbb94ce3 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Wed, 16 Apr 2025 14:57:51 +1000 Subject: [PATCH 11/41] Strip out databaseIngestor, and use functions exposed through RavelCAPI. --- Makefile | 2 +- RESTService/typescriptAPI.cc | 3 ++- RavelCAPI | 2 +- model/databaseIngestor.cc | 45 --------------------------------- model/databaseIngestor.h | 49 ------------------------------------ model/minsky.cc | 3 +++ model/minsky.h | 4 +-- test/Makefile | 2 +- 8 files changed, 10 insertions(+), 100 deletions(-) delete mode 100644 model/databaseIngestor.cc delete mode 100644 model/databaseIngestor.h diff --git a/Makefile b/Makefile index 221961c5f..933feee37 100644 --- a/Makefile +++ b/Makefile @@ -144,7 +144,7 @@ PREFIX=/usr/local # custom one that picks up its scripts from a relative library # directory MODLINK=$(LIBMODS:%=$(ECOLAB_HOME)/lib/%) -MODEL_OBJS=autoLayout.o cairoItems.o canvas.o CSVDialog.o databaseIngestor.o dataOp.o equationDisplay.o godleyIcon.o godleyTable.o godleyTableWindow.o grid.o group.o item.o intOp.o lasso.o lock.o minsky.o operation.o operationRS.o operationRS1.o operationRS2.o phillipsDiagram.o plotWidget.o port.o pubTab.o ravelWrap.o renderNativeWindow.o selection.o sheet.o SVGItem.o switchIcon.o userFunction.o userFunction_units.o variableInstanceList.o variable.o variablePane.o windowInformation.o wire.o +MODEL_OBJS=autoLayout.o cairoItems.o canvas.o CSVDialog.o dataOp.o equationDisplay.o godleyIcon.o godleyTable.o godleyTableWindow.o grid.o group.o item.o intOp.o lasso.o lock.o minsky.o operation.o operationRS.o operationRS1.o operationRS2.o phillipsDiagram.o plotWidget.o port.o pubTab.o ravelWrap.o renderNativeWindow.o selection.o sheet.o SVGItem.o switchIcon.o userFunction.o userFunction_units.o variableInstanceList.o variable.o variablePane.o windowInformation.o wire.o ENGINE_OBJS=coverage.o clipboard.o derivative.o equationDisplayRender.o equations.o evalGodley.o evalOp.o flowCoef.o \ godleyExport.o latexMarkup.o valueId.o variableValue.o node_latex.o node_matlab.o CSVParser.o \ minskyTensorOps.o mdlReader.o saver.o rungeKutta.o diff --git a/RESTService/typescriptAPI.cc b/RESTService/typescriptAPI.cc index 881f1c780..f706e0680 100644 --- a/RESTService/typescriptAPI.cc +++ b/RESTService/typescriptAPI.cc @@ -13,11 +13,11 @@ #include "CSVParser.tcd" #include "CSVTools.tcd" #include "constMap.tcd" -#include "databaseIngestor.tcd" #include "dataSpecSchema.tcd" #include "dataOp.h" #include "dataOp.tcd" #include "dimension.tcd" +#include "dynamicRavelCAPI.tcd" #include "engNotation.tcd" #include "equationDisplay.tcd" #include "evalGodley.tcd" @@ -260,6 +260,7 @@ int main() api.addClass(); api.addClass(); api.addClass(); + // api.addClass(); api.addClass(); api.addClass(); api.addClass(); diff --git a/RavelCAPI b/RavelCAPI index df03eed0e..dd2e99860 160000 --- a/RavelCAPI +++ b/RavelCAPI @@ -1 +1 @@ -Subproject commit df03eed0e955fbe37ac2bce6cf399e525cfa6cfd +Subproject commit dd2e9986029b62c6f075c3027680b6c2a0904b82 diff --git a/model/databaseIngestor.cc b/model/databaseIngestor.cc deleted file mode 100644 index 8d8cd3443..000000000 --- a/model/databaseIngestor.cc +++ /dev/null @@ -1,45 +0,0 @@ -/* - @copyright Steve Keen 2025 - @author Russell Standish - This file is part of Minsky. - - Minsky is free software: you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Minsky is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Minsky. If not, see . -*/ - -#include "databaseIngestor.h" -#include "databaseIngestor.rcd" -#include "minsky_epilogue.h" - -using namespace std; - -namespace minsky -{ - void DatabaseIngestor::connect(const string& dbType, const string& connection) - { - } - - void DatabaseIngestor::createTable - (const std::vector& filenames, const DataSpec& spec) - { - } - - - void DatabaseIngestor::importFromCSV - (const std::vector& filenames, const DataSpec& spec) - { - } - -} - -CLASSDESC_ACCESS_EXPLICIT_INSTANTIATION(minsky::DatabaseIngestor); diff --git a/model/databaseIngestor.h b/model/databaseIngestor.h deleted file mode 100644 index 9859a30ea..000000000 --- a/model/databaseIngestor.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - @copyright Steve Keen 2025 - @author Russell Standish - This file is part of Minsky. - - Minsky is free software: you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Minsky is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Minsky. If not, see . -*/ - -#ifndef DATABASE_INGESTOR_H -#define DATABASE_INGESTOR_H - -#include "CSVParser.h" -#include - -#include -#include - -namespace minsky -{ - class DatabaseIngestor - { - CLASSDESC_ACCESS(DatabaseIngestor); - public: - std::string table; ///< table name to use - ///< connect to a database. Consult SOCI documentation for meaning of the two parameters - void connect(const std::string& dbType, const std::string& connection); - /// create an empty table satisfying \a filenames and \a spec. drops any preexisting table - void createTable - (const std::vector& filenames, const DataSpec& spec); - /// import CSV files, using \a spec - void importFromCSV - (const std::vector& filenames, const DataSpec& spec); - }; -} - -#include "databaseIngestor.cd" -#include "databaseIngestor.xcd" -#endif diff --git a/model/minsky.cc b/model/minsky.cc index b1794c9a2..299a1050c 100644 --- a/model/minsky.cc +++ b/model/minsky.cc @@ -36,6 +36,8 @@ #include "minskyVersion.h" #include "CSVTools.xcd" +#include "dynamicRavelCAPI.rcd" +#include "dynamicRavelCAPI.xcd" #include "fontDisplay.rcd" #include "minsky.rcd" #include "minsky.xcd" @@ -1818,3 +1820,4 @@ CLASSDESC_ACCESS_EXPLICIT_INSTANTIATION(minsky::Minsky); CLASSDESC_ACCESS_EXPLICIT_INSTANTIATION(classdesc::Signature); CLASSDESC_ACCESS_EXPLICIT_INSTANTIATION(classdesc::PolyRESTProcessBase); CLASSDESC_ACCESS_EXPLICIT_INSTANTIATION(minsky::CallableFunction); +CLASSDESC_ACCESS_EXPLICIT_INSTANTIATION(ravel::Database); diff --git a/model/minsky.h b/model/minsky.h index dc8044e75..1f46c2994 100644 --- a/model/minsky.h +++ b/model/minsky.h @@ -26,8 +26,8 @@ #include "cairoItems.h" #include "canvas.h" #include "clipboard.h" -#include "databaseIngestor.h" #include "dimension.h" +#include "dynamicRavelCAPI.h" #include "evalOp.h" #include "equationDisplay.h" #include "equations.h" @@ -255,7 +255,7 @@ namespace minsky GroupPtr model{new Group}; Canvas canvas{model}; - DatabaseIngestor databaseIngestor; + ravel::Database databaseIngestor; void clearAllMaps(bool clearHistory); void clearAllMaps() {clearAllMaps(true);} diff --git a/test/Makefile b/test/Makefile index 98f2b73db..03830d2fc 100755 --- a/test/Makefile +++ b/test/Makefile @@ -11,7 +11,7 @@ VPATH= .. ../schema ../model ../engine ../RESTService ../RavelCAPI/civita ../Rav UNITTESTOBJS=main.o testCSVParser.o testCanvas.o testDerivative.o testExpressionWalker.o testGrid.o testLatexToPango.o testLockGroup.o testMdl.o testMinsky.o testModel.o testPannableTab.o testPhillips.o testPlotWidget.o testPubTab.o testSaver.o testStr.o testTensorOps.o testUnits.o testUserFunction.o testVariable.o testVariablePane.o testXVector.o ticket-1461.o -MINSKYOBJS=localMinsky.o ../libminsky.a +MINSKYOBJS=localMinsky.o ../libminsky.a FLAGS:=-I.. -I../RESTService -I../RavelCAPI/civita -I../RavelCAPI $(FLAGS) FLAGS+=-std=c++20 -DJSON_PACK_NO_FALL_THROUGH_TO_STREAMING -DUSE_UNROLLED -DCLASSDESC_ARITIES=0xf \ -Wno-unknown-warning-option -Wno-unused-local-typedefs -Wno-unused-command-line-argument -I../model -I../engine -I../schema From ceae25428dfeb0cdfdd5670bd6492535fc348067 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Thu, 17 Apr 2025 19:28:07 +1000 Subject: [PATCH 12/41] loadDb script now working via Ravel plugin. --- RESTService/typescriptAPI.cc | 3 +- RavelCAPI | 2 +- engine/CSVParser.cc | 1 + engine/CSVParser.h | 15 ++++++++ gui-js/libs/shared/src/lib/backend/minsky.ts | 39 ++++++++++++++++++-- loadDb.py | 10 +++-- model/minsky.h | 19 +++++++++- schema/dataSpecSchema.h | 1 + 8 files changed, 79 insertions(+), 11 deletions(-) diff --git a/RESTService/typescriptAPI.cc b/RESTService/typescriptAPI.cc index f706e0680..b1f23b51e 100644 --- a/RESTService/typescriptAPI.cc +++ b/RESTService/typescriptAPI.cc @@ -248,6 +248,7 @@ int main() api.addClass(); api.addClass(); api.addClass(); + api.addClass(); api.addClass(); api.addClass(); api.addClass(); @@ -260,7 +261,7 @@ int main() api.addClass(); api.addClass(); api.addClass(); - // api.addClass(); + api.addClass(); api.addClass(); api.addClass(); api.addClass(); diff --git a/RavelCAPI b/RavelCAPI index dd2e99860..37943d775 160000 --- a/RavelCAPI +++ b/RavelCAPI @@ -1 +1 @@ -Subproject commit dd2e9986029b62c6f075c3027680b6c2a0904b82 +Subproject commit 37943d7758189f68cfd07aecf9889624b5a870a5 diff --git a/engine/CSVParser.cc b/engine/CSVParser.cc index 374ad14bc..4800faad2 100644 --- a/engine/CSVParser.cc +++ b/engine/CSVParser.cc @@ -245,6 +245,7 @@ void DataSpec::setDataArea(size_t row, size_t col) dataCols.erase(i); for (unsigned i=m_nColAxes; i(r)=*this; + r.dataRowOffset=dataRowOffset; // TODO: non-normalisation between dataRowOffset and m_nRowAxes causes problems... + r.headerRow=headerRow; + r.mergeDelimiters=mergeDelimiters; + r.counter=counter; + r.dontFail=dontFail; + r.dimensionCols=dimensionCols; + r.dataCols=dataCols; + for (size_t i=0; i {return this.$callMethod('connect',a1,a2);} - async createTable(a1: string[],a2: DataSpec): Promise {return this.$callMethod('createTable',a1,a2);} - async importFromCSV(a1: string[],a2: DataSpec): Promise {return this.$callMethod('importFromCSV',a1,a2);} - async table(...args: string[]): Promise {return this.$callMethod('table',...args);} + async close(): Promise {return this.$callMethod('close');} + async connect(a1: string,a2: string,a3: string): Promise {return this.$callMethod('connect',a1,a2,a3);} + async createTable(a1: string,a2: DataSpec): Promise {return this.$callMethod('createTable',a1,a2);} + async deduplicate(a1: string,a2: DataSpec): Promise {return this.$callMethod('deduplicate',a1,a2);} + async loadDatabase(a1: string[],a2: DataSpec): Promise {return this.$callMethod('loadDatabase',a1,a2);} } export class EngNotation extends CppClass { @@ -2441,6 +2442,15 @@ export class civita__Index extends CppClass { async size(): Promise {return this.$callMethod('size');} } +export class civita__NamedDimension extends CppClass { + dimension: civita__Dimension; + constructor(prefix: string){ + super(prefix); + this.dimension=new civita__Dimension(this.$prefix()+'.dimension'); + } + async name(...args: string[]): Promise {return this.$callMethod('name',...args);} +} + export class civita__TensorVal extends CppClass { constructor(prefix: string){ super(prefix); @@ -2513,6 +2523,27 @@ export class minsky__Canvas__ZoomCrop extends CppClass { async zoom(...args: number[]): Promise {return this.$callMethod('zoom',...args);} } +export class ravel__DataSpec extends CppClass { + dataCols: Container; + dimensionCols: Container; + dimensions: Sequence; + constructor(prefix: string){ + super(prefix); + this.dataCols=new Container(this.$prefix()+'.dataCols'); + this.dimensionCols=new Container(this.$prefix()+'.dimensionCols'); + this.dimensions=new Sequence(this.$prefix()+'.dimensions',civita__NamedDimension); + } + async counter(...args: boolean[]): Promise {return this.$callMethod('counter',...args);} + async dataRowOffset(...args: number[]): Promise {return this.$callMethod('dataRowOffset',...args);} + async decSeparator(...args: number[]): Promise {return this.$callMethod('decSeparator',...args);} + async dontFail(...args: boolean[]): Promise {return this.$callMethod('dontFail',...args);} + async escape(...args: number[]): Promise {return this.$callMethod('escape',...args);} + async headerRow(...args: number[]): Promise {return this.$callMethod('headerRow',...args);} + async mergeDelimiters(...args: boolean[]): Promise {return this.$callMethod('mergeDelimiters',...args);} + async quote(...args: number[]): Promise {return this.$callMethod('quote',...args);} + async separator(...args: number[]): Promise {return this.$callMethod('separator',...args);} +} + export class ravel__HandleState extends CppClass { customOrder: Sequence; customOrderComplement: Sequence; diff --git a/loadDb.py b/loadDb.py index f9672129d..015d6b7fb 100644 --- a/loadDb.py +++ b/loadDb.py @@ -1,13 +1,15 @@ import sys import json +sys.path.insert(0,'.') from pyminsky import minsky, DataSpec #minsky.databaseIngestor.connect("sqlite3","db=citibike.sqlite") -minsky.databaseIngestor.connect("sqlite3","db=foo.sqlite") +minsky.databaseIngestor.connect("sqlite3","db=foo.sqlite","citibike") # sys.argv[0] is this script name filenames=sys.argv[1:] # set up spec for Citibike spec=DataSpec() + spec.setDataArea(1,14) spec.dataCols([0,11,14]) spec.dimensionCols([1,4,8,12,13]) @@ -16,11 +18,11 @@ spec.dimensions[1]({"type":"time","units":""}) spec.dimensions[11]({"type":"value","units":""}) spec.dimensions[14]({"type":"value","units":""}) + spec.duplicateKeyAction("av") spec.numCols(15) #'{"counter":false,"dataColOffset":14,"dataCols":[0,11],"dataRowOffset":1,"decSeparator":".","dimensionCols":[1,4,8,12,13,14],"dimensionNames":["tripduration","starttime","stoptime","start station id","start station name","start station latitude","start station longitude","end station id","end station name","end station latitude","end station longitude","bikeid","usertype","birth year","gender"],"dimensions":[{"type":"value","units":""},{"type":"time","units":""},{"type":"time","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""}],"dontFail":false,"duplicateKeyAction":"av","escape":"\0","headerRow":0,"horizontalDimName":"?","horizontalDimension":{"type":"string","units":""},"maxColumn":1000,"mergeDelimiters":false,"missingValue":NaN,"numCols":15,"quote":"\"","separator":","})' -minsky.databaseIngestor.table("citibike") -minsky.databaseIngestor.createTable(filenames,spec()) -#minsky.databaseIngestor.importFromCSV(filenames,spec()) +minsky.databaseIngestor.createTable(filenames[0],spec()) +minsky.databaseIngestor.loadDatabase(filenames,spec()) diff --git a/model/minsky.h b/model/minsky.h index 1f46c2994..9ed82a929 100644 --- a/model/minsky.h +++ b/model/minsky.h @@ -128,6 +128,23 @@ namespace minsky ecolab::TCLAccessor(name,g,s) {} }; + // a wrapper around ravel::Database that converts the spec format + class DatabaseIngestor + { + ravel::Database db; + CLASSDESC_ACCESS(DatabaseIngestor); + public: + void connect(const std::string& dbType, const std::string& connect, const std::string& table) + {db.connect(dbType,connect,table);} + void close() {db.close();} + + void createTable(const string& filename, const DataSpec& spec) + {db.createTable(filename,spec);} + void loadDatabase(const vector& filenames, const DataSpec& spec) + {db.loadDatabase(filenames,spec);} + void deduplicate(ravel::DuplicateKeyAction::Type action, const DataSpec& spec) + {db.deduplicate(action,spec);} + }; class Minsky: public Exclude, public RungeKutta, public Minsky_multipleEquities { @@ -255,7 +272,7 @@ namespace minsky GroupPtr model{new Group}; Canvas canvas{model}; - ravel::Database databaseIngestor; + DatabaseIngestor databaseIngestor; void clearAllMaps(bool clearHistory); void clearAllMaps() {clearAllMaps(true);} diff --git a/schema/dataSpecSchema.h b/schema/dataSpecSchema.h index 80089dbb3..6b4219687 100644 --- a/schema/dataSpecSchema.h +++ b/schema/dataSpecSchema.h @@ -20,6 +20,7 @@ #ifndef DATASPECSCHEMA_H #define DATASPECSCHEMA_H #include "CSVTools.h" +#include "ravelState.h" #include #include #include From 611bf9ba39fdbe141683e5f05eed19a647411eab Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Mon, 2 Jun 2025 17:39:02 +1000 Subject: [PATCH 13/41] Added a database member to RavelWrap --- RESTService/typescriptAPI.cc | 6 ++ RavelCAPI | 2 +- gui-js/libs/shared/src/lib/backend/minsky.ts | 75 ++++++++++++++++++++ model/ravelWrap.cc | 6 ++ model/ravelWrap.h | 5 ++ 5 files changed, 93 insertions(+), 1 deletion(-) diff --git a/RESTService/typescriptAPI.cc b/RESTService/typescriptAPI.cc index cbbdc288d..eabc65c03 100644 --- a/RESTService/typescriptAPI.cc +++ b/RESTService/typescriptAPI.cc @@ -7,8 +7,11 @@ #include "bookmark.h" #include "bookmark.tcd" #include "cairoSurfaceImage.tcd" +#include "cairoRenderer.tcd" #include "callableFunction.tcd" #include "canvas.tcd" +#define CLASSDESC_typescriptAPI___CAPIRenderer +#include "capiRenderer.tcd" #include "CSVDialog.tcd" #include "CSVParser.tcd" #include "CSVTools.tcd" @@ -260,6 +263,8 @@ int main() api.addClass(); api.addClass(); api.addClass(); + api.addClass(); + api.addClass(); api.addClass(); api.addClass(); api.addClass(); @@ -303,6 +308,7 @@ int main() cout << "class minsky__GodleyIcon__MoveCellArgs {}\n"; cout << "class minsky__RenderNativeWindow__RenderFrameArgs {}\n"; cout << "class minsky__VariableType__TypeT {}\n"; + cout << "class CAPIRenderer {}\n"; cout << "class civita__ITensor__Args {}\n"; cout << "class classdesc__json_pack_t {}\n"; cout << "class classdesc__pack_t {}\n"; diff --git a/RavelCAPI b/RavelCAPI index 36a677a8e..1198ce95d 160000 --- a/RavelCAPI +++ b/RavelCAPI @@ -1 +1 @@ -Subproject commit 36a677a8ecbaabed9543086613126bd205e81e4e +Subproject commit 1198ce95d5298a303ab0f3114ae7449b7d547540 diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index a4adde1ea..ef556fb80 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -10,6 +10,7 @@ class minsky__EventInterface__KeyPressArgs {} class minsky__GodleyIcon__MoveCellArgs {} class minsky__RenderNativeWindow__RenderFrameArgs {} class minsky__VariableType__TypeT {} +class CAPIRenderer {} class civita__ITensor__Args {} class classdesc__json_pack_t {} class classdesc__pack_t {} @@ -1757,6 +1758,7 @@ export class PubTab extends RenderNativeWindow { export class Ravel extends Item { axisDimensions: Map; + db: ravelCAPI__Database; lockGroup: RavelLockGroup; popup: RavelPopup; svgRenderer: SVGRenderer; @@ -1766,6 +1768,7 @@ export class Ravel extends Item { else super(prefix.$prefix()) this.axisDimensions=new Map(this.$prefix()+'.axisDimensions',civita__Dimension); + this.db=new ravelCAPI__Database(this.$prefix()+'.db'); this.lockGroup=new RavelLockGroup(this.$prefix()+'.lockGroup'); this.popup=new RavelPopup(this.$prefix()+'.popup'); this.svgRenderer=new SVGRenderer(this.$prefix()+'.svgRenderer'); @@ -1793,6 +1796,7 @@ export class Ravel extends Item { async handleSortableByValue(): Promise {return this.$callMethod('handleSortableByValue');} async hypercube(): Promise {return this.$callMethod('hypercube');} async inItem(a1: number,a2: number): Promise {return this.$callMethod('inItem',a1,a2);} + async initRavelFromDb(): Promise {return this.$callMethod('initRavelFromDb');} async joinLockGroup(a1: number): Promise {return this.$callMethod('joinLockGroup',a1);} async leaveLockGroup(): Promise {return this.$callMethod('leaveLockGroup');} async lockGroupColours(): Promise {return this.$callMethod('lockGroupColours');} @@ -2488,6 +2492,77 @@ export class minsky__Canvas__ZoomCrop extends CppClass { async zoom(...args: number[]): Promise {return this.$callMethod('zoom',...args);} } +export class ravelCAPI__Database extends CppClass { + constructor(prefix: string){ + super(prefix); + } + async close(): Promise {return this.$callMethod('close');} + async columnNames(): Promise {return this.$callMethod('columnNames');} + async connect(a1: string,a2: string,a3: string): Promise {return this.$callMethod('connect',a1,a2,a3);} + async createTable(a1: string,a2: ravel__DataSpec): Promise {return this.$callMethod('createTable',a1,a2);} + async deduplicate(a1: string,a2: ravel__DataSpec): Promise {return this.$callMethod('deduplicate',a1,a2);} + async fullHypercube(a1: ravelCAPI__Ravel): Promise {return this.$callMethod('fullHypercube',a1);} + async hyperSlice(a1: ravelCAPI__Ravel): Promise {return this.$callMethod('hyperSlice',a1);} + async loadDatabase(a1: string[],a2: ravel__DataSpec): Promise {return this.$callMethod('loadDatabase',a1,a2);} + async setAxisNames(a1: Container,a2: string): Promise {return this.$callMethod('setAxisNames',a1,a2);} +} + +export class ravelCAPI__Ravel extends CppClass { + constructor(prefix: string){ + super(prefix); + } + async addHandle(a1: string,a2: string[]): Promise {return this.$callMethod('addHandle',a1,a2);} + async adjustSlicer(a1: number): Promise {return this.$callMethod('adjustSlicer',a1);} + async allSliceLabels(a1: number,a2: string): Promise {return this.$callMethod('allSliceLabels',a1,a2);} + async applyCustomPermutation(a1: number,a2: number[]): Promise {return this.$callMethod('applyCustomPermutation',a1,a2);} + async available(): Promise {return this.$callMethod('available');} + async clear(): Promise {return this.$callMethod('clear');} + async currentPermutation(a1: number): Promise {return this.$callMethod('currentPermutation',a1);} + async daysUntilExpired(): Promise {return this.$callMethod('daysUntilExpired');} + async description(...args: any[]): Promise {return this.$callMethod('description',...args);} + async displayFilterCaliper(a1: number,a2: boolean): Promise {return this.$callMethod('displayFilterCaliper',a1,a2);} + async explain(a1: number,a2: number): Promise {return this.$callMethod('explain',a1,a2);} + async fromXML(a1: string): Promise {return this.$callMethod('fromXML',a1);} + async getCaliperPositions(a1: number): Promise {return this.$callMethod('getCaliperPositions',a1);} + async getHandleState(a1: number): Promise {return this.$callMethod('getHandleState',a1);} + async getRavelState(): Promise {return this.$callMethod('getRavelState');} + async handleDescription(a1: number): Promise {return this.$callMethod('handleDescription',a1);} + async handleSetReduction(a1: number,a2: string): Promise {return this.$callMethod('handleSetReduction',a1,a2);} + async hyperSlice(a1: civita__ITensor): Promise {return this.$callMethod('hyperSlice',a1);} + async lastError(): Promise {return this.$callMethod('lastError');} + async nextReduction(a1: string): Promise {return this.$callMethod('nextReduction',a1);} + async numAllSliceLabels(a1: number): Promise {return this.$callMethod('numAllSliceLabels',a1);} + async numHandles(): Promise {return this.$callMethod('numHandles');} + async numSliceLabels(a1: number): Promise {return this.$callMethod('numSliceLabels',a1);} + async onMouseDown(a1: number,a2: number): Promise {return this.$callMethod('onMouseDown',a1,a2);} + async onMouseLeave(): Promise {return this.$callMethod('onMouseLeave');} + async onMouseMotion(a1: number,a2: number): Promise {return this.$callMethod('onMouseMotion',a1,a2);} + async onMouseOver(a1: number,a2: number): Promise {return this.$callMethod('onMouseOver',a1,a2);} + async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} + async orderLabels(a1: number,a2: string): Promise {return this.$callMethod('orderLabels',a1,a2);} + async outputHandleIds(): Promise {return this.$callMethod('outputHandleIds');} + async populateFromHypercube(a1: civita__Hypercube): Promise {return this.$callMethod('populateFromHypercube',a1);} + async radius(): Promise {return this.$callMethod('radius');} + async rank(): Promise {return this.$callMethod('rank');} + async redistributeHandles(): Promise {return this.$callMethod('redistributeHandles');} + async render(a1: CAPIRenderer): Promise {return this.$callMethod('render',a1);} + async rescale(a1: number): Promise {return this.$callMethod('rescale',a1);} + async resetExplain(): Promise {return this.$callMethod('resetExplain');} + async selectedHandle(): Promise {return this.$callMethod('selectedHandle');} + async setCaliperPositions(a1: number,a2: number,a3: number): Promise {return this.$callMethod('setCaliperPositions',a1,a2,a3);} + async setCalipers(a1: number,a2: string,a3: string): Promise {return this.$callMethod('setCalipers',a1,a2,a3);} + async setExplain(a1: string,a2: number,a3: number): Promise {return this.$callMethod('setExplain',a1,a2,a3);} + async setHandleDescription(a1: number,a2: string): Promise {return this.$callMethod('setHandleDescription',a1,a2);} + async setHandleState(a1: number,a2: ravel__HandleState): Promise {return this.$callMethod('setHandleState',a1,a2);} + async setOutputHandleIds(a1: number[]): Promise {return this.$callMethod('setOutputHandleIds',a1);} + async setRavelState(a1: ravel__RavelState): Promise {return this.$callMethod('setRavelState',a1);} + async setSlicer(a1: number,a2: string): Promise {return this.$callMethod('setSlicer',a1,a2);} + async sliceLabels(a1: number): Promise {return this.$callMethod('sliceLabels',a1);} + async sortByValue(a1: civita__ITensor,a2: string): Promise {return this.$callMethod('sortByValue',a1,a2);} + async toXML(): Promise {return this.$callMethod('toXML');} + async version(): Promise {return this.$callMethod('version');} +} + export class ravel__DataSpec extends CppClass { dataCols: Container; dimensionCols: Container; diff --git a/model/ravelWrap.cc b/model/ravelWrap.cc index 08ac61b4a..9fc09bef2 100644 --- a/model/ravelWrap.cc +++ b/model/ravelWrap.cc @@ -276,6 +276,12 @@ namespace minsky minsky().requestReset(); } + void Ravel::initRavelFromDb() + { + if (db && m_ports[1]->wires().empty()) + db.fullHypercube(wrappedRavel); + } + bool Ravel::displayFilterCaliper() const { diff --git a/model/ravelWrap.h b/model/ravelWrap.h index f7bc30f12..00304da3d 100644 --- a/model/ravelWrap.h +++ b/model/ravelWrap.h @@ -83,6 +83,8 @@ namespace minsky static SVGRenderer svgRenderer; ///< SVG icon to display when not in editor mode RavelPopup popup; ///< popup Ravel control window bool flipped=false; + ravelCAPI::Database db; ///< backing database + Ravel(); // copy operations needed for clone, but not really used for now // define them as empty operations to prevent double frees if accidentally used @@ -135,6 +137,9 @@ namespace minsky /// collapse all handles (applying nextReduction op where appropriate) /// @param collapse if true, uncollapse if false void collapseAllHandles(bool collapse=true); + + /// if connected to a database, initialise the ravel state from it + void initRavelFromDb(); /// enable/disable calipers on currently selected handle bool displayFilterCaliper() const; From 9013b322d820d837ee7761705f8d037740706875 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Wed, 4 Jun 2025 17:49:22 +1000 Subject: [PATCH 14/41] Added database connection dialog. --- RavelCAPI | 2 +- .../src/app/managers/ContextMenuManager.ts | 11 +++++ .../minsky-web/src/app/app-routing.module.ts | 5 ++ gui-js/libs/shared/src/lib/backend/minsky.ts | 2 +- gui-js/libs/ui-components/src/index.ts | 1 + .../connect-database.component.ts | 49 +++++++++++++++++++ .../connect-database/connect-database.html | 17 +++++++ .../connect-database/connect-database.scss | 13 +++++ 8 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts create mode 100644 gui-js/libs/ui-components/src/lib/connect-database/connect-database.html create mode 100644 gui-js/libs/ui-components/src/lib/connect-database/connect-database.scss diff --git a/RavelCAPI b/RavelCAPI index 1198ce95d..3dbd169ea 160000 --- a/RavelCAPI +++ b/RavelCAPI @@ -1 +1 @@ -Subproject commit 1198ce95d5298a303ab0f3114ae7449b7d547540 +Subproject commit 3dbd169eae6301e063b593f4de2036a5bc0b26f4 diff --git a/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts b/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts index ba7db78d8..f4cbf6ad4 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts @@ -900,6 +900,17 @@ export class ContextMenuManager { checked: editorMode, click: () => {ravel.toggleEditorMode();} }), + new MenuItem({ + label: 'Connect to database', + click: () => { + WindowManager.createPopupWindowWithRouting({ + title: 'Connect to database', + url: '#/headless/connect-database', + height: 100, + width: 250, + }) + }, + }), new MenuItem({ label: 'Export as CSV', submenu: this.exportAsCSVSubmenu(ravel), diff --git a/gui-js/apps/minsky-web/src/app/app-routing.module.ts b/gui-js/apps/minsky-web/src/app/app-routing.module.ts index b50d3cffc..48d0b1db9 100644 --- a/gui-js/apps/minsky-web/src/app/app-routing.module.ts +++ b/gui-js/apps/minsky-web/src/app/app-routing.module.ts @@ -1,6 +1,7 @@ import { NgModule } from '@angular/core'; import { RouterModule, Routes } from '@angular/router'; import { + ConnectDatabaseComponent, CliInputComponent, EditDescriptionComponent, EditGodleyCurrencyComponent, @@ -50,6 +51,10 @@ const routes: Routes = [ path: 'headless/menu', loadChildren: () => import('@minsky/menu').then((m) => m.MenuModule), }, + { + path: 'headless/connect-database', + component: ConnectDatabaseComponent, + }, { path: 'headless/rename-all-instances', component: RenameAllInstancesComponent, diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index ef556fb80..c6216e227 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -2497,13 +2497,13 @@ export class ravelCAPI__Database extends CppClass { super(prefix); } async close(): Promise {return this.$callMethod('close');} - async columnNames(): Promise {return this.$callMethod('columnNames');} async connect(a1: string,a2: string,a3: string): Promise {return this.$callMethod('connect',a1,a2,a3);} async createTable(a1: string,a2: ravel__DataSpec): Promise {return this.$callMethod('createTable',a1,a2);} async deduplicate(a1: string,a2: ravel__DataSpec): Promise {return this.$callMethod('deduplicate',a1,a2);} async fullHypercube(a1: ravelCAPI__Ravel): Promise {return this.$callMethod('fullHypercube',a1);} async hyperSlice(a1: ravelCAPI__Ravel): Promise {return this.$callMethod('hyperSlice',a1);} async loadDatabase(a1: string[],a2: ravel__DataSpec): Promise {return this.$callMethod('loadDatabase',a1,a2);} + async numericalColumnNames(): Promise {return this.$callMethod('numericalColumnNames');} async setAxisNames(a1: Container,a2: string): Promise {return this.$callMethod('setAxisNames',a1,a2);} } diff --git a/gui-js/libs/ui-components/src/index.ts b/gui-js/libs/ui-components/src/index.ts index 2a63964c9..cbd2d91bb 100644 --- a/gui-js/libs/ui-components/src/index.ts +++ b/gui-js/libs/ui-components/src/index.ts @@ -1,3 +1,4 @@ +export * from './lib/connect-database/connect-database.component'; export * from './lib/cli-input/cli-input.component'; export * from './lib/create-variable/create-variable.component'; export * from './lib/edit-description/edit-description.component'; diff --git a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts new file mode 100644 index 000000000..3341aea3d --- /dev/null +++ b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts @@ -0,0 +1,49 @@ +import { ChangeDetectorRef, Component, OnInit } from '@angular/core'; +import { FormControl, FormGroup, FormsModule, ReactiveFormsModule } from '@angular/forms'; +import { ActivatedRoute } from '@angular/router'; +import { ElectronService } from '@minsky/core'; +import { events, Item, Wire } from '@minsky/shared'; +import { MatButtonModule } from '@angular/material/button'; + +@Component({ + selector: 'connect-database', + templateUrl: './connect-database.html', + styleUrls: ['./connect-database.scss'], + standalone: true, + imports: [ + FormsModule, + ReactiveFormsModule, + MatButtonModule, + ], +}) +export class ConnectDatabaseComponent implements OnInit { + + connectDatabaseForm: FormGroup; + dbType="sqlite3"; + constructor( + private route: ActivatedRoute, + private electronService: ElectronService, + private cdRef: ChangeDetectorRef + ) {} + + ngOnInit(): void { + this.route.queryParams.subscribe((params) => { + }); + + this.connectDatabaseForm = new FormGroup({ + dbType: new FormControl(), + connectionString: new FormControl(), + }); + } + + setDbType(event) { + const target = event.target as HTMLSelectElement; + this.dbType=target.value; + } + + async connect() { + this.closeWindow(); + } + + closeWindow() {this.electronService.closeWindow();} +} diff --git a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html new file mode 100644 index 000000000..760dafe0c --- /dev/null +++ b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html @@ -0,0 +1,17 @@ +
+
+ + +
+ + +
diff --git a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.scss b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.scss new file mode 100644 index 000000000..ba9966679 --- /dev/null +++ b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.scss @@ -0,0 +1,13 @@ +@import '../../../../shared/src/lib/theme/common-style.scss'; + +.row { + display: flex; + flex-direction: row; + align-items: center; +} + +.container { + display: flex; + flex-direction: column; + align-items: center; +} From 97902522f868c38dd69606596b9e5ac47a48ee5d Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Fri, 6 Jun 2025 14:21:49 +1000 Subject: [PATCH 15/41] Added connect-database component. --- .../connect-database/connect-database.component.ts | 12 ++++++++++++ .../src/lib/connect-database/connect-database.html | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts index 3341aea3d..d11122f62 100644 --- a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts +++ b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts @@ -40,6 +40,18 @@ export class ConnectDatabaseComponent implements OnInit { const target = event.target as HTMLSelectElement; this.dbType=target.value; } + + async selectFile() { + let options: OpenDialogOptions = { + filters: [ + { extensions: ['sqlite'], name: 'CSV' }, + { extensions: ['*'], name: 'All Files' }, + ], + properties: ['openFile'], + }; + //if (defaultPath) options['defaultPath'] = defaultPath; + this.filePath = await this.electronService.openFileDialog(options); + } async connect() { this.closeWindow(); diff --git a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html index 760dafe0c..6f2c7ecbe 100644 --- a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html +++ b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html @@ -10,7 +10,7 @@ - + From 7bd90dddd3ceb22a31944695203ff88550f3b2f8 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Mon, 9 Jun 2025 11:25:08 +1000 Subject: [PATCH 16/41] Ravel now populated from database. --- .../src/lib/connect-database/connect-database.component.ts | 2 ++ .../src/lib/connect-database/connect-database.html | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts index 42e9eb8cd..aa714cf3f 100644 --- a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts +++ b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts @@ -78,6 +78,8 @@ export class ConnectDatabaseComponent implements OnInit { connect() { this.ravel.db.connect(this.dbType,this.connection,this.table); + this.ravel.initRavelFromDb(); + this.electronService.minsky.canvas.requestRedraw(); this.closeWindow(); } diff --git a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html index 337619339..cf60d7ea7 100644 --- a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html +++ b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html @@ -15,7 +15,7 @@
From eff8c4dfe7f113f9eb7df7b02c5147a6c80798e3 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Mon, 9 Jun 2025 18:06:59 +1000 Subject: [PATCH 17/41] Loading hyperslice data from database into Ravel. --- engine/equations.cc | 1 + engine/minskyTensorOps.cc | 9 +++++---- .../src/app/managers/ContextMenuManager.ts | 2 +- gui-js/libs/shared/src/lib/backend/minsky.ts | 13 +++++++------ .../lib/connect-database/connect-database.html | 2 +- model/canvas.cc | 4 ++-- model/godleyIcon.cc | 7 +++++-- model/godleyIcon.h | 2 +- model/item.h | 4 ++-- model/ravelWrap.cc | 17 +++++++++++++++-- model/ravelWrap.h | 7 ++++++- 11 files changed, 46 insertions(+), 22 deletions(-) diff --git a/engine/equations.cc b/engine/equations.cc index f7011a6df..5186e18e9 100644 --- a/engine/equations.cc +++ b/engine/equations.cc @@ -208,6 +208,7 @@ namespace MathDAG { if (!visited.insert(this).second) return false; // cycle detected, break + if (type()==OperationType::ravel) return true; switch (OperationType::classify(type())) { case reduction: case scan: case tensor: case statistics: diff --git a/engine/minskyTensorOps.cc b/engine/minskyTensorOps.cc index 5d803b773..e258a9c0e 100644 --- a/engine/minskyTensorOps.cc +++ b/engine/minskyTensorOps.cc @@ -1501,13 +1501,14 @@ namespace minsky CLASSDESC_ACCESS(Ravel); public: - RavelTensor(const Ravel& ravel): ravel(ravel) {} + RavelTensor(const Ravel& ravel): ravel(ravel) { + if (ravel.db) chain=const_cast(ravel).createChain(nullptr); + } void setArgument(const TensorPtr& a,const Args&) override { - // not sure how to avoid this const cast here arg=a; - const_cast(ravel).populateHypercube(a->hypercube()); - chain=ravel::createRavelChain(ravel.getState(), a); + // not sure how to avoid this const cast here + chain=const_cast(ravel).createChain(a); } double operator[](size_t i) const override { diff --git a/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts b/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts index f4cbf6ad4..17467f50e 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts @@ -906,7 +906,7 @@ export class ContextMenuManager { WindowManager.createPopupWindowWithRouting({ title: 'Connect to database', url: '#/headless/connect-database', - height: 100, + height: 120, width: 250, }) }, diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index 4f8070762..9151a7756 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -93,7 +93,7 @@ export class Item extends CppClass { async onMouseLeave(): Promise {return this.$callMethod('onMouseLeave');} async onMouseMotion(a1: number,a2: number): Promise {return this.$callMethod('onMouseMotion',a1,a2);} async onMouseOver(a1: number,a2: number): Promise {return this.$callMethod('onMouseOver',a1,a2);} - async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} + async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} async onResizeHandle(a1: number,a2: number): Promise {return this.$callMethod('onResizeHandle',a1,a2);} async onResizeHandles(...args: boolean[]): Promise {return this.$callMethod('onResizeHandles',...args);} async portX(a1: number): Promise {return this.$callMethod('portX',a1);} @@ -266,7 +266,7 @@ export class VariableBase extends Item { async onMouseLeave(): Promise {return this.$callMethod('onMouseLeave');} async onMouseMotion(a1: number,a2: number): Promise {return this.$callMethod('onMouseMotion',a1,a2);} async onMouseOver(a1: number,a2: number): Promise {return this.$callMethod('onMouseOver',a1,a2);} - async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} + async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} async onResizeHandle(a1: number,a2: number): Promise {return this.$callMethod('onResizeHandle',a1,a2);} async onResizeHandles(...args: boolean[]): Promise {return this.$callMethod('onResizeHandles',...args);} async portX(a1: number): Promise {return this.$callMethod('portX',a1);} @@ -741,7 +741,7 @@ export class GodleyIcon extends Item { async onMouseLeave(): Promise {return this.$callMethod('onMouseLeave');} async onMouseMotion(a1: number,a2: number): Promise {return this.$callMethod('onMouseMotion',a1,a2);} async onMouseOver(a1: number,a2: number): Promise {return this.$callMethod('onMouseOver',a1,a2);} - async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} + async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} async removeControlledItems(a1: GroupItems): Promise {return this.$callMethod('removeControlledItems',a1);} async resize(a1: LassoBox): Promise {return this.$callMethod('resize',a1);} async rowSum(a1: number): Promise {return this.$callMethod('rowSum',a1);} @@ -1109,7 +1109,7 @@ export class Group extends Item { async onMouseLeave(): Promise {return this.$callMethod('onMouseLeave');} async onMouseMotion(a1: number,a2: number): Promise {return this.$callMethod('onMouseMotion',a1,a2);} async onMouseOver(a1: number,a2: number): Promise {return this.$callMethod('onMouseOver',a1,a2);} - async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} + async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} async onResizeHandle(a1: number,a2: number): Promise {return this.$callMethod('onResizeHandle',a1,a2);} async onResizeHandles(...args: boolean[]): Promise {return this.$callMethod('onResizeHandles',...args);} async portX(a1: number): Promise {return this.$callMethod('portX',a1);} @@ -1779,6 +1779,7 @@ export class Ravel extends Item { async applyState(a1: ravel__RavelState): Promise {return this.$callMethod('applyState',a1);} async broadcastStateToLockGroup(): Promise {return this.$callMethod('broadcastStateToLockGroup');} async collapseAllHandles(a1: boolean): Promise {return this.$callMethod('collapseAllHandles',a1);} + async createChain(a1: civita__ITensor): Promise {return this.$callMethod('createChain',a1);} async description(): Promise {return this.$callMethod('description');} async dimension(a1: number): Promise {return this.$callMethod('dimension',a1);} async dimensionType(...args: any[]): Promise {return this.$callMethod('dimensionType',...args);} @@ -1809,7 +1810,7 @@ export class Ravel extends Item { async onMouseLeave(): Promise {return this.$callMethod('onMouseLeave');} async onMouseMotion(a1: number,a2: number): Promise {return this.$callMethod('onMouseMotion',a1,a2);} async onMouseOver(a1: number,a2: number): Promise {return this.$callMethod('onMouseOver',a1,a2);} - async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} + async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} async pickSliceLabels(a1: number,a2: string[]): Promise {return this.$callMethod('pickSliceLabels',a1,a2);} async pickedSliceLabels(...args: any[]): Promise {return this.$callMethod('pickedSliceLabels',...args);} async populateHypercube(a1: civita__Hypercube): Promise {return this.$callMethod('populateHypercube',a1);} @@ -2031,7 +2032,7 @@ export class Selection extends CppClass { async onMouseLeave(): Promise {return this.$callMethod('onMouseLeave');} async onMouseMotion(a1: number,a2: number): Promise {return this.$callMethod('onMouseMotion',a1,a2);} async onMouseOver(a1: number,a2: number): Promise {return this.$callMethod('onMouseOver',a1,a2);} - async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} + async onMouseUp(a1: number,a2: number): Promise {return this.$callMethod('onMouseUp',a1,a2);} async onResizeHandle(a1: number,a2: number): Promise {return this.$callMethod('onResizeHandle',a1,a2);} async onResizeHandles(...args: boolean[]): Promise {return this.$callMethod('onResizeHandles',...args);} async portX(a1: number): Promise {return this.$callMethod('portX',a1);} diff --git a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html index cf60d7ea7..e4520a3d0 100644 --- a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html +++ b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html @@ -18,7 +18,7 @@ -
+
diff --git a/model/canvas.cc b/model/canvas.cc index 84323ebe9..b9af9d02d 100644 --- a/model/canvas.cc +++ b/model/canvas.cc @@ -146,9 +146,9 @@ namespace minsky if (itemFocus && clickType==ClickType::inItem) { - itemFocus->onMouseUp(x,y); + bool requestReset=itemFocus->onMouseUp(x,y); itemFocus.reset(); // prevent spurious mousemove events being processed - minsky().requestReset(); + if (requestReset) minsky().requestReset(); } if (fromPort.get()) { diff --git a/model/godleyIcon.cc b/model/godleyIcon.cc index 4ad848960..4c807f2a2 100644 --- a/model/godleyIcon.cc +++ b/model/godleyIcon.cc @@ -610,8 +610,11 @@ namespace minsky void GodleyIcon::onMouseDown(float x, float y) {if (m_editorMode) editor.mouseDown(toEditorX(x),toEditorY(y));} - void GodleyIcon::onMouseUp(float x, float y) - {if (m_editorMode) editor.mouseUp(toEditorX(x),toEditorY(y));} + bool GodleyIcon::onMouseUp(float x, float y) + { + if (m_editorMode) editor.mouseUp(toEditorX(x),toEditorY(y)); + return m_editorMode; + } bool GodleyIcon::onMouseMotion(float x, float y) { diff --git a/model/godleyIcon.h b/model/godleyIcon.h index 4779b36a1..a16c588e4 100644 --- a/model/godleyIcon.h +++ b/model/godleyIcon.h @@ -151,7 +151,7 @@ namespace minsky void insertControlled(Selection& selection) override; void onMouseDown(float, float) override; - void onMouseUp(float, float) override; + bool onMouseUp(float, float) override; bool onMouseMotion(float, float) override; bool onMouseOver(float, float) override; void onMouseLeave() override; diff --git a/model/item.h b/model/item.h index db185662e..134fe5a82 100644 --- a/model/item.h +++ b/model/item.h @@ -258,8 +258,8 @@ namespace minsky virtual bool onItem(float x, float y) const; /// respond to mouse down events virtual void onMouseDown(float x, float y) {} - /// respond to mouse up events - virtual void onMouseUp(float x, float y) {} + /// respond to mouse up events. Return true if model needs to be reset. + virtual bool onMouseUp(float x, float y) {return false;} /// respond to mouse motion events with button pressed /// @return true if it needs to be rerendered virtual bool onMouseMotion(float x, float y) {return false;} diff --git a/model/ravelWrap.cc b/model/ravelWrap.cc index 9fc09bef2..e42ebfd9e 100644 --- a/model/ravelWrap.cc +++ b/model/ravelWrap.cc @@ -160,12 +160,13 @@ namespace minsky wrappedRavel.onMouseDown((xx-x())*invZ,(yy-y())*invZ); } - void Ravel::onMouseUp(float xx, float yy) + bool Ravel::onMouseUp(float xx, float yy) { const double invZ=1/zoomFactor(); wrappedRavel.onMouseUp((xx-x())*invZ,(yy-y())*invZ); resortHandleIfDynamic(); broadcastStateToLockGroup(); + return m_ports[1]->numWires(); // only reset if input port connected. Don't if sourced from database } bool Ravel::onMouseMotion(float xx, float yy) { @@ -279,9 +280,21 @@ namespace minsky void Ravel::initRavelFromDb() { if (db && m_ports[1]->wires().empty()) - db.fullHypercube(wrappedRavel); + { + minsky().flags&=~Minsky::reset_needed; //disable resetting until user gets a chance to manipulate ravel + db.fullHypercube(wrappedRavel); + } } + vector Ravel::createChain(const TensorPtr& arg) + { + if (arg) + { + populateHypercube(arg->hypercube()); + return ravel::createRavelChain(getState(), arg); + } + return {db.hyperSlice(wrappedRavel)}; + } bool Ravel::displayFilterCaliper() const { diff --git a/model/ravelWrap.h b/model/ravelWrap.h index 75f947927..2303a8719 100644 --- a/model/ravelWrap.h +++ b/model/ravelWrap.h @@ -110,7 +110,7 @@ namespace minsky void resize(const LassoBox&) override; bool inItem(float x, float y) const override; void onMouseDown(float x, float y) override; - void onMouseUp(float x, float y) override; + bool onMouseUp(float x, float y) override; bool onMouseMotion(float x, float y) override; bool onMouseOver(float x, float y) override; void onMouseLeave() override {wrappedRavel.onMouseLeave();} @@ -140,6 +140,11 @@ namespace minsky /// if connected to a database, initialise the ravel state from it void initRavelFromDb(); + + /// make a tensor expression corresponding to the state of this + /// Ravel, applied to \a arg. If \a arg is nullptr, then the + /// returned expression is extracted from the database. + std::vector createChain(const TensorPtr& arg); /// enable/disable calipers on currently selected handle bool displayFilterCaliper() const; From e9ec1428e69630e7ad4731adc0c5342ff9933ff3 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Tue, 24 Jun 2025 11:14:53 +1000 Subject: [PATCH 18/41] Made database operations cancellable. --- RESTService/addon.cc | 2 ++ RavelCAPI | 2 +- gui-js/libs/shared/src/lib/backend/minsky.ts | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/RESTService/addon.cc b/RESTService/addon.cc index 460a64490..c088b68b4 100644 --- a/RESTService/addon.cc +++ b/RESTService/addon.cc @@ -242,6 +242,7 @@ namespace minsky if (reset_flag()) requestReset(); civita::ITensor::cancel(false); + ravelCAPI::Ravel::cancel(false); // disable quoting wide characters in UTF-8 strings auto result=write(registry.process(command, arguments)->asBuffer(),json5_parser::raw_utf8); commandHook(command,arguments); @@ -594,6 +595,7 @@ struct MinskyAddon: public Addon Value cancelProgress(const Napi::CallbackInfo& info) { *addOnMinsky.progressState.cancel=true; civita::ITensor::cancel(true); + ravelCAPI::Ravel::cancel(true); return info.Env().Null(); } diff --git a/RavelCAPI b/RavelCAPI index 2112a6fc2..c15b8d3f1 160000 --- a/RavelCAPI +++ b/RavelCAPI @@ -1 +1 @@ -Subproject commit 2112a6fc2f4a337777d271916e79415f7df36353 +Subproject commit c15b8d3f111ca6f34f8f68aa12f2bf171509ea5d diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index 2067d3cd3..0de5e90f6 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -2516,6 +2516,7 @@ export class ravelCAPI__Ravel extends CppClass { async allSliceLabels(a1: number,a2: string): Promise {return this.$callMethod('allSliceLabels',a1,a2);} async applyCustomPermutation(a1: number,a2: number[]): Promise {return this.$callMethod('applyCustomPermutation',a1,a2);} async available(): Promise {return this.$callMethod('available');} + async cancel(a1: boolean): Promise {return this.$callMethod('cancel',a1);} async clear(): Promise {return this.$callMethod('clear');} async currentPermutation(a1: number): Promise {return this.$callMethod('currentPermutation',a1);} async daysUntilExpired(): Promise {return this.$callMethod('daysUntilExpired');} From 1a5944a42e04254e96403079b52f05652d581c8f Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Tue, 24 Jun 2025 14:58:36 +1000 Subject: [PATCH 19/41] Update RavelCAPI ref. --- RavelCAPI | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RavelCAPI b/RavelCAPI index c15b8d3f1..fcaf36a82 160000 --- a/RavelCAPI +++ b/RavelCAPI @@ -1 +1 @@ -Subproject commit c15b8d3f111ca6f34f8f68aa12f2bf171509ea5d +Subproject commit fcaf36a822bd00743eef28cbb8eea698f905d05e From 360005914ebe3fed78dd6640f4c1ffc9ded9cb50 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Wed, 25 Jun 2025 12:53:28 +1000 Subject: [PATCH 20/41] Save DB connections to rvl files Fix multiple progress bar deadlock --- engine/saver.cc | 1 + gui-js/apps/minsky-electron/src/app/backend-init.ts | 4 ++++ gui-js/libs/shared/src/lib/backend/minsky.ts | 3 ++- schema/schema3.cc | 9 +++++++++ schema/schema3.h | 3 ++- 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/engine/saver.cc b/engine/saver.cc index 8e15c0886..407d1efc6 100644 --- a/engine/saver.cc +++ b/engine/saver.cc @@ -20,6 +20,7 @@ #include "saver.h" #include "schema3.h" #include "CSVTools.xcd" +#include "dynamicRavelCAPI.xcd" #include "minsky_epilogue.h" namespace minsky diff --git a/gui-js/apps/minsky-electron/src/app/backend-init.ts b/gui-js/apps/minsky-electron/src/app/backend-init.ts index 6bfac4b18..09ad2f4ef 100644 --- a/gui-js/apps/minsky-electron/src/app/backend-init.ts +++ b/gui-js/apps/minsky-electron/src/app/backend-init.ts @@ -148,6 +148,10 @@ restService.setBusyCursorCallback(function (busy: boolean) { if (!initProgressBar && busy) initProgressBar=setTimeout(()=>{ progress.browserWindow={parent: WindowManager.getMainWindow(), height: 200}; + if (progressBar) { + progressBar.setCompleted(); + progressBar.close(); + } progressBar=new ProgressBar(progress); progressBar.on('ready',()=>{progressBar._window?.webContents.executeJavaScript(injectCancelButton);}); progressBar.value=progress.value; diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index 0de5e90f6..7b77ff06a 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -2424,7 +2424,7 @@ export class civita__TensorVal extends CppClass { super(prefix); } async allocVal(): Promise {return this.$callMethod('allocVal');} - async assign(a1: Map): Promise {return this.$callMethod('assign',a1);} + async assign(...args: any[]): Promise {return this.$callMethod('assign',...args);} async at(a1: number): Promise {return this.$callMethod('at',a1);} async atHCIndex(a1: number): Promise {return this.$callMethod('atHCIndex',a1);} async begin(): Promise {return this.$callMethod('begin');} @@ -2497,6 +2497,7 @@ export class ravelCAPI__Database extends CppClass { } async close(): Promise {return this.$callMethod('close');} async connect(a1: string,a2: string,a3: string): Promise {return this.$callMethod('connect',a1,a2,a3);} + async connection(): Promise {return this.$callMethod('connection');} async createTable(a1: string,a2: ravel__DataSpec): Promise {return this.$callMethod('createTable',a1,a2);} async deduplicate(a1: string,a2: ravel__DataSpec): Promise {return this.$callMethod('deduplicate',a1,a2);} async fullHypercube(a1: ravelCAPI__Ravel): Promise {return this.$callMethod('fullHypercube',a1);} diff --git a/schema/schema3.cc b/schema/schema3.cc index 0163d5e62..cbde82bdc 100644 --- a/schema/schema3.cc +++ b/schema/schema3.cc @@ -19,6 +19,7 @@ #include "dataOp.h" #include "schema3.h" #include "CSVTools.xcd" +#include "dynamicRavelCAPI.xcd" #include "sheet.h" #include "userFunction.h" #include "minsky_epilogue.h" @@ -193,6 +194,9 @@ namespace schema3 items.back().editorMode=r->editorMode(); } if (r->flipped) items.back().rotation=180; + auto dbConnection=r->db.connection(); + if (!dbConnection.dbType.empty()) + items.back().dbConnection=dbConnection; } if (auto* l=dynamic_cast(i)) if (l->locked()) @@ -500,6 +504,11 @@ namespace schema3 if (auto* x1=dynamic_cast(&x)) { + if (y.dbConnection) + { + x1->db.connect(y.dbConnection->dbType,y.dbConnection->connection,y.dbConnection->table); + x1->initRavelFromDb(); + } if (y.ravelState) { x1->applyState(y.ravelState->toRavelRavelState()); diff --git a/schema/schema3.h b/schema/schema3.h index e0ed588f2..6f3357a62 100644 --- a/schema/schema3.h +++ b/schema/schema3.h @@ -113,8 +113,9 @@ namespace schema3 Optional csvDataSpec; //CSV import data Optional> dataOpData; Optional expression; // userfunction - Optional filename; + Optional filename; Optional ravelState; + Optional dbConnection; Optional lockGroup; Optional> lockGroupHandles; Optional dimensions; From a2bfeb7881f73d7c7c631791252d04d354492fee Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Wed, 25 Jun 2025 15:27:52 +1000 Subject: [PATCH 21/41] Fix test build. --- test/testSaver.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/testSaver.cc b/test/testSaver.cc index bd11e2d2e..c47b817e6 100644 --- a/test/testSaver.cc +++ b/test/testSaver.cc @@ -20,6 +20,7 @@ #include "saver.h" #include "schema3.h" #include "CSVTools.xcd" +#include "dynamicRavelCAPI.xcd" #include "minsky_epilogue.h" #undef True #include From a3d65b71e9575d237d48801b45f01eb162b498d9 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Wed, 25 Jun 2025 15:40:52 +1000 Subject: [PATCH 22/41] Update RavelCAPI ref. --- RavelCAPI | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RavelCAPI b/RavelCAPI index fcaf36a82..62bfe30e7 160000 --- a/RavelCAPI +++ b/RavelCAPI @@ -1 +1 @@ -Subproject commit fcaf36a822bd00743eef28cbb8eea698f905d05e +Subproject commit 62bfe30e7c66ff57441d02a1194e9930e92810a7 From c70fa02dc5fdec11c96d3b1063071673fa7dd4d9 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Wed, 25 Jun 2025 17:48:31 +1000 Subject: [PATCH 23/41] Cache DB query results. --- gui-js/libs/shared/src/lib/backend/minsky.ts | 3 +++ model/ravelWrap.cc | 8 +++++++- model/ravelWrap.h | 3 +++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index 7b77ff06a..4b4478cf5 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -2257,6 +2257,7 @@ export class VariableValue extends CppClass { async atHCIndex(a1: number): Promise {return this.$callMethod('atHCIndex',a1);} async begin(): Promise {return this.$callMethod('begin');} async cancel(a1: boolean): Promise {return this.$callMethod('cancel',a1);} + async checkCancel(): Promise {return this.$callMethod('checkCancel');} async data(): Promise {return this.$callMethod('data');} async detailedText(...args: string[]): Promise {return this.$callMethod('detailedText',...args);} async enableSlider(...args: boolean[]): Promise {return this.$callMethod('enableSlider',...args);} @@ -2387,6 +2388,7 @@ export class civita__ITensor extends CppClass { async at(a1: number): Promise {return this.$callMethod('at',a1);} async atHCIndex(a1: number): Promise {return this.$callMethod('atHCIndex',a1);} async cancel(a1: boolean): Promise {return this.$callMethod('cancel',a1);} + async checkCancel(): Promise {return this.$callMethod('checkCancel');} async data(): Promise {return this.$callMethod('data');} async hypercube(...args: any[]): Promise {return this.$callMethod('hypercube',...args);} async imposeDimensions(a1: Container>): Promise {return this.$callMethod('imposeDimensions',a1);} @@ -2429,6 +2431,7 @@ export class civita__TensorVal extends CppClass { async atHCIndex(a1: number): Promise {return this.$callMethod('atHCIndex',a1);} async begin(): Promise {return this.$callMethod('begin');} async cancel(a1: boolean): Promise {return this.$callMethod('cancel',a1);} + async checkCancel(): Promise {return this.$callMethod('checkCancel');} async data(): Promise {return this.$callMethod('data');} async end(): Promise {return this.$callMethod('end');} async hypercube(...args: any[]): Promise {return this.$callMethod('hypercube',...args);} diff --git a/model/ravelWrap.cc b/model/ravelWrap.cc index 340466fa4..0eed0caab 100644 --- a/model/ravelWrap.cc +++ b/model/ravelWrap.cc @@ -292,7 +292,13 @@ namespace minsky populateHypercube(arg->hypercube()); return ravel::createRavelChain(getState(), arg); } - return {db.hyperSlice(wrappedRavel)}; + pack_t buf; buf< allSliceLabelsImpl(int axis, ravel::HandleSort::Order) const; ravelCAPI::Ravel wrappedRavel; + /// serialised copy of previous state for caching purposes + classdesc::pack_t lastState; + civita::TensorPtr cachedDbResult; ///< cache of database query result ravel::Op::ReductionOp m_nextReduction=ravel::Op::sum; public: static SVGRenderer svgRenderer; ///< SVG icon to display when not in editor mode From 3bb374dd27f802d7bf3911bf5a4871e06f36fc14 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Thu, 26 Jun 2025 15:43:39 +1000 Subject: [PATCH 24/41] Dead code removal. --- gui-js/libs/shared/src/lib/backend/minsky.ts | 39 ----- model/CSVDialog.cc | 150 ------------------- model/CSVDialog.h | 15 +- model/variable.cc | 7 - model/variable.h | 3 - test/00/RESTService.sh | 2 +- 6 files changed, 2 insertions(+), 214 deletions(-) diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index 4b4478cf5..c918279cb 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -345,61 +345,22 @@ export class BoundingBox extends CppClass { } export class CSVDialog extends CppClass { - backgroundColour: ecolab__cairo__Colour; - item: Item; spec: DataSpec; - wire: Wire; constructor(prefix: string){ super(prefix); - this.backgroundColour=new ecolab__cairo__Colour(this.$prefix()+'.backgroundColour'); - this.item=new Item(this.$prefix()+'.item'); this.spec=new DataSpec(this.$prefix()+'.spec'); - this.wire=new Wire(this.$prefix()+'.wire'); } async classifyColumns(): Promise {return this.$callMethod('classifyColumns');} - async colWidth(...args: number[]): Promise {return this.$callMethod('colWidth',...args);} - async columnOver(a1: number): Promise {return this.$callMethod('columnOver',a1);} - async controlMouseDown(a1: number,a2: number): Promise {return this.$callMethod('controlMouseDown',a1,a2);} async correctedUniqueValues(): Promise {return this.$callMethod('correctedUniqueValues');} - async destroyFrame(): Promise {return this.$callMethod('destroyFrame');} - async draw(): Promise {return this.$callMethod('draw');} - async flashNameRow(...args: boolean[]): Promise {return this.$callMethod('flashNameRow',...args);} - async frameArgs(): Promise {return this.$callMethod('frameArgs');} - async getItemAt(a1: number,a2: number): Promise {return this.$callMethod('getItemAt',a1,a2);} - async getWireAt(a1: number,a2: number): Promise {return this.$callMethod('getWireAt',a1,a2);} async guessSpecAndLoadFile(): Promise {return this.$callMethod('guessSpecAndLoadFile');} - async hasScrollBars(): Promise {return this.$callMethod('hasScrollBars');} - async init(): Promise {return this.$callMethod('init');} - async keyPress(a1: minsky__EventInterface__KeyPressArgs): Promise {return this.$callMethod('keyPress',a1);} async loadFile(): Promise {return this.$callMethod('loadFile');} async loadFileFromName(a1: string): Promise {return this.$callMethod('loadFileFromName',a1);} - async mouseDown(a1: number,a2: number): Promise {return this.$callMethod('mouseDown',a1,a2);} - async mouseMove(a1: number,a2: number): Promise {return this.$callMethod('mouseMove',a1,a2);} - async mouseUp(a1: number,a2: number): Promise {return this.$callMethod('mouseUp',a1,a2);} - async moveTo(a1: number,a2: number): Promise {return this.$callMethod('moveTo',a1,a2);} async numInitialLines(...args: number[]): Promise {return this.$callMethod('numInitialLines',...args);} async parseLines(a1: number): Promise {return this.$callMethod('parseLines',a1);} async populateHeader(a1: number): Promise {return this.$callMethod('populateHeader',a1);} async populateHeaders(): Promise {return this.$callMethod('populateHeaders');} - async position(): Promise {return this.$callMethod('position');} - async registerImage(): Promise {return this.$callMethod('registerImage');} - async renderFrame(a1: minsky__RenderNativeWindow__RenderFrameArgs): Promise {return this.$callMethod('renderFrame',a1);} - async renderToEMF(a1: string): Promise {return this.$callMethod('renderToEMF',a1);} - async renderToPDF(a1: string): Promise {return this.$callMethod('renderToPDF',a1);} - async renderToPNG(a1: string): Promise {return this.$callMethod('renderToPNG',a1);} - async renderToPS(a1: string): Promise {return this.$callMethod('renderToPS',a1);} - async renderToSVG(a1: string): Promise {return this.$callMethod('renderToSVG',a1);} - async reportDrawTime(a1: number): Promise {return this.$callMethod('reportDrawTime',a1);} async reportFromFile(a1: string,a2: string): Promise {return this.$callMethod('reportFromFile',a1,a2);} - async requestRedraw(): Promise {return this.$callMethod('requestRedraw');} - async resolutionScaleFactor(...args: any[]): Promise {return this.$callMethod('resolutionScaleFactor',...args);} - async rowOver(a1: number): Promise {return this.$callMethod('rowOver',a1);} - async scaleFactor(): Promise {return this.$callMethod('scaleFactor');} - async tableWidth(): Promise {return this.$callMethod('tableWidth');} async url(...args: string[]): Promise {return this.$callMethod('url',...args);} - async xoffs(...args: number[]): Promise {return this.$callMethod('xoffs',...args);} - async zoom(a1: number,a2: number,a3: number): Promise {return this.$callMethod('zoom',a1,a2,a3);} - async zoomFactor(): Promise {return this.$callMethod('zoomFactor');} } export class Canvas extends RenderNativeWindow { diff --git a/model/CSVDialog.cc b/model/CSVDialog.cc index 27f2394cd..5a686d02b 100644 --- a/model/CSVDialog.cc +++ b/model/CSVDialog.cc @@ -148,156 +148,6 @@ vector> parseLines(const Parser& parser, const vector& li return r; } -namespace -{ - struct CroppedPango: public Pango - { - cairo_t* cairo; - double w, x=0, y=0; - CroppedPango(cairo_t* cairo, double width): Pango(cairo), cairo(cairo), w(width) {} - void setxy(double xx, double yy) {x=xx; y=yy;} - void show() { - const CairoSave cs(cairo); - cairo_rectangle(cairo,x,y,w,height()); - cairo_clip(cairo); - cairo_move_to(cairo,x,y); - Pango::show(); - } - }; -} - -bool CSVDialog::redraw(int, int, int, int) -{ - cairo_t* cairo=surface->cairo(); - rowHeight=15; - vector> parsedLines=parseLines(); - - // LHS row labels - { - Pango pango(cairo); - pango.setText("Dimension"); - cairo_move_to(cairo,xoffs-pango.width()-5,0); - pango.show(); - pango.setText("Type"); - cairo_move_to(cairo,xoffs-pango.width()-5,rowHeight); - pango.show(); - pango.setText("Format"); - cairo_move_to(cairo,xoffs-pango.width()-5,2*rowHeight); - pango.show(); - if (flashNameRow) - pango.setMarkup("Name"); - else - pango.setText("Name"); - cairo_move_to(cairo,xoffs-pango.width()-5,3*rowHeight); - pango.show(); - pango.setText("Header"); - cairo_move_to(cairo,xoffs-pango.width()-5,(4+spec.headerRow)*rowHeight); - pango.show(); - - } - - CroppedPango pango(cairo, colWidth); - pango.setFontSize(0.8*rowHeight); - - set done; - double x=xoffs, y=0; - size_t col=0; - for (; done.size()(spec.dimensions[col].type)); - pango.setxy(x,y); - pango.show(); - } - y+=rowHeight; - if (spec.dimensionCols.contains(col) && colcairo(),0,0.7,0); - else - cairo_set_source_rgb(surface->cairo(),0,0,1); - else if (rowcairo(),1,0,0); - else if (colcairo(),0,0,1); - pango.show(); - } - else - done.insert(row); - y+=rowHeight; - } - { - const CairoSave cs(cairo); - cairo_set_source_rgb(cairo,.5,.5,.5); - cairo_move_to(cairo,x-2.5,0); - cairo_rel_line_to(cairo,0,(parsedLines.size()+4)*rowHeight); - cairo_stroke(cairo); - } - x+=colWidth+5; - y=0; - } - m_tableWidth=(col-1)*(colWidth+5); - for (size_t row=0; row> CSVDialog::parseLines(size_t maxColumn) { vector> parsedLines; diff --git a/model/CSVDialog.h b/model/CSVDialog.h index 1e3775c87..aae55c2eb 100644 --- a/model/CSVDialog.h +++ b/model/CSVDialog.h @@ -32,24 +32,16 @@ namespace minsky { - class CSVDialog: public RenderNativeWindow + class CSVDialog { std::vector initialLines; ///< initial lines of file - double rowHeight=0; - double m_tableWidth; CLASSDESC_ACCESS(DataSpec); - bool redraw(int, int, int width, int height) override; public: static const unsigned numInitialLines=100; - double xoffs=80; - double colWidth=50; - bool flashNameRow=false; DataSpec spec; /// filename, or web url std::string url; - /// width of table (in pixels) - double tableWidth() const {return m_tableWidth;} /// loads an initial sequence of lines from \a url. If fname /// contains "://", is is treated as a URL, and downloaded from @@ -60,11 +52,6 @@ namespace minsky /// common implementation of loading the initial sequence of lines void loadFileFromName(const std::string& fname); void reportFromFile(const std::string& input, const std::string& output) const; - void requestRedraw() {if (surface.get()) surface->requestRedraw();} - /// return column mouse is over - std::size_t columnOver(double x) const; - /// return row mouse is over - std::size_t rowOver(double y) const; std::vector > parseLines(size_t maxColumn=std::numeric_limits::max()); /// populate all column names from the headers row void populateHeaders(); diff --git a/model/variable.cc b/model/variable.cc index 3f098b847..6dc804bb1 100644 --- a/model/variable.cc +++ b/model/variable.cc @@ -482,13 +482,6 @@ void VariableBase::reloadCSV() loadValueFromCSVFile(*v, {v->csvDialog.url}, v->csvDialog.spec); } - -void VariableBase::destroyFrame() -{ - if (auto vv=vValue()) - vv->csvDialog.destroyFrame(); -} - void VariableBase::insertControlled(Selection& selection) { selection.ensureItemInserted(controller.lock()); diff --git a/model/variable.h b/model/variable.h index 2ac54e68d..3c4a1a30a 100644 --- a/model/variable.h +++ b/model/variable.h @@ -225,9 +225,6 @@ namespace minsky /// reload CSV file if previously imported void reloadCSV(); - /// clean up popup window structures on window close - void destroyFrame() override; - bool miniPlotEnabled() const {return bool(miniPlot);} bool miniPlotEnabled(bool); void resetMiniPlot(); diff --git a/test/00/RESTService.sh b/test/00/RESTService.sh index 9063f6007..b5919feea 100755 --- a/test/00/RESTService.sh +++ b/test/00/RESTService.sh @@ -9,7 +9,7 @@ EOF if [ $? -ne 0 ]; then fail; fi cat >reference <{"csvDialog":{"backgroundColour":{"a":1,"b":0.80000000000000004,"g":0.80000000000000004,"r":0.80000000000000004},"colWidth":50,"flashNameRow":false,"item":{},"spec":{"counter":false,"dataColOffset":0,"dataCols":[],"dataRowOffset":0,"decSeparator":".","dimensionCols":[],"dimensionNames":[],"dimensions":[],"dontFail":false,"duplicateKeyAction":"throwException","escape":"\u0000","headerRow":0,"horizontalDimName":"?","horizontalDimension":{"type":"string","units":""},"maxColumn":1000,"mergeDelimiters":false,"missingValue":NaN,"numCols":0,"quote":"\"","separator":","},"url":"","wire":{},"xoffs":80},"detailedText":"","enableSlider":true,"godleyOverridden":false,"name":"constant:one","rhs":{},"sliderMax":-1.7976931348623157e+308,"sliderMin":1.7976931348623157e+308,"sliderStep":0,"sliderStepRel":false,"tensorInit":{},"tooltip":"","units":[],"unitsCached":false} +/minsky/variableValues/@elem/"constant:one"=>{"csvDialog":{"spec":{"counter":false,"dataColOffset":0,"dataCols":[],"dataRowOffset":0,"decSeparator":".","dimensionCols":[],"dimensionNames":[],"dimensions":[],"dontFail":false,"duplicateKeyAction":"throwException","escape":"\u0000","headerRow":0,"horizontalDimName":"?","horizontalDimension":{"type":"string","units":""},"maxColumn":1000,"mergeDelimiters":false,"missingValue":NaN,"numCols":0,"quote":"\"","separator":","},"url":""},"detailedText":"","enableSlider":true,"godleyOverridden":false,"name":"constant:one","rhs":{},"sliderMax":-1.7976931348623157e+308,"sliderMin":1.7976931348623157e+308,"sliderStep":0,"sliderStepRel":false,"tensorInit":{},"tooltip":"","units":[],"unitsCached":false} EOF diff -q -w output reference From 986dd3947168dc3b29e70622f75126c8fbcd393f Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Fri, 27 Jun 2025 12:45:58 +1000 Subject: [PATCH 25/41] Refactored CSVDialog to be a polymorphic interface, to detach importCSV from parameters. --- RESTService/typescriptAPI.cc | 1 + engine/variableValue.cc | 10 +++ engine/variableValue.h | 8 +-- .../minsky-electron/src/app/backend-init.ts | 12 ++-- .../src/app/events/electron.events.ts | 5 +- .../app/managers/ApplicationMenuManager.ts | 4 +- .../src/app/managers/CommandsManager.ts | 9 ++- .../src/app/managers/ContextMenuManager.ts | 22 +++--- .../communication/communication.service.ts | 7 +- gui-js/libs/shared/src/lib/backend/minsky.ts | 17 ++++- .../lib/import-csv/import-csv.component.ts | 70 ++++++++----------- model/CSVDialog.h | 2 + model/variable.cc | 13 ++-- model/variable.h | 1 + schema/schema3.cc | 4 +- schema/schema3.h | 6 +- test/00/RESTService.sh | 2 +- test/00/importCSV.sh | 2 +- test/testCSVParser.cc | 7 +- test/testCSVParserGemini.cc | 7 +- 20 files changed, 116 insertions(+), 93 deletions(-) diff --git a/RESTService/typescriptAPI.cc b/RESTService/typescriptAPI.cc index eabc65c03..c6b28cd95 100644 --- a/RESTService/typescriptAPI.cc +++ b/RESTService/typescriptAPI.cc @@ -252,6 +252,7 @@ int main() api.addClass(); api.addClass(); api.addClass(); + api.addClass(); api.addClass(); api.addClass(); api.addClass(); diff --git a/engine/variableValue.cc b/engine/variableValue.cc index bd93c9aac..fe123990e 100644 --- a/engine/variableValue.cc +++ b/engine/variableValue.cc @@ -85,6 +85,16 @@ namespace minsky static VariableValuePtr s_one(make_shared("constant:one","1")); return s_one; } + + void VariableValue::importFromCSV(const std::vector& filenames) + { + if (!filenames.empty()) + url=filenames[0]; + loadValueFromCSVFile(*this, filenames, spec); + minsky().populateMissingDimensionsFromVariable(*this); + if (!hypercube().dimsAreDistinct()) + throw runtime_error("Axes of imported data should all have distinct names"); + } bool VariableValue::idxInRange() const {return m_type==undefined || idx()+size()<= diff --git a/engine/variableValue.h b/engine/variableValue.h index dca5b089e..7bd74a0e7 100644 --- a/engine/variableValue.h +++ b/engine/variableValue.h @@ -41,7 +41,7 @@ namespace minsky typedef std::shared_ptr GroupPtr; using namespace civita; - struct VariableValueData: public civita::ITensorVal, public Slider + struct VariableValueData: public civita::ITensorVal, public Slider, public CSVDialog { using ITensorVal::operator=; @@ -61,10 +61,6 @@ namespace minsky bool godleyOverridden=false; std::string name; // name of this variable classdesc::Exclude> m_scope; - - /// for importing CSV files - CSVDialog csvDialog; - }; class VariableValue: public VariableType, public VariableValueData @@ -177,6 +173,8 @@ namespace minsky std::string valueId() const {return valueIdFromScope(m_scope.lock(),canonicalName(name));} + void importFromCSV(const std::vector&) override; + /// export this to a CSV file. If \a comment is non-empty, it is written as the first line of the file. If tabular is false, there is one data point per line, if true, the the longest dimension is written as a series on the line. void exportAsCSV(const std::string& filename, const std::string& comment="", bool tabular=false) const; diff --git a/gui-js/apps/minsky-electron/src/app/backend-init.ts b/gui-js/apps/minsky-electron/src/app/backend-init.ts index 09ad2f4ef..38dd506be 100644 --- a/gui-js/apps/minsky-electron/src/app/backend-init.ts +++ b/gui-js/apps/minsky-electron/src/app/backend-init.ts @@ -62,12 +62,12 @@ export async function backend(command: string, ...args: any[]): Promise { if (typeof(error)!=="string") error=error?.message; log.error('Async Rest API: ',command,arg,'=>Exception caught: ' + error); if (!dialog) throw error; // rethrow to force error in jest environment - if (error && command !== 'minsky.canvas.item.importFromCSV') - dialog.showMessageBoxSync(WindowManager.getMainWindow(),{ - message: error, - type: 'error', - }); - return error; + if (error && !command.endsWith('importFromCSV')) + dialog.showMessageBoxSync(WindowManager.getMainWindow(),{ + message: error, + type: 'error', + }); + return error; } } diff --git a/gui-js/apps/minsky-electron/src/app/events/electron.events.ts b/gui-js/apps/minsky-electron/src/app/events/electron.events.ts index b1a93a3bc..aade25e6b 100644 --- a/gui-js/apps/minsky-electron/src/app/events/electron.events.ts +++ b/gui-js/apps/minsky-electron/src/app/events/electron.events.ts @@ -17,6 +17,7 @@ import { ImportStockPayload, GodleyIcon, DownloadCSVPayload, + VariableBase, } from '@minsky/shared'; import { BrowserWindow, dialog, ipcMain } from 'electron'; import { BookmarkManager } from '../managers/BookmarkManager'; @@ -215,8 +216,8 @@ ipcMain.handle(events.NEW_SYSTEM, async () => { ipcMain.handle( events.IMPORT_CSV, async (event) => { - const itemInfo = await CommandsManager.getFocusItemInfo(); - CommandsManager.importCSV(itemInfo, true); + let v=new VariableBase(minsky.canvas.itemFocus); + CommandsManager.importCSV(minsky.variableValues.elem(await v.valueId()), true); return; } ); diff --git a/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts b/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts index 8d139cc4c..032e50741 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts @@ -3,6 +3,7 @@ import { importCSVvariableName, InstallCase, minsky, + VariableBase, } from '@minsky/shared'; import { dialog, @@ -206,7 +207,8 @@ export class ApplicationMenuManager { label: 'Import Data', async click() { minsky.canvas.addVariable(importCSVvariableName, 'parameter'); - CommandsManager.importCSV(await CommandsManager.getFocusItemInfo(), true); + let v=new VariableBase(minsky.canvas.itemFocus); + CommandsManager.importCSV(minsky.variableValues.elem(await v.valueId()), true); } }, { diff --git a/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts b/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts index acca5a08e..cbc942a52 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts @@ -1,6 +1,7 @@ import { CanvasItem, ClassType, + CSVDialog, events, Functions, HandleDimensionPayload, @@ -912,11 +913,13 @@ export class CommandsManager { this.setLogSimulationCheckmark(true); } - static async importCSV(itemInfo: CanvasItem, isInvokedUsingToolbar = false) { + static async importCSV(csvDialog: CSVDialog, isInvokedUsingToolbar = false) { + const itemInfo: CanvasItem={classType: ClassType.Variable, id: csvDialog.$prefix(), displayContents: false}; if (!WindowManager.focusIfWindowIsPresent(itemInfo.id)) { + const window = await this.initializePopupWindow({ itemInfo, - url: `#/headless/import-csv?systemWindowId=0&itemId=${itemInfo.id}&isInvokedUsingToolbar=${isInvokedUsingToolbar}`, + url: `#/headless/import-csv?systemWindowId=0&csvDialog=${csvDialog.$prefix()}&isInvokedUsingToolbar=${isInvokedUsingToolbar}`, height: 600, width: 1300, minWidth: 650, @@ -942,7 +945,7 @@ export class CommandsManager { window.loadURL( WindowManager.getWindowUrl( - `#/headless/import-csv?systemWindowId=${systemWindowId}&itemId=${itemInfo.id}&isInvokedUsingToolbar=${isInvokedUsingToolbar}&examplesPath=${join(dirname(app.getAppPath()),'examples','data')}` + `#/headless/import-csv?systemWindowId=${systemWindowId}&csvDialog=${csvDialog.$prefix()}&isInvokedUsingToolbar=${isInvokedUsingToolbar}&examplesPath=${join(dirname(app.getAppPath()),'examples','data')}` ) ); } diff --git a/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts b/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts index 17467f50e..735007847 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts @@ -1175,8 +1175,8 @@ export class ContextMenuManager { menuItems.push( new MenuItem({ label: 'Import CSV', - click: () => { - CommandsManager.importCSV(itemInfo); + click: async () => { + CommandsManager.importCSV(minsky.variableValues.elem(await v.valueId())); }, }) ); @@ -1381,51 +1381,51 @@ export class ContextMenuManager { new MenuItem({ label: 'Set as header row', click: ()=>{ - value.csvDialog.spec.headerRow(row); + value.spec.headerRow(row); refresh(); }, }), new MenuItem({ label: 'Auto-classify columns as axis/data', click: async ()=>{ - value.csvDialog.classifyColumns(); + value.classifyColumns(); refresh(); }, }), new MenuItem({ label: 'Populate column labels', click: async ()=>{ - value.csvDialog.populateHeaders(); + value.populateHeaders(); refresh(); }, }), new MenuItem({ label: 'Populate current column label', click: ()=>{ - value.csvDialog.populateHeader(col); + value.populateHeader(col); refresh(); }, }), new MenuItem({ label: 'Set start of data row, and column', click: ()=>{ - value.csvDialog.spec.setDataArea(row,col); + value.spec.setDataArea(row,col); refresh(); }, }), new MenuItem({ label: 'Set start of data row', click: async ()=>{ - let c=await value.csvDialog.spec.nColAxes(); - value.csvDialog.spec.setDataArea(row,c); + let c=await value.spec.nColAxes(); + value.spec.setDataArea(row,c); refresh(); }, }), new MenuItem({ label: 'Set start of data column', click: async ()=>{ - let r=await value.csvDialog.spec.nRowAxes(); - value.csvDialog.spec.setDataArea(r,col); + let r=await value.spec.nRowAxes(); + value.spec.setDataArea(r,col); refresh(); }, }), diff --git a/gui-js/libs/core/src/lib/services/communication/communication.service.ts b/gui-js/libs/core/src/lib/services/communication/communication.service.ts index c43a64c55..bdd72a551 100644 --- a/gui-js/libs/core/src/lib/services/communication/communication.service.ts +++ b/gui-js/libs/core/src/lib/services/communication/communication.service.ts @@ -428,12 +428,7 @@ export class CommunicationService { async importData() { this.electronService.minsky.canvas.addVariable(importCSVvariableName, 'parameter'); - - const payload: MinskyProcessPayload = { - mouseX: await this.electronService.minsky.canvas.itemFocus.x(), - mouseY: await this.electronService.minsky.canvas.itemFocus.y(), - }; - this.electronService.invoke(events.IMPORT_CSV, payload); + this.electronService.invoke(events.IMPORT_CSV); } resetScrollTimeout = () => { diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index c918279cb..f2c458e7f 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -353,6 +353,7 @@ export class CSVDialog extends CppClass { async classifyColumns(): Promise {return this.$callMethod('classifyColumns');} async correctedUniqueValues(): Promise {return this.$callMethod('correctedUniqueValues');} async guessSpecAndLoadFile(): Promise {return this.$callMethod('guessSpecAndLoadFile');} + async importFromCSV(a1: string[]): Promise {return this.$callMethod('importFromCSV',a1);} async loadFile(): Promise {return this.$callMethod('loadFile');} async loadFileFromName(a1: string): Promise {return this.$callMethod('loadFileFromName',a1);} async numInitialLines(...args: number[]): Promise {return this.$callMethod('numInitialLines',...args);} @@ -2201,14 +2202,14 @@ export class VariablePaneCell extends CppClass { } export class VariableValue extends CppClass { - csvDialog: CSVDialog; rhs: civita__ITensor; + spec: DataSpec; tensorInit: civita__TensorVal; units: Units; constructor(prefix: string){ super(prefix); - this.csvDialog=new CSVDialog(this.$prefix()+'.csvDialog'); this.rhs=new civita__ITensor(this.$prefix()+'.rhs'); + this.spec=new DataSpec(this.$prefix()+'.spec'); this.tensorInit=new civita__TensorVal(this.$prefix()+'.tensorInit'); this.units=new Units(this.$prefix()+'.units'); } @@ -2219,15 +2220,19 @@ export class VariableValue extends CppClass { async begin(): Promise {return this.$callMethod('begin');} async cancel(a1: boolean): Promise {return this.$callMethod('cancel',a1);} async checkCancel(): Promise {return this.$callMethod('checkCancel');} + async classifyColumns(): Promise {return this.$callMethod('classifyColumns');} + async correctedUniqueValues(): Promise {return this.$callMethod('correctedUniqueValues');} async data(): Promise {return this.$callMethod('data');} async detailedText(...args: string[]): Promise {return this.$callMethod('detailedText',...args);} async enableSlider(...args: boolean[]): Promise {return this.$callMethod('enableSlider',...args);} async end(): Promise {return this.$callMethod('end');} async exportAsCSV(a1: string,a2: string,a3: boolean): Promise {return this.$callMethod('exportAsCSV',a1,a2,a3);} async godleyOverridden(...args: boolean[]): Promise {return this.$callMethod('godleyOverridden',...args);} + async guessSpecAndLoadFile(): Promise {return this.$callMethod('guessSpecAndLoadFile');} async hypercube(...args: any[]): Promise {return this.$callMethod('hypercube',...args);} async idx(): Promise {return this.$callMethod('idx');} async idxInRange(): Promise {return this.$callMethod('idxInRange');} + async importFromCSV(a1: string[]): Promise {return this.$callMethod('importFromCSV',a1);} async imposeDimensions(a1: Container>): Promise {return this.$callMethod('imposeDimensions',a1);} async incrSlider(a1: number): Promise {return this.$callMethod('incrSlider',a1);} async index(...args: any[]): Promise {return this.$callMethod('index',...args);} @@ -2235,9 +2240,16 @@ export class VariableValue extends CppClass { async isFlowVar(): Promise {return this.$callMethod('isFlowVar');} async isZero(): Promise {return this.$callMethod('isZero');} async lhs(): Promise {return this.$callMethod('lhs');} + async loadFile(): Promise {return this.$callMethod('loadFile');} + async loadFileFromName(a1: string): Promise {return this.$callMethod('loadFileFromName',a1);} async maxSliderSteps(): Promise {return this.$callMethod('maxSliderSteps');} async name(...args: string[]): Promise {return this.$callMethod('name',...args);} + async numInitialLines(...args: number[]): Promise {return this.$callMethod('numInitialLines',...args);} + async parseLines(a1: number): Promise {return this.$callMethod('parseLines',a1);} + async populateHeader(a1: number): Promise {return this.$callMethod('populateHeader',a1);} + async populateHeaders(): Promise {return this.$callMethod('populateHeaders');} async rank(): Promise {return this.$callMethod('rank');} + async reportFromFile(a1: string,a2: string): Promise {return this.$callMethod('reportFromFile',a1,a2);} async reset_idx(): Promise {return this.$callMethod('reset_idx');} async setArgument(a1: civita__ITensor,a2: civita__ITensor__Args): Promise {return this.$callMethod('setArgument',a1,a2);} async setArguments(...args: any[]): Promise {return this.$callMethod('setArguments',...args);} @@ -2256,6 +2268,7 @@ export class VariableValue extends CppClass { async type(): Promise {return this.$callMethod('type');} async typeName(a1: number): Promise {return this.$callMethod('typeName',a1);} async unitsCached(...args: boolean[]): Promise {return this.$callMethod('unitsCached',...args);} + async url(...args: string[]): Promise {return this.$callMethod('url',...args);} async value(): Promise {return this.$callMethod('value');} async valueAt(a1: number): Promise {return this.$callMethod('valueAt',a1);} async valueId(): Promise {return this.$callMethod('valueId');} diff --git a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts index 7cc8f127d..b81c675e5 100644 --- a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts +++ b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts @@ -3,6 +3,7 @@ import { AbstractControl, FormControl, FormGroup, FormsModule, ReactiveFormsModu import { ActivatedRoute } from '@angular/router'; import { ElectronService } from '@minsky/core'; import { + CSVDialog, dateTimeFormats, events, importCSVvariableName, @@ -17,6 +18,7 @@ import { NgIf, NgFor, NgStyle } from '@angular/common'; import { MatOptionModule } from '@angular/material/core'; import { MatAutocompleteModule } from '@angular/material/autocomplete'; import { MatButtonModule } from '@angular/material/button'; +import JSON5 from 'json5'; enum ColType { axis = "axis", @@ -92,12 +94,10 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni fileLoaded = false; - itemId: string; systemWindowId: number; isInvokedUsingToolbar: boolean; examplesPath: string; - valueId: string; - variableValuesSubCommand: VariableValue; + csvDialog: CSVDialog; timeFormatStrings = dateTimeFormats; parsedLines: string[][] = []; csvCols: any[]; @@ -187,7 +187,8 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni ) { super(); this.route.queryParams.pipe(takeUntil(this.destroy$)).subscribe((params) => { - this.itemId = params.itemId; + electronService.log(JSON5.stringify(params)); + this.csvDialog = new CSVDialog(params.csvDialog); this.systemWindowId = params.systemWindowId; this.isInvokedUsingToolbar = params.isInvokedUsingToolbar; this.examplesPath = params.examplesPath; @@ -239,10 +240,6 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni ngAfterViewInit() { (async () => { - this.valueId = await this.getValueId(); - this.variableValuesSubCommand = this.electronService.minsky.variableValues.elem(this.valueId); - - await this.getCSVDialogSpec(); this.updateForm(); this.load(2); @@ -289,9 +286,9 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni }); } - async getValueId() { - return new VariableBase(this.electronService.minsky.namedItems.elem(this.itemId)).valueId(); - } +// async getValueId() { +// return new VariableBase(this.electronService.minsky.namedItems.elem(this.itemId)).valueId(); +// } async selectFile(defaultPath: string = '') { let options: OpenDialogOptions = { @@ -324,20 +321,20 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni this.setParameterNameFromUrl(); if (this.url.value.includes('://')) { - const savePath = await this.electronService.downloadCSV({ windowUid: this.itemId, url: this.url.value }); + const savePath = await this.electronService.downloadCSV({ windowUid: this.csvDialog.$prefix(), url: this.url.value }); this.url.setValue(savePath); this.files = [savePath]; } - const fileUrlOnServer = await this.variableValuesSubCommand.csvDialog.url(); + const fileUrlOnServer = await this.csvDialog.url(); if (this.url.value !== fileUrlOnServer) { - await this.variableValuesSubCommand.csvDialog.url(this.url.value); - await this.variableValuesSubCommand.csvDialog.guessSpecAndLoadFile(); + await this.csvDialog.url(this.url.value); + await this.csvDialog.guessSpecAndLoadFile(); await this.getCSVDialogSpec(); this.updateForm(); } else { - await this.variableValuesSubCommand.csvDialog.loadFile(); + await this.csvDialog.loadFile(); } await this.parseLines(); @@ -359,9 +356,9 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni } async getCSVDialogSpec() { - this.variableValuesSubCommand.csvDialog.spec.toSchema(); - this.dialogState = await this.variableValuesSubCommand.csvDialog.$properties() as Record; - this.uniqueValues = await this.variableValuesSubCommand.csvDialog.correctedUniqueValues(); + this.csvDialog.spec.toSchema(); + this.dialogState = await this.csvDialog.$properties() as Record; + this.uniqueValues = await this.csvDialog.correctedUniqueValues(); } updateColumnTypes() { @@ -384,7 +381,7 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni } async parseLines() { - this.parsedLines = await this.variableValuesSubCommand.csvDialog.parseLines(this.dialogState.spec.maxColumn) as string[][]; + this.parsedLines = await this.csvDialog.parseLines(this.dialogState.spec.maxColumn) as string[][]; await this.getCSVDialogSpec(); let header = this.dialogState.spec.headerRow; @@ -413,7 +410,7 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni onSeparatorChange() { this.updateSpecFromForm(); - this.variableValuesSubCommand.csvDialog.spec.$properties(this.dialogState.spec); + this.csvDialog.spec.$properties(this.dialogState.spec); this.parseLines(); } @@ -553,9 +550,8 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni if (this.dialogState.spec.dataCols.length === 0) this.dialogState.spec.counter = true; - let v = new VariableBase(this.electronService.minsky.canvas.item); // returns an error message on error - const res = await v.importFromCSV(this.files, this.dialogState.spec) as unknown as string; + const res = await this.csvDialog.importFromCSV(this.files) as unknown as string; if (typeof res === 'string') { const positiveResponseText = 'Yes'; @@ -575,20 +571,16 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni return; } - const currentItemId = await v.id(); - const currentItemName = await v.name(); - - if ( - this.isInvokedUsingToolbar && - currentItemId === this.itemId && - currentItemName === importCSVvariableName && - this.parameterName.value - ) { - await this.electronService.minsky.canvas.renameItem(this.parameterName.value); - v.tooltip(this.shortDescription.value); - v.detailedText(this.detailedDescription.value); + if (this.isInvokedUsingToolbar && this.parameterName.value) { + // rename variable if newly added variable is still focussed + let v=new VariableBase(this.electronService.minsky.canvas.itemFocus); + let vv=new VariableValue(this.csvDialog.$prefix()); + if (await v?.valueId()===await vv?.valueId()) { + v.tooltip(this.shortDescription.value); + v.detailedText(this.detailedDescription.value); + v.name(this.parameterName.value); + } } - this.closeWindow(); } @@ -605,19 +597,19 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni }); if (!filePath) return; - await this.variableValuesSubCommand.csvDialog.reportFromFile(this.url.value, filePath); + await this.csvDialog.reportFromFile(this.url.value, filePath); return; } async contextMenu(row: number, col: number) { // update C++ spec with current state this.updateSpecFromForm(); - await this.variableValuesSubCommand.csvDialog.spec.$properties(this.dialogState.spec); + await this.csvDialog.spec.$properties(this.dialogState.spec); this.electronService.send(events.CONTEXT_MENU, { x: row, y: col, type: 'csv-import', - command: this.variableValuesSubCommand.$prefix(), + command: this.csvDialog.$prefix(), }); } diff --git a/model/CSVDialog.h b/model/CSVDialog.h index aae55c2eb..ef56d6f39 100644 --- a/model/CSVDialog.h +++ b/model/CSVDialog.h @@ -63,6 +63,8 @@ namespace minsky /// could slightly underestimate the value, and is never less than /// 1, even for empty columns std::vector correctedUniqueValues(); + /// import names CSV files using spec above + virtual void importFromCSV(const std::vector& filenames)=0; }; bool isNumerical(const std::string& s); diff --git a/model/variable.cc b/model/variable.cc index 6dc804bb1..7f2b888e3 100644 --- a/model/variable.cc +++ b/model/variable.cc @@ -464,13 +464,8 @@ void VariableBase::exportAsCSV(const std::string& filename, bool tabular) const void VariableBase::importFromCSV(const vector& filenames, const DataSpecSchema& spec) const { if (auto v=vValue()) { - v->csvDialog.spec=spec; - if (!filenames.empty()) - v->csvDialog.url=filenames[0]; - loadValueFromCSVFile(*v, filenames, v->csvDialog.spec); - minsky().populateMissingDimensionsFromVariable(*v); - if (!v->hypercube().dimsAreDistinct()) - throw_error("Axes of imported data should all have distinct names"); + v->spec=spec; + v->importFromCSV(filenames); } } @@ -478,8 +473,8 @@ void VariableBase::importFromCSV(const vector& filenames, const DataSpec void VariableBase::reloadCSV() { if (auto v=vValue()) - if (!v->csvDialog.url.empty()) - loadValueFromCSVFile(*v, {v->csvDialog.url}, v->csvDialog.spec); + if (!v->url.empty()) + loadValueFromCSVFile(*v, {v->url}, v->spec); } void VariableBase::insertControlled(Selection& selection) diff --git a/model/variable.h b/model/variable.h index 3c4a1a30a..033b85c89 100644 --- a/model/variable.h +++ b/model/variable.h @@ -219,6 +219,7 @@ namespace minsky /// export this variable as a CSV file /// @param tabular - if true, the longest dimension is split across columns as a horizontal dimension void exportAsCSV(const std::string& filename, bool tabular) const; + /// \deprecated To be removed in version 4. /// import CSV files, using \a spec void importFromCSV(const std::vector& filenames, const DataSpecSchema& spec) const; diff --git a/schema/schema3.cc b/schema/schema3.cc index cbde82bdc..d2c204c1b 100644 --- a/schema/schema3.cc +++ b/schema/schema3.cc @@ -827,9 +827,9 @@ namespace schema3 if (auto val=v->vValue()) { if (i.second.csvDataSpec) - val->csvDialog.spec=*i.second.csvDataSpec; + val->spec=*i.second.csvDataSpec; if (i.second.url) - val->csvDialog.url=*i.second.url; + val->url=*i.second.url; if (i.second.tensorData) { auto buf=minsky::decode(*i.second.tensorData); diff --git a/schema/schema3.h b/schema/schema3.h index 6f3357a62..35b461797 100644 --- a/schema/schema3.h +++ b/schema/schema3.h @@ -148,9 +148,9 @@ namespace schema3 if (auto vv=v.vValue()) { units=vv->units.str(); - if (!vv->csvDialog.url.empty()) - csvDataSpec=vv->csvDialog.spec.toSchema(); - url=vv->csvDialog.url; + if (!vv->url.empty()) + csvDataSpec=vv->spec.toSchema(); + url=vv->url; } } Item(int id, const minsky::OperationBase& o, const std::vector& ports): diff --git a/test/00/RESTService.sh b/test/00/RESTService.sh index b5919feea..5bfae404c 100755 --- a/test/00/RESTService.sh +++ b/test/00/RESTService.sh @@ -9,7 +9,7 @@ EOF if [ $? -ne 0 ]; then fail; fi cat >reference <{"csvDialog":{"spec":{"counter":false,"dataColOffset":0,"dataCols":[],"dataRowOffset":0,"decSeparator":".","dimensionCols":[],"dimensionNames":[],"dimensions":[],"dontFail":false,"duplicateKeyAction":"throwException","escape":"\u0000","headerRow":0,"horizontalDimName":"?","horizontalDimension":{"type":"string","units":""},"maxColumn":1000,"mergeDelimiters":false,"missingValue":NaN,"numCols":0,"quote":"\"","separator":","},"url":""},"detailedText":"","enableSlider":true,"godleyOverridden":false,"name":"constant:one","rhs":{},"sliderMax":-1.7976931348623157e+308,"sliderMin":1.7976931348623157e+308,"sliderStep":0,"sliderStepRel":false,"tensorInit":{},"tooltip":"","units":[],"unitsCached":false} +/minsky/variableValues/@elem/"constant:one"=>{"detailedText":"","enableSlider":true,"godleyOverridden":false,"name":"constant:one","rhs":{},"sliderMax":-1.7976931348623157e+308,"sliderMin":1.7976931348623157e+308,"sliderStep":0,"sliderStepRel":false,"spec":{"counter":false,"dataColOffset":0,"dataCols":[],"dataRowOffset":0,"decSeparator":".","dimensionCols":[],"dimensionNames":[],"dimensions":[],"dontFail":false,"duplicateKeyAction":"throwException","escape":"\u0000","headerRow":0,"horizontalDimName":"?","horizontalDimension":{"type":"string","units":""},"maxColumn":1000,"mergeDelimiters":false,"missingValue":NaN,"numCols":0,"quote":"\"","separator":","},"tensorInit":{},"tooltip":"","units":[],"unitsCached":false,"url":""} EOF diff -q -w output reference diff --git a/test/00/importCSV.sh b/test/00/importCSV.sh index 503646c0f..a73b8230a 100755 --- a/test/00/importCSV.sh +++ b/test/00/importCSV.sh @@ -7,7 +7,7 @@ cat >input.py <&) override {} + }; + + TEST_FIXTURE(TestCSVDialog,classifyColumns) { string input="10,2022/10/2,hello,\n" "'5,150,000','2023/1/3','foo bar',\n" diff --git a/test/testCSVParserGemini.cc b/test/testCSVParserGemini.cc index 450a73bdb..ce7d4802d 100644 --- a/test/testCSVParserGemini.cc +++ b/test/testCSVParserGemini.cc @@ -42,10 +42,15 @@ class CSVParserTest : public ::testing::TestWithParam { DataSpec spec; }; +struct TestCSVDialog: CSVDialog +{ + void importFromCSV(const std::vector&) override {} +}; + // Fixture for CSVDialog tests class CSVDialogTest : public ::testing::Test { protected: - CSVDialog dialog; + TestCSVDialog dialog; DataSpec& spec = dialog.spec; // Access DataSpec through CSVDialog string url; From 6c046ceb2b140baa1b7a083a4b2a2ab32233cfce Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Fri, 27 Jun 2025 17:18:50 +1000 Subject: [PATCH 26/41] Added databaseIngestor class. Still needs debugging work. --- Makefile | 2 +- RESTService/typescriptAPI.cc | 1 + engine/databaseIngestor.cc | 13 +++++ engine/databaseIngestor.h | 39 +++++++++++++ gui-js/libs/shared/src/lib/backend/minsky.ts | 24 ++++++++ loadDb.py | 59 ++++++++++++++------ model/minsky.cc | 3 + model/minsky.h | 4 +- 8 files changed, 126 insertions(+), 19 deletions(-) create mode 100644 engine/databaseIngestor.cc create mode 100644 engine/databaseIngestor.h diff --git a/Makefile b/Makefile index bf14c8a6a..3f075ba63 100644 --- a/Makefile +++ b/Makefile @@ -162,7 +162,7 @@ PREFIX=/usr/local # directory MODLINK=$(LIBMODS:%=$(ECOLAB_HOME)/lib/%) MODEL_OBJS=autoLayout.o cairoItems.o canvas.o CSVDialog.o dataOp.o equationDisplay.o godleyIcon.o godleyTable.o godleyTableWindow.o grid.o group.o item.o intOp.o lasso.o lock.o minsky.o operation.o operationRS.o operationRS1.o operationRS2.o phillipsDiagram.o plotWidget.o port.o pubTab.o ravelWrap.o renderNativeWindow.o selection.o sheet.o SVGItem.o switchIcon.o userFunction.o userFunction_units.o variableInstanceList.o variable.o variablePane.o windowInformation.o wire.o -ENGINE_OBJS=clipboard.o derivative.o equationDisplayRender.o equations.o evalGodley.o evalOp.o flowCoef.o \ +ENGINE_OBJS=clipboard.o databaseIngestor.o derivative.o equationDisplayRender.o equations.o evalGodley.o evalOp.o flowCoef.o \ godleyExport.o latexMarkup.o valueId.o variableValue.o node_latex.o node_matlab.o CSVParser.o \ minskyTensorOps.o mdlReader.o saver.o rungeKutta.o SCHEMA_OBJS=schema3.o schema2.o schema1.o schema0.o schemaHelper.o variableType.o \ diff --git a/RESTService/typescriptAPI.cc b/RESTService/typescriptAPI.cc index c6b28cd95..454ecc496 100644 --- a/RESTService/typescriptAPI.cc +++ b/RESTService/typescriptAPI.cc @@ -19,6 +19,7 @@ #include "dataSpecSchema.tcd" #include "dataOp.h" #include "dataOp.tcd" +#include "databaseIngestor.tcd" #include "dimension.tcd" #include "dynamicRavelCAPI.tcd" #include "engNotation.tcd" diff --git a/engine/databaseIngestor.cc b/engine/databaseIngestor.cc new file mode 100644 index 000000000..4665f11ba --- /dev/null +++ b/engine/databaseIngestor.cc @@ -0,0 +1,13 @@ +#include "databaseIngestor.h" +#include "databaseIngestor.cd" +#include "databaseIngestor.xcd" +#include "databaseIngestor.rcd" +#include "minsky_epilogue.h" + +namespace minsky +{ + void DatabaseIngestor::importFromCSV(const std::vector& filenames) + { + db.loadDatabase(filenames, spec); + } +} diff --git a/engine/databaseIngestor.h b/engine/databaseIngestor.h new file mode 100644 index 000000000..d256acf85 --- /dev/null +++ b/engine/databaseIngestor.h @@ -0,0 +1,39 @@ +/* + @copyright Steve Keen 2025 + @author Russell Standish + This file is part of Minsky. + + Minsky is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Minsky is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Minsky. If not, see . +*/ + + +#ifndef DATABASEINGESTOR_H +#define DATABASEINGESTOR_H + +#include "CSVDialog.h" +#include "dynamicRavelCAPI.h" + +namespace minsky +{ + class DatabaseIngestor: public CSVDialog + { + public: + ravelCAPI::Database db; + void createTable(const std::string& filename) + {db.createTable(filename,spec);} + void importFromCSV(const std::vector&) override; + }; +} + +#endif diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index f2c458e7f..67512be5f 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -564,6 +564,28 @@ export class DataSpecSchema extends CppClass { async separator(...args: number[]): Promise {return this.$callMethod('separator',...args);} } +export class DatabaseIngestor extends CppClass { + db: ravelCAPI__Database; + spec: DataSpec; + constructor(prefix: string){ + super(prefix); + this.db=new ravelCAPI__Database(this.$prefix()+'.db'); + this.spec=new DataSpec(this.$prefix()+'.spec'); + } + async classifyColumns(): Promise {return this.$callMethod('classifyColumns');} + async correctedUniqueValues(): Promise {return this.$callMethod('correctedUniqueValues');} + async guessSpecAndLoadFile(): Promise {return this.$callMethod('guessSpecAndLoadFile');} + async importFromCSV(a1: string[]): Promise {return this.$callMethod('importFromCSV',a1);} + async loadFile(): Promise {return this.$callMethod('loadFile');} + async loadFileFromName(a1: string): Promise {return this.$callMethod('loadFileFromName',a1);} + async numInitialLines(...args: number[]): Promise {return this.$callMethod('numInitialLines',...args);} + async parseLines(a1: number): Promise {return this.$callMethod('parseLines',a1);} + async populateHeader(a1: number): Promise {return this.$callMethod('populateHeader',a1);} + async populateHeaders(): Promise {return this.$callMethod('populateHeaders');} + async reportFromFile(a1: string,a2: string): Promise {return this.$callMethod('reportFromFile',a1,a2);} + async url(...args: string[]): Promise {return this.$callMethod('url',...args);} +} + export class EngNotation extends CppClass { constructor(prefix: string){ super(prefix); @@ -1229,6 +1251,7 @@ export class Lock extends Item { export class Minsky extends CppClass { canvas: Canvas; conversions: civita__Conversions; + databaseIngestor: DatabaseIngestor; dimensions: Map; equationDisplay: EquationDisplay; evalGodley: EvalGodley; @@ -1250,6 +1273,7 @@ export class Minsky extends CppClass { super(prefix); this.canvas=new Canvas(this.$prefix()+'.canvas'); this.conversions=new civita__Conversions(this.$prefix()+'.conversions'); + this.databaseIngestor=new DatabaseIngestor(this.$prefix()+'.databaseIngestor'); this.dimensions=new Map(this.$prefix()+'.dimensions',civita__Dimension); this.equationDisplay=new EquationDisplay(this.$prefix()+'.equationDisplay'); this.evalGodley=new EvalGodley(this.$prefix()+'.evalGodley'); diff --git a/loadDb.py b/loadDb.py index 015d6b7fb..8622377d6 100644 --- a/loadDb.py +++ b/loadDb.py @@ -2,27 +2,52 @@ import json sys.path.insert(0,'.') from pyminsky import minsky, DataSpec -#minsky.databaseIngestor.connect("sqlite3","db=citibike.sqlite") -minsky.databaseIngestor.connect("sqlite3","db=foo.sqlite","citibike") +minsky.databaseIngestor.db.connect("sqlite3","db=citibike.sqlite","citibike") # sys.argv[0] is this script name filenames=sys.argv[1:] # set up spec for Citibike -spec=DataSpec() +spec=minsky.databaseIngestor.spec -spec.setDataArea(1,14) -spec.dataCols([0,11,14]) -spec.dimensionCols([1,4,8,12,13]) -spec.dimensionNames(["tripduration","starttime","stoptime","start station id","start station name","start station latitude","start station longitude","end station id","end station name","end station latitude","end station longitude","bikeid","usertype","birth year","gender"]) -spec.dimensions(15*[{"type":"string","units":""}]) -spec.dimensions[1]({"type":"time","units":""}) -spec.dimensions[11]({"type":"value","units":""}) -spec.dimensions[14]({"type":"value","units":""}) +spec.dataRowOffset(1) +spec.dataCols([0]) +spec.dimensionCols([1,4,8,11,12,13,14]) +spec.dimensionNames([ + "tripduration", + "starttime", + "stoptime", + "start station id", + "start station name", + "start station latitude", + "start station longitude", + "end station id", + "end station name", + "end station latitude", + "end station longitude", + "bikeid", + "usertype", + "birth year", + "gender" +]) -spec.duplicateKeyAction("av") -spec.numCols(15) +spec.dimensions([ + {"type":"value"}, + {"type":"time"}, + {"type":"time"}, + {"type":"string"}, + {"type":"string"}, + {"type":"string"}, + {"type":"string"}, + {"type":"string"}, + {"type":"string"}, + {"type":"string"}, + {"type":"string"}, + {"type":"string"}, + {"type":"string"}, + {"type":"time"}, + {"type":"string"} +]) +spec.dontFail(True) -#'{"counter":false,"dataColOffset":14,"dataCols":[0,11],"dataRowOffset":1,"decSeparator":".","dimensionCols":[1,4,8,12,13,14],"dimensionNames":["tripduration","starttime","stoptime","start station id","start station name","start station latitude","start station longitude","end station id","end station name","end station latitude","end station longitude","bikeid","usertype","birth year","gender"],"dimensions":[{"type":"value","units":""},{"type":"time","units":""},{"type":"time","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""},{"type":"string","units":""}],"dontFail":false,"duplicateKeyAction":"av","escape":"\0","headerRow":0,"horizontalDimName":"?","horizontalDimension":{"type":"string","units":""},"maxColumn":1000,"mergeDelimiters":false,"missingValue":NaN,"numCols":15,"quote":"\"","separator":","})' - -minsky.databaseIngestor.createTable(filenames[0],spec()) -minsky.databaseIngestor.loadDatabase(filenames,spec()) +#minsky.databaseIngestor.db.createTable(filenames[0]) +#minsky.databaseIngestor.importFromCSV(filenames) diff --git a/model/minsky.cc b/model/minsky.cc index a51509810..1d45ca8e1 100644 --- a/model/minsky.cc +++ b/model/minsky.cc @@ -35,6 +35,9 @@ #include "minskyVersion.h" #include "CSVTools.xcd" +#include "databaseIngestor.cd" +#include "databaseIngestor.rcd" +#include "databaseIngestor.xcd" #include "dynamicRavelCAPI.rcd" #include "dynamicRavelCAPI.xcd" #include "fontDisplay.rcd" diff --git a/model/minsky.h b/model/minsky.h index 57bf2ecb3..515f71f57 100644 --- a/model/minsky.h +++ b/model/minsky.h @@ -26,6 +26,7 @@ #include "cairoItems.h" #include "canvas.h" #include "clipboard.h" +#include "databaseIngestor.h" #include "dimension.h" #include "dynamicRavelCAPI.h" #include "evalOp.h" @@ -143,7 +144,8 @@ namespace minsky FontDisplay fontSampler; PhillipsDiagram phillipsDiagram; std::vector publicationTabs; - + DatabaseIngestor databaseIngestor; + void addNewPublicationTab(const std::string& name) {publicationTabs.emplace_back(name);} void addCanvasItemToPublicationTab(size_t i) { if (canvas.item && i Date: Mon, 21 Jul 2025 12:02:32 +1000 Subject: [PATCH 27/41] Release 3.19.0-beta.2 --- Doxyversion | 2 +- doc/version.tex | 2 +- gui-js/libs/shared/src/lib/backend/minsky.ts | 1 + gui-js/libs/shared/src/lib/constants/version.ts | 2 +- gui-js/package-lock.json | 4 ++-- gui-js/package.json | 2 +- minskyVersion.h | 2 +- 7 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Doxyversion b/Doxyversion index a5c785da0..0ce9fcc52 100644 --- a/Doxyversion +++ b/Doxyversion @@ -1 +1 @@ -PROJECT_NAME=Minsky: 3.19.0-beta.1 +PROJECT_NAME=Minsky: 3.19.0-beta.2 diff --git a/doc/version.tex b/doc/version.tex index 68416fcaf..a6c80fa6c 100644 --- a/doc/version.tex +++ b/doc/version.tex @@ -1 +1 @@ -\author{Version 3.19.0-beta.1} +\author{Version 3.19.0-beta.2} diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index 67512be5f..5250bbcda 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -574,6 +574,7 @@ export class DatabaseIngestor extends CppClass { } async classifyColumns(): Promise {return this.$callMethod('classifyColumns');} async correctedUniqueValues(): Promise {return this.$callMethod('correctedUniqueValues');} + async createTable(a1: string): Promise {return this.$callMethod('createTable',a1);} async guessSpecAndLoadFile(): Promise {return this.$callMethod('guessSpecAndLoadFile');} async importFromCSV(a1: string[]): Promise {return this.$callMethod('importFromCSV',a1);} async loadFile(): Promise {return this.$callMethod('loadFile');} diff --git a/gui-js/libs/shared/src/lib/constants/version.ts b/gui-js/libs/shared/src/lib/constants/version.ts index 17fa9aa60..64abb2987 100644 --- a/gui-js/libs/shared/src/lib/constants/version.ts +++ b/gui-js/libs/shared/src/lib/constants/version.ts @@ -1 +1 @@ -export const version="3.19.0-beta.1"; +export const version="3.19.0-beta.2"; diff --git a/gui-js/package-lock.json b/gui-js/package-lock.json index cc51af2e3..d6215ba8e 100644 --- a/gui-js/package-lock.json +++ b/gui-js/package-lock.json @@ -1,12 +1,12 @@ { "name": "ravel", - "version": "3.19.0-beta.1", + "version": "3.19.0-beta.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ravel", - "version": "3.19.0-beta.1", + "version": "3.19.0-beta.2", "hasInstallScript": true, "license": "GPL-3.0-or-later", "dependencies": { diff --git a/gui-js/package.json b/gui-js/package.json index b017769db..50206de02 100644 --- a/gui-js/package.json +++ b/gui-js/package.json @@ -1,6 +1,6 @@ { "name": "ravel", - "version":"3.19.0-beta.1", + "version":"3.19.0-beta.2", "author": "High Performance Coders", "description": "Graphical dynamical systems simulator oriented towards economics", "repository": { diff --git a/minskyVersion.h b/minskyVersion.h index a21415c44..f482a5295 100644 --- a/minskyVersion.h +++ b/minskyVersion.h @@ -1 +1 @@ -#define MINSKY_VERSION "3.19.0-beta.1" +#define MINSKY_VERSION "3.19.0-beta.2" From 4567a1ebb71b988f1003076fc9ff115b77ae923c Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Tue, 22 Jul 2025 16:55:30 +1000 Subject: [PATCH 28/41] Progress bar support for database ingestor Import data to database in GUI --- RavelCAPI | 2 +- engine/databaseIngestor.cc | 29 ++- .../src/app/events/electron.events.ts | 8 + .../app/managers/ApplicationMenuManager.ts | 27 ++- .../src/app/managers/CommandsManager.ts | 7 +- .../minsky-web/src/app/app-routing.module.ts | 5 + gui-js/libs/shared/src/lib/backend/minsky.ts | 1 + .../shared/src/lib/constants/constants.ts | 1 + gui-js/libs/ui-components/src/index.ts | 16 +- .../lib/combo-box/combo-box.component.scss | 41 ++++ .../src/lib/combo-box/combo-box.component.ts | 227 ++++++++++++++++++ .../lib/import-csv/import-csv.component.html | 8 +- .../lib/import-csv/import-csv.component.ts | 18 +- .../new-database/new-database.component.ts | 110 +++++++++ .../src/lib/new-database/new-database.html | 31 +++ .../src/lib/new-database/new-database.scss | 13 + loadDb.py | 4 +- model/minsky.h | 1 - 18 files changed, 522 insertions(+), 27 deletions(-) create mode 100644 gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.scss create mode 100644 gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.ts create mode 100644 gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts create mode 100644 gui-js/libs/ui-components/src/lib/new-database/new-database.html create mode 100644 gui-js/libs/ui-components/src/lib/new-database/new-database.scss diff --git a/RavelCAPI b/RavelCAPI index 62bfe30e7..260d84a5a 160000 --- a/RavelCAPI +++ b/RavelCAPI @@ -1 +1 @@ -Subproject commit 62bfe30e7c66ff57441d02a1194e9930e92810a7 +Subproject commit 260d84a5ad9eebcdb1be7a44c1ab4ad938d0ae3f diff --git a/engine/databaseIngestor.cc b/engine/databaseIngestor.cc index 4665f11ba..e75799d6a 100644 --- a/engine/databaseIngestor.cc +++ b/engine/databaseIngestor.cc @@ -2,12 +2,39 @@ #include "databaseIngestor.cd" #include "databaseIngestor.xcd" #include "databaseIngestor.rcd" +#include "progress.h" +#include "minsky.h" #include "minsky_epilogue.h" +#include + +using namespace std; namespace minsky { + namespace + { + unique_ptr ingestorProgress; + + void progress(const char* filename, double fraction) + { + if (ingestorProgress) ingestorProgress->setProgress(fraction); + } + } + void DatabaseIngestor::importFromCSV(const std::vector& filenames) { - db.loadDatabase(filenames, spec); + // set the custom progress callback if global Minsky object is derived only + auto& m=minsky(); + ProgressUpdater pu(m.progressState,"Importing",filenames.size()); + if (typeid(m)!=typeid(Minsky)) + db.loadDatabaseCallback(progress); + for (auto& f: filenames) + { + filesystem::path p(f); + ingestorProgress=std::make_unique(m.progressState,"Importing: "+p.filename().string(),100); + db.loadDatabase({f}, spec); + ++m.progressState; + } + ingestorProgress.reset(); } } diff --git a/gui-js/apps/minsky-electron/src/app/events/electron.events.ts b/gui-js/apps/minsky-electron/src/app/events/electron.events.ts index aade25e6b..740acf981 100644 --- a/gui-js/apps/minsky-electron/src/app/events/electron.events.ts +++ b/gui-js/apps/minsky-electron/src/app/events/electron.events.ts @@ -222,6 +222,14 @@ ipcMain.handle( } ); +ipcMain.handle( + events.IMPORT_CSV_TO_DB, + async (event, {dropTable}) => { + CommandsManager.importCSV(minsky.databaseIngestor, false, dropTable); + return; + } +); + ipcMain.on(events.CONTEXT_MENU, async (event, { x, y, type, command}) => { await ContextMenuManager.initContextMenu(event, x, y, type, command); }); diff --git a/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts b/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts index 032e50741..dbceb41fd 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts @@ -205,11 +205,28 @@ export class ApplicationMenuManager { }, { label: 'Import Data', - async click() { - minsky.canvas.addVariable(importCSVvariableName, 'parameter'); - let v=new VariableBase(minsky.canvas.itemFocus); - CommandsManager.importCSV(minsky.variableValues.elem(await v.valueId()), true); - } + submenu: [ + { + label: 'to parameter', + async click() { + minsky.canvas.addVariable(importCSVvariableName, 'parameter'); + let v=new VariableBase(minsky.canvas.itemFocus); + CommandsManager.importCSV(minsky.variableValues.elem(await v.valueId()), true); + } + }, + { + label: 'to database', + async click() { + WindowManager.createPopupWindowWithRouting({ + width: 420, + height: 500, + title: '', + url: `#/headless/new-database`, + modal: true, + }); + } + }, + ] }, { label: 'Insert File as Group', diff --git a/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts b/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts index cbc942a52..6e6f6d1df 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts @@ -913,13 +913,14 @@ export class CommandsManager { this.setLogSimulationCheckmark(true); } - static async importCSV(csvDialog: CSVDialog, isInvokedUsingToolbar = false) { + /// @param dropTable whether to create a new table if loading to a database + static async importCSV(csvDialog: CSVDialog, isInvokedUsingToolbar = false, dropTable=false) { const itemInfo: CanvasItem={classType: ClassType.Variable, id: csvDialog.$prefix(), displayContents: false}; if (!WindowManager.focusIfWindowIsPresent(itemInfo.id)) { const window = await this.initializePopupWindow({ itemInfo, - url: `#/headless/import-csv?systemWindowId=0&csvDialog=${csvDialog.$prefix()}&isInvokedUsingToolbar=${isInvokedUsingToolbar}`, + url: `#/headless/import-csv?systemWindowId=0&csvDialog=${csvDialog.$prefix()}&isInvokedUsingToolbar=${isInvokedUsingToolbar}&dropTable=${dropTable}`, height: 600, width: 1300, minWidth: 650, @@ -945,7 +946,7 @@ export class CommandsManager { window.loadURL( WindowManager.getWindowUrl( - `#/headless/import-csv?systemWindowId=${systemWindowId}&csvDialog=${csvDialog.$prefix()}&isInvokedUsingToolbar=${isInvokedUsingToolbar}&examplesPath=${join(dirname(app.getAppPath()),'examples','data')}` + `#/headless/import-csv?systemWindowId=${systemWindowId}&csvDialog=${csvDialog.$prefix()}&isInvokedUsingToolbar=${isInvokedUsingToolbar}&dropTable=${dropTable}&examplesPath=${join(dirname(app.getAppPath()),'examples','data')}` ) ); } diff --git a/gui-js/apps/minsky-web/src/app/app-routing.module.ts b/gui-js/apps/minsky-web/src/app/app-routing.module.ts index 48d0b1db9..e37b74030 100644 --- a/gui-js/apps/minsky-web/src/app/app-routing.module.ts +++ b/gui-js/apps/minsky-web/src/app/app-routing.module.ts @@ -14,6 +14,7 @@ import { FindAllInstancesComponent, GodleyWidgetViewComponent, ImportCsvComponent, + NewDatabaseComponent, NewPubTabComponent, PageNotFoundComponent, SummaryComponent, @@ -103,6 +104,10 @@ const routes: Routes = [ path: 'headless/pick-slices', component: PickSlicesComponent, }, + { + path: 'headless/new-database', + component: NewDatabaseComponent, + }, { path: 'headless/lock-handles', component: LockHandlesComponent, diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index 5250bbcda..ea30f0490 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -2505,6 +2505,7 @@ export class ravelCAPI__Database extends CppClass { async fullHypercube(a1: ravelCAPI__Ravel): Promise {return this.$callMethod('fullHypercube',a1);} async hyperSlice(a1: ravelCAPI__Ravel): Promise {return this.$callMethod('hyperSlice',a1);} async loadDatabase(a1: string[],a2: ravel__DataSpec): Promise {return this.$callMethod('loadDatabase',a1,a2);} + async loadDatabaseCallback(a1: minsky__dummy): Promise {return this.$callMethod('loadDatabaseCallback',a1);} async numericalColumnNames(): Promise {return this.$callMethod('numericalColumnNames');} async setAxisNames(a1: Container,a2: string): Promise {return this.$callMethod('setAxisNames',a1,a2);} async tableNames(): Promise {return this.$callMethod('tableNames');} diff --git a/gui-js/libs/shared/src/lib/constants/constants.ts b/gui-js/libs/shared/src/lib/constants/constants.ts index 8da3c8772..f67456708 100644 --- a/gui-js/libs/shared/src/lib/constants/constants.ts +++ b/gui-js/libs/shared/src/lib/constants/constants.ts @@ -39,6 +39,7 @@ export const events = { GODLEY_VIEW_IMPORT_STOCK: 'godley-view-import-stock', HELP_FOR: 'help-for', IMPORT_CSV: 'import-csv', + IMPORT_CSV_TO_DB: 'import-csv-to-db', INIT_MENU_FOR_GODLEY_VIEW: 'init-menu-for-godley-view', KEY_PRESS: 'key-press', LOG: 'log', diff --git a/gui-js/libs/ui-components/src/index.ts b/gui-js/libs/ui-components/src/index.ts index cbd2d91bb..64f519a85 100644 --- a/gui-js/libs/ui-components/src/index.ts +++ b/gui-js/libs/ui-components/src/index.ts @@ -1,33 +1,33 @@ export * from './lib/connect-database/connect-database.component'; export * from './lib/cli-input/cli-input.component'; export * from './lib/create-variable/create-variable.component'; +export * from './lib/directives/latex.directive'; export * from './lib/edit-description/edit-description.component'; -export * from './lib/edit-handle-description/edit-handle-description.component'; -export * from './lib/edit-handle-dimension/edit-handle-dimension.component'; -export * from './lib/pick-slices/pick-slices.component'; -export * from './lib/lock-handles/lock-handles.component'; export * from './lib/edit-godley-currency/edit-godley-currency.component'; export * from './lib/edit-godley-title/edit-godley-title.component'; export * from './lib/edit-group/edit-group.component'; +export * from './lib/edit-handle-description/edit-handle-description.component'; +export * from './lib/edit-handle-dimension/edit-handle-dimension.component'; export * from './lib/edit-integral/edit-integral.component'; export * from './lib/edit-operation/edit-operation.component'; export * from './lib/edit-user-function/edit-user-function.component'; export * from './lib/equations/equations.component'; export * from './lib/find-all-instances/find-all-instances.component'; export * from './lib/godley-widget-view/godley-widget-view.component'; -export * from './lib/variable-pane/variable-pane.component'; export * from './lib/header/header.component'; export * from './lib/import-csv/import-csv.component'; export * from './lib/input-modal/input-modal.component'; +export * from './lib/lock-handles/lock-handles.component'; +export * from './lib/new-database/new-database.component'; export * from './lib/new-pub-tab/new-pub-tab.component'; export * from './lib/page-not-found/page-not-found.component'; -export * from './lib/summary/summary.component'; export * from './lib/pen-styles/pen-styles.component'; +export * from './lib/pick-slices/pick-slices.component'; export * from './lib/plot-widget-options/plot-widget-options.component'; export * from './lib/plot-widget-view/plot-widget-view.component'; export * from './lib/ravel-widget-view/ravel-widget-view.component'; export * from './lib/rename-all-instances/rename-all-instances.component'; - +export * from './lib/summary/summary.component'; +export * from './lib/variable-pane/variable-pane.component'; export * from './lib/wiring/wiring.component'; -export * from './lib/directives/latex.directive'; diff --git a/gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.scss b/gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.scss new file mode 100644 index 000000000..697f24841 --- /dev/null +++ b/gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.scss @@ -0,0 +1,41 @@ +:host { + display: inline-block; + position: relative; + ::ng-deep { + ul { + pointer-events: none; + opacity: .5; + background: #FFF; + z-index: 1; + } + ul li[aria-selected=true] { background: #ccc; } + ul.empty::after { + content: 'Not Found'; + display: block; + cursor: none; + } + ul li[disabled] { color: #ccc;} + } + &.expanded ::ng-deep ul { + pointer-events: initial; + opacity: 1; + } +} + +:host:not(.no-style) { + display: inline-block; + ::ng-deep { + input:not(.editor) { display: none } + + ul { + display: none; position: absolute; box-sizing: border-box; + top: 100%; min-width: 100%; margin: 0; padding: 4px 6px; + border: 1px solid #CCC; + } + ul li { display: block; } + ul li[disabled] { display: none} + } + &.expanded ::ng-deep ul { + display: block; + } +} \ No newline at end of file diff --git a/gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.ts b/gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.ts new file mode 100644 index 000000000..9268dbb93 --- /dev/null +++ b/gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.ts @@ -0,0 +1,227 @@ +import { + Component, ElementRef, Input, ViewContainerRef, SimpleChanges, + ViewChild, ContentChild, TemplateRef, AfterViewInit, OnChanges, OnDestroy +} from '@angular/core'; +import { NgModel } from '@angular/forms'; + +@Component({ + selector: 'combo-box', + template: ` + + + + `, + styleUrls: [ './combo-box.component.scss' ] +}) +export class ComboBoxComponent implements AfterViewInit, OnChanges, OnDestroy { + @Input() clearInput: boolean; + @Input() disabled: boolean; + @ContentChild(NgModel) ngModel: NgModel; + id = `combobox-${parseInt('' + Math.random()*10000)}`; + docClickHandler = this.onDocumentClick.bind(this); + + inputOrgEl: HTMLInputElement; // Final input value, it becomes not editable + inputEl: HTMLInputElement; // display and keyboard input representing inputEl + ulEl: HTMLUListElement; // list of options given from user. + + options: Array<{text: string, value: string}> = []; // made from
    and
  • + savedSelection: any; // the last selected option + + constructor(private elRef: ElementRef) {} + + ngOnChanges(changes: SimpleChanges) { + if (!changes.disabled.firstChange) { + this.inputEl.disabled = changes.disabled.currentValue; + } + } + + ngAfterViewInit() { + if (!this.setElements()) return false; // check if it has and
      + document.addEventListener('click', this.docClickHandler); + this.options = Array.from(this.ulEl.children).map( (el:any) => { // build options + let [text, value] = [el.innerText, el.getAttribute('value')]; + (value === undefined) && (value = text); + return {text: text.trim(), value: value.trim()}; + }); + this.disabled && (this.inputEl.disabled = true); + } + + ngOnDestroy() { + document.removeEventListener('click', this.docClickHandler); + } + + onInput(event) { // when key input, open popup and select an option if matching + const text = this.inputEl.value; + this.openPopup(text); + this.select(event); + } + + onDocumentClick(event) { + const combobox = event.target.closest('combo-box'); + const clickInMe = combobox === this.elRef.nativeElement; + !clickInMe && this.closePopup(); + } + + onBlur(event) { // when leave input field with invalid value, recover the last value + const text = this.inputEl.value; + const option = this.options.find(el => el.text === text); + !option && this.savedSelection && this.savedSelection.value && + (this.inputEl.value = this.savedSelection.text); + } + + onKeydown(event) { // editable input filed keyboard listener + switch(event.key) { + case 'ArrowUp': case 'Up': // select previous one + this.setHighlighted(this.clearHighlighted(), -1); + break; + case 'ArrowDown': case 'Down': // select next one + this.setHighlighted(this.clearHighlighted(), +1); + break; + case 'Enter': // select current one + this.select(this.ulEl.querySelector('[aria-selected=true]')); + case 'Tab': + case 'Escape': case 'Esc': // close openPopup + this.closePopup(); + break; + } + event.key.match(/Up|Down|Enter|Esc/) && event.preventDefault(); + } + + onFocus(event) { + const isExpanded = this.elRef.nativeElement.classList.contains('expanded'); + if (this.clearInput && !isExpanded) { + this.inputEl.value = ''; + this.openPopup(''); + }; + } + + closePopup() { // when click/Enter/Tab, close popup + this.inputEl.setAttribute('aria-expanded', 'false'); + this.elRef.nativeElement.classList.remove('expanded'); + this.clearHighlighted(); + } + + openPopup(text) { // when focus/input, open popup with matching text + this.inputEl.setAttribute('aria-expanded', 'true'); + this.elRef.nativeElement.classList.add('expanded'); + + let matchingEls = 0; + if (text) { // show only text matching ones with value not empty + matchingEls = Array.from(this.ulEl.children).reduce( (cnt, el:any) => { + const matching = el.innerText.match(new RegExp(text, 'i')); + const value = el.getAttribute('value'); + matching && value ? + el.removeAttribute('disabled') : el.setAttribute('disabled', ''); + return matching ? ++cnt : cnt; + }, 0); + } else { // if text is blank, show it all + matchingEls = Array.from(this.ulEl.children).reduce( (cnt, el:any) => { + el.removeAttribute('disabled'); + return ++cnt; + }, 0); + } + this.clearHighlighted(); + if (matchingEls) { + this.ulEl.classList.remove('empty'); + this.setHighlighted(null, 1); + } else { // if nothing selected, show 'Not Found' + this.ulEl.classList.add('empty'); + } + } + + private select(event: any) { // when click/Enter/Tab, select an option + if (!event) return false; + let text = (event.target || event).innerText; + const obj = this.options.find(el => el.text === text); + if (obj) { + text=obj.value ? obj.text : obj.value; + this.savedSelection = obj; + } + this.ngModel && this.ngModel.control.setValue(text); + this.inputEl.value = text; + + if (!(event instanceof Event)) { // move cursor to the end + const range = document.createRange(); + range.selectNodeContents(this.inputEl); + range.collapse(false); + const sel = window.getSelection(); + sel.removeAllRanges(); + sel.addRange(range); + } + } + + private clearHighlighted() { // unset selected
    • in
        + const highlightedEl = this.ulEl.querySelector('[aria-selected=true]'); + highlightedEl && highlightedEl.removeAttribute('id'); + highlightedEl && highlightedEl.setAttribute('aria-selected', 'false'); + return highlightedEl; + } + + private setHighlighted(el, option) { // set selected
      • in
          + const allAvailEls : Array = + Array.from(this.ulEl.querySelectorAll(':not([disabled])')); + const curIndex = el ? allAvailEls.indexOf(el) : -1; + const nxtIndex = + (curIndex + allAvailEls.length + option) % allAvailEls.length; + allAvailEls[nxtIndex].setAttribute('id', `${this.id}-selected`); + allAvailEls[nxtIndex].setAttribute('aria-selected', 'true'); + } + + private setElements() { // called by ngAfterViewInit, set attributes and events + const thisEl = this.elRef.nativeElement; + this.inputOrgEl = thisEl.querySelector('input:not(.editor)'); + this.ulEl = thisEl.querySelector('ul, select'); + this.inputEl = thisEl.querySelector('input.editor'); + if (this.inputOrgEl && this.ulEl && this.inputEl) { + this.inputOrgEl.setAttribute('tabindex', '-1'); + + this.ulEl.setAttribute('tabindex', '0'); + this.ulEl.setAttribute('role', 'listbox'); + this.ulEl.setAttribute('id', this.id); + Array.from(this.ulEl.children).forEach(liEl => { + liEl.setAttribute('role', 'option'); + liEl.setAttribute('aria-selected', 'false'); + }); + + const placeholderEl: any = this.ulEl.querySelector('[value=""]'); + placeholderEl && + this.inputEl.setAttribute('placeholder',placeholderEl.innerText); + `class,name,required,id,style.title,dir`.split(',').forEach(el => { + const attrVal = this.inputOrgEl.getAttribute(el); + if (attrVal !== null && el === 'class') { + attrVal.replace(/ng-[a-z]+/g,'').split(' ').forEach(className => { + className && this.inputEl.classList.add(className); + }); + } else if (attrVal !== null) { + el === 'id' && this.inputOrgEl.removeAttribute('id'); + this.inputEl.setAttribute(el, attrVal); + } + }); + this.inputEl.setAttribute('role', 'combobox') + this.inputEl.setAttribute('aria-owns', this.id); + this.inputEl.setAttribute('aria-expanded', 'false'); + this.inputEl.setAttribute('aria-activedescendant', `${this.id}-selected`); + + // set events for
        • click, keydown + Array.from(this.ulEl.children).forEach(liEl => { + liEl.addEventListener('click', event => { + this.select(event), this.inputEl.focus(); + this.closePopup(); + }); + }); + + const selectedEl = this.ulEl.querySelector('[selected]'); + selectedEl && setTimeout(_ => this.select(selectedEl)); + + return true; + } else { + console.error('Error on . Missing /
            '); + } + + } + +} diff --git a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.html b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.html index fad567f3b..9e42ffeef 100644 --- a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.html +++ b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.html @@ -386,7 +386,13 @@ - + + + + + diff --git a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts index b81c675e5..a5a9263d2 100644 --- a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts +++ b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts @@ -96,6 +96,7 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni systemWindowId: number; isInvokedUsingToolbar: boolean; + newTable: boolean; examplesPath: string; csvDialog: CSVDialog; timeFormatStrings = dateTimeFormats; @@ -134,6 +135,9 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni public get counter(): AbstractControl { return this.form.get('counter'); } + public get dropTable(): AbstractControl { + return this.form.get('dropTable'); + } public get separator(): AbstractControl { return this.form.get('separator'); } @@ -190,7 +194,8 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni electronService.log(JSON5.stringify(params)); this.csvDialog = new CSVDialog(params.csvDialog); this.systemWindowId = params.systemWindowId; - this.isInvokedUsingToolbar = params.isInvokedUsingToolbar; + this.isInvokedUsingToolbar = params.isInvokedUsingToolbar==="true"; + this.newTable = params.dropTable==="true"; this.examplesPath = params.examplesPath; }); @@ -200,6 +205,7 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni detailedDescription: new FormControl(''), dontFail: new FormControl(false), counter: new FormControl(false), + dropTable: new FormControl(false), decSeparator: new FormControl('.'), duplicateKeyAction: new FormControl('throwException'), escape: new FormControl(''), @@ -269,6 +275,7 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni updateForm() { this.url.setValue(this.dialogState.url); + if (!this.files && this.dialogState.url) this.files=[this.dialogState.url]; this.dontFail.setValue(this.dialogState.spec.dontFail); this.counter.setValue(this.dialogState.spec.counter); @@ -328,15 +335,13 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni const fileUrlOnServer = await this.csvDialog.url(); - if (this.url.value !== fileUrlOnServer) { + if (!this.url.value || this.url.value !== fileUrlOnServer) { await this.csvDialog.url(this.url.value); await this.csvDialog.guessSpecAndLoadFile(); await this.getCSVDialogSpec(); this.updateForm(); - } else { - await this.csvDialog.loadFile(); } - + await this.csvDialog.loadFile(); await this.parseLines(); for (const tab of this.tabs) { @@ -550,6 +555,9 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni if (this.dialogState.spec.dataCols.length === 0) this.dialogState.spec.counter = true; + if (!this.files || !this.files[0]) this.files=[this.url.value]; + if (this.files && (this.dropTable.value || this.newTable)) + this.electronService.minsky.databaseIngestor.createTable(this.files[0]); // returns an error message on error const res = await this.csvDialog.importFromCSV(this.files) as unknown as string; diff --git a/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts b/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts new file mode 100644 index 000000000..8c1553880 --- /dev/null +++ b/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts @@ -0,0 +1,110 @@ +import { ChangeDetectorRef, Component, OnInit, SimpleChanges } from '@angular/core'; +import { FormsModule, } from '@angular/forms'; +import { ActivatedRoute } from '@angular/router'; +import { ElectronService } from '@minsky/core'; +import { events} from '@minsky/shared'; +import { MatAutocompleteModule } from '@angular/material/autocomplete'; +import { MatButtonModule } from '@angular/material/button'; +import { MatOptionModule } from '@angular/material/core'; +import { NgIf, NgFor} from '@angular/common'; +import { OpenDialogOptions, SaveDialogOptions } from 'electron'; +import { CommonModule } from '@angular/common'; // Often useful for ngIf, ngFor +import JSON5 from 'json5'; +import { ComboBoxComponent } from '../combo-box/combo-box.component'; + +@Component({ + selector: 'connect-database', + templateUrl: './new-database.html', + styleUrls: ['./new-database.scss'], + standalone: true, + imports: [ + FormsModule, + ComboBoxComponent, + CommonModule, + MatAutocompleteModule, + MatButtonModule, + MatOptionModule, + NgIf, + NgFor, + ], +}) +export class NewDatabaseComponent implements OnInit { + dbType="sqlite3"; + connection: string; + table=""; + tables=[]; + constructor( + private route: ActivatedRoute, + private electronService: ElectronService, + private cdRef: ChangeDetectorRef + ) { + } + + ngOnInit(): void { + this.route.queryParams.subscribe((params) => { + }); + } + + setDbType(event) { + const target = event.target as HTMLSelectElement; + this.dbType=target.value; + } + + setConnection(event) { + const target = event.target as HTMLSelectElement; + this.connection=target.value; + } + + // get list of tables + async getTables() { + this.electronService.minsky.databaseIngestor.db.connect(this.dbType,this.connection,""); + this.tables=await this.electronService.minsky.databaseIngestor.db.tableNames(); + } + + setTable(event) { + const target = event.target as HTMLSelectElement; + this.table=target.value; + } + + setTableInput(event) { + let input=document.getElementById("table") as HTMLInputElement; + input.value=event?.option?.value; + } + + async selectFile() { + let options: OpenDialogOptions = { + title: 'Select existing database', + filters: [ + { extensions: ['sqlite'], name: 'SQLite' }, + { extensions: ['*'], name: 'All Files' }, + ], + }; + //if (defaultPath) options['defaultPath'] = defaultPath; + let filePath = await this.electronService.openFileDialog(options); + if (filePath) + this.connection=`db=${filePath}`; + else { + // if the user cancelled, then try to create a new database file + options.title='Create new database'; + let filePath = await this.electronService.saveFileDialog(options); + if (filePath) + this.connection=`db=${filePath}`; + } + if (this.connection) { + let connectionInput=document.getElementById("connection") as HTMLInputElement; + connectionInput.hidden=false; + connectionInput.value=this.connection; + this.getTables(); + } + } + + connect() { + this.electronService.minsky.databaseIngestor.db.connect(this.dbType,this.connection,this.table); + // TODO - set dropTable according to whether an existing table is selected, or a new given + let dropTable=!(this.table in this.tables); + this.electronService.invoke(events.IMPORT_CSV_TO_DB, {dropTable}); + this.closeWindow(); + } + + closeWindow() {this.electronService.closeWindow();} +} diff --git a/gui-js/libs/ui-components/src/lib/new-database/new-database.html b/gui-js/libs/ui-components/src/lib/new-database/new-database.html new file mode 100644 index 000000000..b3398c7a9 --- /dev/null +++ b/gui-js/libs/ui-components/src/lib/new-database/new-database.html @@ -0,0 +1,31 @@ +
            +
            + + +
            + + + + {{table}} + +
            + + +
            +
            diff --git a/gui-js/libs/ui-components/src/lib/new-database/new-database.scss b/gui-js/libs/ui-components/src/lib/new-database/new-database.scss new file mode 100644 index 000000000..ba9966679 --- /dev/null +++ b/gui-js/libs/ui-components/src/lib/new-database/new-database.scss @@ -0,0 +1,13 @@ +@import '../../../../shared/src/lib/theme/common-style.scss'; + +.row { + display: flex; + flex-direction: row; + align-items: center; +} + +.container { + display: flex; + flex-direction: column; + align-items: center; +} diff --git a/loadDb.py b/loadDb.py index 8622377d6..1a06d6b19 100644 --- a/loadDb.py +++ b/loadDb.py @@ -49,5 +49,5 @@ ]) spec.dontFail(True) -#minsky.databaseIngestor.db.createTable(filenames[0]) -#minsky.databaseIngestor.importFromCSV(filenames) +minsky.databaseIngestor.db.createTable(filenames[0]) +minsky.databaseIngestor.importFromCSV(filenames) diff --git a/model/minsky.h b/model/minsky.h index 515f71f57..27c64937d 100644 --- a/model/minsky.h +++ b/model/minsky.h @@ -244,7 +244,6 @@ namespace minsky GroupPtr model{new Group}; Canvas canvas{model}; - //DatabaseIngestor databaseIngestor; void clearAllMaps(bool clearHistory); void clearAllMaps() {clearAllMaps(true);} From 4253feadbc04b373d0bd4a175645d5bf2d3e513e Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Tue, 22 Jul 2025 16:58:17 +1000 Subject: [PATCH 29/41] Remove unused combo-box code, as it doesn't work properly --- .../lib/combo-box/combo-box.component.scss | 41 ---- .../src/lib/combo-box/combo-box.component.ts | 227 ------------------ 2 files changed, 268 deletions(-) delete mode 100644 gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.scss delete mode 100644 gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.ts diff --git a/gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.scss b/gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.scss deleted file mode 100644 index 697f24841..000000000 --- a/gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.scss +++ /dev/null @@ -1,41 +0,0 @@ -:host { - display: inline-block; - position: relative; - ::ng-deep { - ul { - pointer-events: none; - opacity: .5; - background: #FFF; - z-index: 1; - } - ul li[aria-selected=true] { background: #ccc; } - ul.empty::after { - content: 'Not Found'; - display: block; - cursor: none; - } - ul li[disabled] { color: #ccc;} - } - &.expanded ::ng-deep ul { - pointer-events: initial; - opacity: 1; - } -} - -:host:not(.no-style) { - display: inline-block; - ::ng-deep { - input:not(.editor) { display: none } - - ul { - display: none; position: absolute; box-sizing: border-box; - top: 100%; min-width: 100%; margin: 0; padding: 4px 6px; - border: 1px solid #CCC; - } - ul li { display: block; } - ul li[disabled] { display: none} - } - &.expanded ::ng-deep ul { - display: block; - } -} \ No newline at end of file diff --git a/gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.ts b/gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.ts deleted file mode 100644 index 9268dbb93..000000000 --- a/gui-js/libs/ui-components/src/lib/combo-box/combo-box.component.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { - Component, ElementRef, Input, ViewContainerRef, SimpleChanges, - ViewChild, ContentChild, TemplateRef, AfterViewInit, OnChanges, OnDestroy -} from '@angular/core'; -import { NgModel } from '@angular/forms'; - -@Component({ - selector: 'combo-box', - template: ` - - - - `, - styleUrls: [ './combo-box.component.scss' ] -}) -export class ComboBoxComponent implements AfterViewInit, OnChanges, OnDestroy { - @Input() clearInput: boolean; - @Input() disabled: boolean; - @ContentChild(NgModel) ngModel: NgModel; - id = `combobox-${parseInt('' + Math.random()*10000)}`; - docClickHandler = this.onDocumentClick.bind(this); - - inputOrgEl: HTMLInputElement; // Final input value, it becomes not editable - inputEl: HTMLInputElement; // display and keyboard input representing inputEl - ulEl: HTMLUListElement; // list of options given from user. - - options: Array<{text: string, value: string}> = []; // made from
              and
            • - savedSelection: any; // the last selected option - - constructor(private elRef: ElementRef) {} - - ngOnChanges(changes: SimpleChanges) { - if (!changes.disabled.firstChange) { - this.inputEl.disabled = changes.disabled.currentValue; - } - } - - ngAfterViewInit() { - if (!this.setElements()) return false; // check if it has and
                - document.addEventListener('click', this.docClickHandler); - this.options = Array.from(this.ulEl.children).map( (el:any) => { // build options - let [text, value] = [el.innerText, el.getAttribute('value')]; - (value === undefined) && (value = text); - return {text: text.trim(), value: value.trim()}; - }); - this.disabled && (this.inputEl.disabled = true); - } - - ngOnDestroy() { - document.removeEventListener('click', this.docClickHandler); - } - - onInput(event) { // when key input, open popup and select an option if matching - const text = this.inputEl.value; - this.openPopup(text); - this.select(event); - } - - onDocumentClick(event) { - const combobox = event.target.closest('combo-box'); - const clickInMe = combobox === this.elRef.nativeElement; - !clickInMe && this.closePopup(); - } - - onBlur(event) { // when leave input field with invalid value, recover the last value - const text = this.inputEl.value; - const option = this.options.find(el => el.text === text); - !option && this.savedSelection && this.savedSelection.value && - (this.inputEl.value = this.savedSelection.text); - } - - onKeydown(event) { // editable input filed keyboard listener - switch(event.key) { - case 'ArrowUp': case 'Up': // select previous one - this.setHighlighted(this.clearHighlighted(), -1); - break; - case 'ArrowDown': case 'Down': // select next one - this.setHighlighted(this.clearHighlighted(), +1); - break; - case 'Enter': // select current one - this.select(this.ulEl.querySelector('[aria-selected=true]')); - case 'Tab': - case 'Escape': case 'Esc': // close openPopup - this.closePopup(); - break; - } - event.key.match(/Up|Down|Enter|Esc/) && event.preventDefault(); - } - - onFocus(event) { - const isExpanded = this.elRef.nativeElement.classList.contains('expanded'); - if (this.clearInput && !isExpanded) { - this.inputEl.value = ''; - this.openPopup(''); - }; - } - - closePopup() { // when click/Enter/Tab, close popup - this.inputEl.setAttribute('aria-expanded', 'false'); - this.elRef.nativeElement.classList.remove('expanded'); - this.clearHighlighted(); - } - - openPopup(text) { // when focus/input, open popup with matching text - this.inputEl.setAttribute('aria-expanded', 'true'); - this.elRef.nativeElement.classList.add('expanded'); - - let matchingEls = 0; - if (text) { // show only text matching ones with value not empty - matchingEls = Array.from(this.ulEl.children).reduce( (cnt, el:any) => { - const matching = el.innerText.match(new RegExp(text, 'i')); - const value = el.getAttribute('value'); - matching && value ? - el.removeAttribute('disabled') : el.setAttribute('disabled', ''); - return matching ? ++cnt : cnt; - }, 0); - } else { // if text is blank, show it all - matchingEls = Array.from(this.ulEl.children).reduce( (cnt, el:any) => { - el.removeAttribute('disabled'); - return ++cnt; - }, 0); - } - this.clearHighlighted(); - if (matchingEls) { - this.ulEl.classList.remove('empty'); - this.setHighlighted(null, 1); - } else { // if nothing selected, show 'Not Found' - this.ulEl.classList.add('empty'); - } - } - - private select(event: any) { // when click/Enter/Tab, select an option - if (!event) return false; - let text = (event.target || event).innerText; - const obj = this.options.find(el => el.text === text); - if (obj) { - text=obj.value ? obj.text : obj.value; - this.savedSelection = obj; - } - this.ngModel && this.ngModel.control.setValue(text); - this.inputEl.value = text; - - if (!(event instanceof Event)) { // move cursor to the end - const range = document.createRange(); - range.selectNodeContents(this.inputEl); - range.collapse(false); - const sel = window.getSelection(); - sel.removeAllRanges(); - sel.addRange(range); - } - } - - private clearHighlighted() { // unset selected
              • in
                  - const highlightedEl = this.ulEl.querySelector('[aria-selected=true]'); - highlightedEl && highlightedEl.removeAttribute('id'); - highlightedEl && highlightedEl.setAttribute('aria-selected', 'false'); - return highlightedEl; - } - - private setHighlighted(el, option) { // set selected
                • in
                    - const allAvailEls : Array = - Array.from(this.ulEl.querySelectorAll(':not([disabled])')); - const curIndex = el ? allAvailEls.indexOf(el) : -1; - const nxtIndex = - (curIndex + allAvailEls.length + option) % allAvailEls.length; - allAvailEls[nxtIndex].setAttribute('id', `${this.id}-selected`); - allAvailEls[nxtIndex].setAttribute('aria-selected', 'true'); - } - - private setElements() { // called by ngAfterViewInit, set attributes and events - const thisEl = this.elRef.nativeElement; - this.inputOrgEl = thisEl.querySelector('input:not(.editor)'); - this.ulEl = thisEl.querySelector('ul, select'); - this.inputEl = thisEl.querySelector('input.editor'); - if (this.inputOrgEl && this.ulEl && this.inputEl) { - this.inputOrgEl.setAttribute('tabindex', '-1'); - - this.ulEl.setAttribute('tabindex', '0'); - this.ulEl.setAttribute('role', 'listbox'); - this.ulEl.setAttribute('id', this.id); - Array.from(this.ulEl.children).forEach(liEl => { - liEl.setAttribute('role', 'option'); - liEl.setAttribute('aria-selected', 'false'); - }); - - const placeholderEl: any = this.ulEl.querySelector('[value=""]'); - placeholderEl && - this.inputEl.setAttribute('placeholder',placeholderEl.innerText); - `class,name,required,id,style.title,dir`.split(',').forEach(el => { - const attrVal = this.inputOrgEl.getAttribute(el); - if (attrVal !== null && el === 'class') { - attrVal.replace(/ng-[a-z]+/g,'').split(' ').forEach(className => { - className && this.inputEl.classList.add(className); - }); - } else if (attrVal !== null) { - el === 'id' && this.inputOrgEl.removeAttribute('id'); - this.inputEl.setAttribute(el, attrVal); - } - }); - this.inputEl.setAttribute('role', 'combobox') - this.inputEl.setAttribute('aria-owns', this.id); - this.inputEl.setAttribute('aria-expanded', 'false'); - this.inputEl.setAttribute('aria-activedescendant', `${this.id}-selected`); - - // set events for
                  • click, keydown - Array.from(this.ulEl.children).forEach(liEl => { - liEl.addEventListener('click', event => { - this.select(event), this.inputEl.focus(); - this.closePopup(); - }); - }); - - const selectedEl = this.ulEl.querySelector('[selected]'); - selectedEl && setTimeout(_ => this.select(selectedEl)); - - return true; - } else { - console.error('Error on . Missing /
                      '); - } - - } - -} From 5eb0e06cc4ae86e6b901f1606a04a4be28a2705b Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Tue, 22 Jul 2025 18:11:45 +1000 Subject: [PATCH 30/41] Database data import refinements. --- engine/databaseIngestor.cc | 9 +++++---- .../src/app/managers/ApplicationMenuManager.ts | 4 ++-- .../src/lib/new-database/new-database.component.ts | 11 +++-------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/engine/databaseIngestor.cc b/engine/databaseIngestor.cc index e75799d6a..8b033aadf 100644 --- a/engine/databaseIngestor.cc +++ b/engine/databaseIngestor.cc @@ -13,7 +13,7 @@ namespace minsky { namespace { - unique_ptr ingestorProgress; + ProgressUpdater* ingestorProgress; void progress(const char* filename, double fraction) { @@ -23,18 +23,19 @@ namespace minsky void DatabaseIngestor::importFromCSV(const std::vector& filenames) { - // set the custom progress callback if global Minsky object is derived only auto& m=minsky(); ProgressUpdater pu(m.progressState,"Importing",filenames.size()); + // set the custom progress callback if global Minsky object is derived only if (typeid(m)!=typeid(Minsky)) db.loadDatabaseCallback(progress); for (auto& f: filenames) { filesystem::path p(f); - ingestorProgress=std::make_unique(m.progressState,"Importing: "+p.filename().string(),100); + ProgressUpdater pu(m.progressState,"Importing: "+p.filename().string(),100); + ingestorProgress=&pu; db.loadDatabase({f}, spec); ++m.progressState; } - ingestorProgress.reset(); + ingestorProgress=nullptr; } } diff --git a/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts b/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts index dbceb41fd..bd0217cbd 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts @@ -218,8 +218,8 @@ export class ApplicationMenuManager { label: 'to database', async click() { WindowManager.createPopupWindowWithRouting({ - width: 420, - height: 500, + width: 250, + height: 150, title: '', url: `#/headless/new-database`, modal: true, diff --git a/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts b/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts index 8c1553880..fdf29d43e 100644 --- a/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts +++ b/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts @@ -6,26 +6,21 @@ import { events} from '@minsky/shared'; import { MatAutocompleteModule } from '@angular/material/autocomplete'; import { MatButtonModule } from '@angular/material/button'; import { MatOptionModule } from '@angular/material/core'; -import { NgIf, NgFor} from '@angular/common'; import { OpenDialogOptions, SaveDialogOptions } from 'electron'; import { CommonModule } from '@angular/common'; // Often useful for ngIf, ngFor import JSON5 from 'json5'; -import { ComboBoxComponent } from '../combo-box/combo-box.component'; @Component({ - selector: 'connect-database', + selector: 'new-database', templateUrl: './new-database.html', styleUrls: ['./new-database.scss'], standalone: true, imports: [ FormsModule, - ComboBoxComponent, CommonModule, MatAutocompleteModule, MatButtonModule, MatOptionModule, - NgIf, - NgFor, ], }) export class NewDatabaseComponent implements OnInit { @@ -68,7 +63,7 @@ export class NewDatabaseComponent implements OnInit { setTableInput(event) { let input=document.getElementById("table") as HTMLInputElement; - input.value=event?.option?.value; + this.table=input.value=event?.option?.value; } async selectFile() { @@ -101,7 +96,7 @@ export class NewDatabaseComponent implements OnInit { connect() { this.electronService.minsky.databaseIngestor.db.connect(this.dbType,this.connection,this.table); // TODO - set dropTable according to whether an existing table is selected, or a new given - let dropTable=!(this.table in this.tables); + let dropTable=!this.tables.includes(this.tables); this.electronService.invoke(events.IMPORT_CSV_TO_DB, {dropTable}); this.closeWindow(); } From 5384d18cfa88948a7d21e25acf47c336abf3ad58 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Wed, 23 Jul 2025 12:29:59 +1000 Subject: [PATCH 31/41] Enable/disable RavelPro menu items according to whether a RavelPro plugin is installed. --- RavelCAPI | 2 +- .../minsky-electron/src/app/managers/ApplicationMenuManager.ts | 1 + .../apps/minsky-electron/src/app/managers/ContextMenuManager.ts | 1 + gui-js/libs/shared/src/lib/backend/minsky.ts | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/RavelCAPI b/RavelCAPI index 260d84a5a..2a2c28f74 160000 --- a/RavelCAPI +++ b/RavelCAPI @@ -1 +1 @@ -Subproject commit 260d84a5ad9eebcdb1be7a44c1ab4ad938d0ae3f +Subproject commit 2a2c28f7463bd5f78c844d7f4645e3a6cfb2ec8b diff --git a/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts b/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts index bd0217cbd..772c4194f 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts @@ -216,6 +216,7 @@ export class ApplicationMenuManager { }, { label: 'to database', + enabled: await minsky.databaseIngestor.db.ravelPro(), async click() { WindowManager.createPopupWindowWithRouting({ width: 250, diff --git a/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts b/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts index 735007847..570999dff 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts @@ -902,6 +902,7 @@ export class ContextMenuManager { }), new MenuItem({ label: 'Connect to database', + enabled: await ravel.db.ravelPro(), click: () => { WindowManager.createPopupWindowWithRouting({ title: 'Connect to database', diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index ea30f0490..a963ca578 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -2507,6 +2507,7 @@ export class ravelCAPI__Database extends CppClass { async loadDatabase(a1: string[],a2: ravel__DataSpec): Promise {return this.$callMethod('loadDatabase',a1,a2);} async loadDatabaseCallback(a1: minsky__dummy): Promise {return this.$callMethod('loadDatabaseCallback',a1);} async numericalColumnNames(): Promise {return this.$callMethod('numericalColumnNames');} + async ravelPro(): Promise {return this.$callMethod('ravelPro');} async setAxisNames(a1: Container,a2: string): Promise {return this.$callMethod('setAxisNames',a1,a2);} async tableNames(): Promise {return this.$callMethod('tableNames');} } From 94c725dd828c956156806485803600dd4a97aa01 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Wed, 23 Jul 2025 16:47:19 +1000 Subject: [PATCH 32/41] Save the directory location of model files opened or saved, as well as CSV files imported to assist opening the file dialogs. For #1791. --- .../src/app/events/electron.events.ts | 4 +- .../app/managers/ApplicationMenuManager.ts | 10 +-- .../src/app/managers/CommandsManager.ts | 34 +++++----- .../src/app/managers/RecordingManager.ts | 9 +-- .../src/app/managers/StoreManager.ts | 4 ++ .../src/app/managers/WindowManager.ts | 65 ++++++++++++++++++- .../connect-database.component.ts | 1 + .../lib/import-csv/import-csv.component.ts | 8 +-- .../new-database/new-database.component.ts | 5 ++ 9 files changed, 106 insertions(+), 34 deletions(-) diff --git a/gui-js/apps/minsky-electron/src/app/events/electron.events.ts b/gui-js/apps/minsky-electron/src/app/events/electron.events.ts index 740acf981..e5f398992 100644 --- a/gui-js/apps/minsky-electron/src/app/events/electron.events.ts +++ b/gui-js/apps/minsky-electron/src/app/events/electron.events.ts @@ -69,7 +69,7 @@ ipcMain.handle(events.CLOSE_WINDOW, (event) => { }); ipcMain.handle(events.OPEN_FILE_DIALOG, async (event, options) => { - const fileDialog = await dialog.showOpenDialog(options); + const fileDialog = await WindowManager.showOpenDialog(options); if (fileDialog.canceled || !fileDialog.filePaths) return ""; if (options?.properties?.includes('multiSelections')) @@ -78,7 +78,7 @@ ipcMain.handle(events.OPEN_FILE_DIALOG, async (event, options) => { }); ipcMain.handle(events.SAVE_FILE_DIALOG, async (event, options) => { - const fileDialog = await dialog.showSaveDialog(options); + const fileDialog = await WindowManager.showSaveDialog(options); if (fileDialog.canceled) return ""; return fileDialog.filePath; }); diff --git a/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts b/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts index 772c4194f..81d83e6af 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/ApplicationMenuManager.ts @@ -143,8 +143,9 @@ export class ApplicationMenuManager { enabled: true, async click() { try { - const _dialog = await dialog.showOpenDialog({ + const _dialog = await WindowManager.showOpenDialog({ properties: ['openFile'], + defaultPath: ':models', filters: [ { name: 'Minsky/Ravel', extensions: ['rvl','mky'] }, { name: '*.xml', extensions: ['xml'] }, @@ -233,7 +234,8 @@ export class ApplicationMenuManager { label: 'Insert File as Group', async click() { try { - const insertGroupDialog = await dialog.showOpenDialog({ + const insertGroupDialog = await WindowManager.showOpenDialog({ + defaultPaths: ':models', properties: ['openFile'], }); @@ -572,9 +574,9 @@ export class ApplicationMenuManager { } private static async exportPlot(extension: string, command: (file:string)=>void) { - const exportPlotDialog = await dialog.showSaveDialog({ + const exportPlotDialog = await WindowManager.showSaveDialog({ title: `Export plot as ${extension}`, - defaultPath: 'plot', + defaultPath: ':models/plot', properties: ['showOverwriteConfirmation', 'createDirectory'], filters: [{ extensions: [extension], name: extension.toUpperCase() }], }); diff --git a/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts b/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts index 6e6f6d1df..97512285b 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts @@ -118,9 +118,9 @@ export class CommandsManager { extension: string, name: string ): Promise { - const exportImage = await dialog.showSaveDialog({ + const exportImage = await WindowManager.showSaveDialog({ title: 'Export item as...', - defaultPath: `export.${extension}`, + defaultPath: `:models/export.${extension}`, properties: ['showOverwriteConfirmation', 'createDirectory'], filters: [ {name, extensions: [extension]}, @@ -171,9 +171,9 @@ export class CommandsManager { } static async exportItemAsCSV(item: any, tabular=false): Promise { - const exportItemDialog = await dialog.showSaveDialog({ + const exportItemDialog = await WindowManager.showSaveDialog({ title: 'Export item as csv', - defaultPath: 'item.csv', + defaultPath: ':models/item.csv', properties: ['showOverwriteConfirmation', 'createDirectory'], filters: [ { extensions: ['csv'], name: 'CSV' }, @@ -325,8 +325,8 @@ export class CommandsManager { } static async saveSelectionAsFile(): Promise { - const saveDialog = await dialog.showSaveDialog({ - defaultPath: 'selection.mky', + const saveDialog = await WindowManager.showSaveDialog({ + defaultPath: ':models/selection.mky', }); const { canceled, filePath } = saveDialog; @@ -368,7 +368,7 @@ export class CommandsManager { } static async getFilePathUsingSaveDialog(): Promise { - const saveDialog = await dialog.showSaveDialog({}); + const saveDialog = await WindowManager.showSaveDialog({defaultPath:':models'}); const { canceled, filePath} = saveDialog; @@ -382,9 +382,9 @@ export class CommandsManager { static async getFilePathFromExportCanvasDialog( type: string, label: string ): Promise { - const exportCanvasDialog = await dialog.showSaveDialog({ + const exportCanvasDialog = await WindowManager.showSaveDialog({ title: 'Export canvas', - defaultPath: `canvas.${type}`, + defaultPath: `:models/canvas.${type}`, properties: ['showOverwriteConfirmation', 'createDirectory'], filters: [ {name: label, extensions: [type]}, @@ -488,7 +488,7 @@ export class CommandsManager { static async saveGroupAsFile(): Promise { const defaultExtension = await minsky.model.defaultExtension(); - const saveDialog = await dialog.showSaveDialog({ + const saveDialog = await WindowManager.showSaveDialog({ filters: [ { name: defaultExtension, @@ -496,7 +496,7 @@ export class CommandsManager { }, { name: 'All', extensions: ['*'] }, ], - defaultPath: `group${defaultExtension}`, + defaultPath: `:models/group${defaultExtension}`, properties: ['showOverwriteConfirmation'], }); @@ -513,7 +513,7 @@ export class CommandsManager { ext: string, command: (x: string)=>void = null ): Promise { - const saveDialog = await dialog.showSaveDialog({ + const saveDialog = await WindowManager.showSaveDialog({ filters: [ { name: '.' + ext, @@ -521,7 +521,7 @@ export class CommandsManager { }, { name: 'All', extensions: ['*'] }, ], - defaultPath: `godley.${ext}`, + defaultPath: `:models/godley.${ext}`, properties: ['showOverwriteConfirmation'], }); @@ -894,9 +894,9 @@ export class CommandsManager { return; } - const logSimulation = await dialog.showSaveDialog({ + const logSimulation = await WindowManager.showSaveDialog({ title: 'Save As', - defaultPath: 'log_simulation.csv', + defaultPath: ':models/log_simulation.csv', properties: ['showOverwriteConfirmation', 'createDirectory'], filters: [{ extensions: ['csv'], name: 'CSV' }], }); @@ -989,7 +989,7 @@ export class CommandsManager { ], defaultPath: this.currentMinskyModelFilePath || - `model${defaultExtension}`, + `:models/model${defaultExtension}`, properties: ['showOverwriteConfirmation'], } } @@ -1003,7 +1003,7 @@ export class CommandsManager { } static async saveAs() { - const saveDialog = await dialog.showSaveDialog(await CommandsManager.defaultSaveOptions()); + const saveDialog = await WindowManager.showSaveDialog(await CommandsManager.defaultSaveOptions()); const { canceled, filePath: filePath } = saveDialog; diff --git a/gui-js/apps/minsky-electron/src/app/managers/RecordingManager.ts b/gui-js/apps/minsky-electron/src/app/managers/RecordingManager.ts index 9caeb1b77..493a6464d 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/RecordingManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/RecordingManager.ts @@ -23,7 +23,8 @@ export class RecordingManager { static async handleRecordingReplay() { this.stopRecording(); - const replayRecordingDialog = await dialog.showOpenDialog({ + const replayRecordingDialog = await WindowManager.showOpenDialog({ + defaultPath: 'models', filters: [ { extensions: ['json'], name: 'JSON' }, { extensions: ['*'], name: 'All Files' }, @@ -47,7 +48,7 @@ export class RecordingManager { const index = dialog.showMessageBoxSync(options); if (options.buttons[index] === positiveResponseText) { - const saveDialog = await dialog.showSaveDialog({}); + const saveDialog = await WindowManager.showSaveDialog({defaultPath: 'models'}); const { canceled, filePath } = saveDialog; @@ -128,13 +129,13 @@ export class RecordingManager { return; } - const saveRecordingDialog = await dialog.showSaveDialog({ + const saveRecordingDialog = await WindowManager.showSaveDialog({ properties: ['showOverwriteConfirmation', 'createDirectory'], filters: [ { extensions: ['json'], name: 'JSON' }, { extensions: ['*'], name: 'All Files' }, ], - defaultPath: 'recording.json', + defaultPath: ':models/recording.json', }); const { canceled, filePath } = saveRecordingDialog; diff --git a/gui-js/apps/minsky-electron/src/app/managers/StoreManager.ts b/gui-js/apps/minsky-electron/src/app/managers/StoreManager.ts index aa24844df..ceb99eee6 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/StoreManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/StoreManager.ts @@ -15,6 +15,8 @@ interface MinskyStore { recentFiles: Array; backgroundColor: string; preferences: MinskyPreferences; + defaultModelDirectory: string; + defaultDataDirectory: string; ravelPlugin: string; // used for post installation installation of Ravel } @@ -24,6 +26,8 @@ class StoreManager { defaults: { recentFiles: [], backgroundColor: defaultBackgroundColor, + defaultModelDirectory: "", + defaultDataDirectory: "", preferences: { godleyTableShowValues: false, godleyTableOutputStyle: 'sign', diff --git a/gui-js/apps/minsky-electron/src/app/managers/WindowManager.ts b/gui-js/apps/minsky-electron/src/app/managers/WindowManager.ts index 6af5b0bb5..38f34eef5 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/WindowManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/WindowManager.ts @@ -11,10 +11,11 @@ import { Utility, } from '@minsky/shared'; import { StoreManager } from './StoreManager'; -import { BrowserWindow, dialog, Menu, screen } from 'electron'; +import { BrowserWindow, dialog, Menu, OpenDialogOptions, OpenDialogReturnValue, SaveDialogOptions, + SaveDialogReturnValue, screen } from 'electron'; import log from 'electron-log'; import os from 'os'; -import { join } from 'path'; +import { join, dirname } from 'path'; import { format } from 'url'; //const logWindows = debug('minsky:electron_windows'); @@ -131,6 +132,66 @@ export class WindowManager { return initialURL; } + /// If options contains defaultPath that has starts with ':model/' + /// or ':data/', then the last directory visited with that type is + /// substituted. + /// returns the directory key for the StoreManager. + static processDefaultDirectory(options: OpenDialogOptions|SaveDialogOptions) { + let splitDefaultPath=/([^\/]*)\/?(.*)/.exec(options.defaultPath); + let defaultType=splitDefaultPath[0]; + let defaultDirectoryKey=""; + + switch (defaultType) { + case ':models': + defaultDirectoryKey='defaultModelDirectory'; + break; + case ':data': + defaultDirectoryKey='defaultDataDirectory'; + break; + } + + if (defaultDirectoryKey) { + let defaultDirectory=StoreManager.store.get(defaultDirectoryKey) as string; + if (defaultDirectory) + options['defaultPath']=defaultDirectory; + } + return defaultDirectoryKey; + } + + /// wrappers around the standard electron dialogs that saves the directory opened as a defaultPath + /// if options.defaultPath is set to either models or data. + static async showOpenDialog(...args: any[]) + { + let options=args[args.length-1] as OpenDialogOptions; + let defaultDirectoryKey=this.processDefaultDirectory(options); + + if (args.length>1) + var res=await dialog.showOpenDialog(args[0],options); + else + var res=await dialog.showOpenDialog(options); + if (!res.canceled && defaultDirectoryKey) { + StoreManager.store.set(defaultDirectoryKey,dirname(res.filePaths[0])); + } + return res; + } + + /// wrappers around the standard electron dialogs that saves the directory opened as a defaultPath + /// if options.defaultPath is set to either models or data. + static async showSaveDialog(...args: any[]) + { + let options=args[args.length-1] as SaveDialogOptions; + let defaultDirectoryKey=this.processDefaultDirectory(options); + + if (args.length>1) + var res=await dialog.showSaveDialog(args[0], options); + else + var res=await dialog.showSaveDialog(options); + if (!res.canceled && defaultDirectoryKey) { + StoreManager.store.set(defaultDirectoryKey,dirname(res.filePath)); + } + return res; + } + /// if window already exists attached to \a url, then raise it /// @return window if it exists, null otherwise static raiseWindow(url: string): BrowserWindow { diff --git a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts index aa714cf3f..5db755a91 100644 --- a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts +++ b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts @@ -61,6 +61,7 @@ export class ConnectDatabaseComponent implements OnInit { async selectFile() { let options: OpenDialogOptions = { + defaultPath: ':models', filters: [ { extensions: ['sqlite'], name: 'CSV' }, { extensions: ['*'], name: 'All Files' }, diff --git a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts index a5a9263d2..28a7eeea0 100644 --- a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts +++ b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts @@ -293,18 +293,16 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni }); } -// async getValueId() { -// return new VariableBase(this.electronService.minsky.namedItems.elem(this.itemId)).valueId(); -// } - async selectFile(defaultPath: string = '') { let options: OpenDialogOptions = { + defaultPath: ':data', filters: [ { extensions: ['csv'], name: 'CSV' }, { extensions: ['*'], name: 'All Files' }, ], properties: ['openFile', 'multiSelections'], }; + // support examples directory if (defaultPath) options['defaultPath'] = defaultPath; const filePaths = await this.electronService.openFileDialog(options); @@ -598,7 +596,7 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni .split('.csv')[0]; const filePath = await this.electronService.saveFileDialog({ - defaultPath: `${filePathWithoutExt}-error-report.csv`, + defaultPath: `:data/${filePathWithoutExt}-error-report.csv`, title: 'Save report', properties: ['showOverwriteConfirmation', 'createDirectory'], filters: [{ extensions: ['csv'], name: 'CSV' }], diff --git a/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts b/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts index fdf29d43e..0adff31ca 100644 --- a/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts +++ b/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts @@ -69,6 +69,7 @@ export class NewDatabaseComponent implements OnInit { async selectFile() { let options: OpenDialogOptions = { title: 'Select existing database', + defaultPath: ':models', filters: [ { extensions: ['sqlite'], name: 'SQLite' }, { extensions: ['*'], name: 'All Files' }, @@ -94,6 +95,10 @@ export class NewDatabaseComponent implements OnInit { } connect() { + if (!this.connection || !this.table) { + this.electronService.showMessageBoxSync({message: "Connection string or table not present"}); + return; + } this.electronService.minsky.databaseIngestor.db.connect(this.dbType,this.connection,this.table); // TODO - set dropTable according to whether an existing table is selected, or a new given let dropTable=!this.tables.includes(this.tables); From 48ac825493f259e33f1e9adc67ded692ef307ff6 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Wed, 23 Jul 2025 17:44:19 +1000 Subject: [PATCH 33/41] Fix regression failures. --- test/00/t0022a.sh | 27 +-------------------------- test/common-test.sh | 2 +- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/test/00/t0022a.sh b/test/00/t0022a.sh index f279e515b..9c4c71958 100755 --- a/test/00/t0022a.sh +++ b/test/00/t0022a.sh @@ -1,32 +1,7 @@ #! /bin/sh here=`pwd` -if test $? -ne 0; then exit 2; fi -tmp=/tmp/$$ -mkdir $tmp -if test $? -ne 0; then exit 2; fi -cd $tmp -if test $? -ne 0; then exit 2; fi - -fail() -{ - echo "FAILED" 1>&2 - cd $here - chmod -R u+w $tmp - rm -rf $tmp - exit 1 -} - -pass() -{ - echo "PASSED" 1>&2 - cd $here - chmod -R u+w $tmp - rm -rf $tmp - exit 0 -} - -trap "fail" 1 2 3 15 +. test/common-test.sh # check description/tooltip functionality cat >input.py < Date: Thu, 24 Jul 2025 17:08:08 +1000 Subject: [PATCH 34/41] Exposed setNumericalAxes functionality on the GUI. --- RavelCAPI | 2 +- .../src/app/managers/ContextMenuManager.ts | 12 + .../minsky-web/src/app/app-routing.module.ts | 77 +-- gui-js/libs/shared/src/lib/backend/minsky.ts | 2 +- gui-js/libs/ui-components/src/index.ts | 1 + .../ravel-select-horizontal-dim.component.ts | 59 ++ .../ravel-select-horizontal-dim.html | 19 + .../ravel-select-horizontal-dim.scss | 24 + gui-js/package-lock.json | 558 ++++++++++++------ 9 files changed, 529 insertions(+), 225 deletions(-) create mode 100644 gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.component.ts create mode 100644 gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.html create mode 100644 gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.scss diff --git a/RavelCAPI b/RavelCAPI index 2a2c28f74..d3326d2ec 160000 --- a/RavelCAPI +++ b/RavelCAPI @@ -1 +1 @@ -Subproject commit 2a2c28f7463bd5f78c844d7f4645e3a6cfb2ec8b +Subproject commit d3326d2ecc55ff4e3693b4d565c96b575c95b47b diff --git a/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts b/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts index 570999dff..47b9bbbd1 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts @@ -912,6 +912,18 @@ export class ContextMenuManager { }) }, }), + new MenuItem({ + label: 'Set numerical axes', + enabled: await ravel.db.ravelPro(), + click: () => { + WindowManager.createPopupWindowWithRouting({ + title: 'Set numerical axes', + url: '#/headless/ravel-select-horizontal-dim', + height: 180, + width: 400, + }) + }, + }), new MenuItem({ label: 'Export as CSV', submenu: this.exportAsCSVSubmenu(ravel), diff --git a/gui-js/apps/minsky-web/src/app/app-routing.module.ts b/gui-js/apps/minsky-web/src/app/app-routing.module.ts index e37b74030..9262601e4 100644 --- a/gui-js/apps/minsky-web/src/app/app-routing.module.ts +++ b/gui-js/apps/minsky-web/src/app/app-routing.module.ts @@ -20,6 +20,7 @@ import { SummaryComponent, PlotWidgetOptionsComponent, PlotWidgetViewComponent, + RavelSelectHorizontalDimComponent, RavelViewComponent, RenameAllInstancesComponent, VariablePaneComponent, @@ -57,92 +58,96 @@ const routes: Routes = [ component: ConnectDatabaseComponent, }, { - path: 'headless/rename-all-instances', - component: RenameAllInstancesComponent, - }, - { - path: 'headless/edit-operation', - component: EditOperationComponent, + path: 'headless/edit-description', + component: EditDescriptionComponent, }, { - path: 'headless/edit-userfunction', - component: EditUserFunctionComponent, + path: 'headless/edit-godley-currency', + component: EditGodleyCurrencyComponent, }, { - path: 'headless/edit-intop', - component: EditIntegralComponent, + path: 'headless/edit-godley-title', + component: EditGodleyTitleComponent, }, { path: 'headless/edit-group', component: EditGroupComponent, }, { - path: 'headless/edit-godley-title', - component: EditGodleyTitleComponent, + path: 'headless/edit-handle-description', + component: EditHandleDescriptionComponent, }, { - path: 'headless/edit-godley-currency', - component: EditGodleyCurrencyComponent, + path: 'headless/edit-handle-dimension', + component: EditHandleDimensionComponent, }, { - path: 'headless/edit-description', - component: EditDescriptionComponent, + path: 'headless/edit-intop', + component: EditIntegralComponent, }, { - path: 'headless/new-pub-tab', - component: NewPubTabComponent, + path: 'headless/edit-operation', + component: EditOperationComponent, }, { - path: 'headless/edit-handle-description', - component: EditHandleDescriptionComponent, + path: 'headless/edit-userfunction', + component: EditUserFunctionComponent, }, { - path: 'headless/edit-handle-dimension', - component: EditHandleDimensionComponent, + path: 'headless/find-all-instances', + component: FindAllInstancesComponent, }, { - path: 'headless/pick-slices', - component: PickSlicesComponent, + path: 'headless/godley-widget-view', + component: GodleyWidgetViewComponent, }, { - path: 'headless/new-database', - component: NewDatabaseComponent, + path: 'headless/import-csv', + component: ImportCsvComponent, }, { path: 'headless/lock-handles', component: LockHandlesComponent, }, { - path: 'headless/find-all-instances', - component: FindAllInstancesComponent, + path: 'headless/new-database', + component: NewDatabaseComponent, }, { - path: 'headless/variable-pane', - component: VariablePaneComponent, + path: 'headless/new-pub-tab', + component: NewPubTabComponent, }, { - path: 'headless/plot-widget-view', - component: PlotWidgetViewComponent, + path: 'headless/pick-slices', + component: PickSlicesComponent, }, { - path: 'headless/godley-widget-view', - component: GodleyWidgetViewComponent, + path: 'headless/plot-widget-view', + component: PlotWidgetViewComponent, }, { path: 'headless/plot-widget-options', component: PlotWidgetOptionsComponent, }, + { + path: 'headless/ravel-select-horizontal-dim', + component: RavelSelectHorizontalDimComponent, + }, { path: 'headless/ravel-widget-view', component: RavelViewComponent, }, + { + path: 'headless/rename-all-instances', + component: RenameAllInstancesComponent, + }, { path: 'headless/terminal', component: CliInputComponent, }, { - path: 'headless/import-csv', - component: ImportCsvComponent, + path: 'headless/variable-pane', + component: VariablePaneComponent, }, { path: '**', diff --git a/gui-js/libs/shared/src/lib/backend/minsky.ts b/gui-js/libs/shared/src/lib/backend/minsky.ts index a963ca578..90cdb22d9 100644 --- a/gui-js/libs/shared/src/lib/backend/minsky.ts +++ b/gui-js/libs/shared/src/lib/backend/minsky.ts @@ -2508,7 +2508,7 @@ export class ravelCAPI__Database extends CppClass { async loadDatabaseCallback(a1: minsky__dummy): Promise {return this.$callMethod('loadDatabaseCallback',a1);} async numericalColumnNames(): Promise {return this.$callMethod('numericalColumnNames');} async ravelPro(): Promise {return this.$callMethod('ravelPro');} - async setAxisNames(a1: Container,a2: string): Promise {return this.$callMethod('setAxisNames',a1,a2);} + async setAxisNames(a1: string[],a2: string): Promise {return this.$callMethod('setAxisNames',a1,a2);} async tableNames(): Promise {return this.$callMethod('tableNames');} } diff --git a/gui-js/libs/ui-components/src/index.ts b/gui-js/libs/ui-components/src/index.ts index 64f519a85..20009ee5a 100644 --- a/gui-js/libs/ui-components/src/index.ts +++ b/gui-js/libs/ui-components/src/index.ts @@ -25,6 +25,7 @@ export * from './lib/pen-styles/pen-styles.component'; export * from './lib/pick-slices/pick-slices.component'; export * from './lib/plot-widget-options/plot-widget-options.component'; export * from './lib/plot-widget-view/plot-widget-view.component'; +export * from './lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.component'; export * from './lib/ravel-widget-view/ravel-widget-view.component'; export * from './lib/rename-all-instances/rename-all-instances.component'; export * from './lib/summary/summary.component'; diff --git a/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.component.ts b/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.component.ts new file mode 100644 index 000000000..1331a3436 --- /dev/null +++ b/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.component.ts @@ -0,0 +1,59 @@ +import { ChangeDetectorRef, Component, OnInit, SimpleChanges } from '@angular/core'; +import { FormsModule, } from '@angular/forms'; +import { ActivatedRoute } from '@angular/router'; +import { ElectronService } from '@minsky/core'; +import { events, Ravel} from '@minsky/shared'; +import { MatAutocompleteModule } from '@angular/material/autocomplete'; +import { MatButtonModule } from '@angular/material/button'; +import { MatOptionModule } from '@angular/material/core'; +import { OpenDialogOptions, SaveDialogOptions } from 'electron'; +import { CommonModule } from '@angular/common'; // Often useful for ngIf, ngFor +import JSON5 from 'json5'; + +@Component({ + selector: 'ravel-select-horizontal-dim', + templateUrl: './ravel-select-horizontal-dim.html', + styleUrls: ['./ravel-select-horizontal-dim.scss'], + standalone: true, + imports: [ + FormsModule, + CommonModule, + MatAutocompleteModule, + MatButtonModule, + MatOptionModule, + ], +}) +export class RavelSelectHorizontalDimComponent implements OnInit { + dataCols=[]; + horizontalDimCols=new Set; + horizontalDimName="?"; + ravel: Ravel; + + constructor( + private route: ActivatedRoute, + private electronService: ElectronService, + private cdRef: ChangeDetectorRef + ) { + this.ravel=new Ravel(this.electronService.minsky.canvas.item); + } + + async ngOnInit() { + this.dataCols=await this.ravel.db.numericalColumnNames(); + } + + clickDim(event) { + const target = event.target as HTMLInputElement; + if (target.checked) + this.horizontalDimCols.add(target.name); + else + this.horizontalDimCols.delete(target.name); + } + + setHorizontalNames() { + this.ravel.db.setAxisNames([...this.horizontalDimCols],this.horizontalDimName); + this.ravel.initRavelFromDb(); + this.closeWindow(); + } + + closeWindow() {this.electronService.closeWindow();} +} diff --git a/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.html b/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.html new file mode 100644 index 000000000..92059b039 --- /dev/null +++ b/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.html @@ -0,0 +1,19 @@ +
                      + + +
                      + +
                      + + +
                      +                       +
                      +
                      + + +
                      +
                      diff --git a/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.scss b/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.scss new file mode 100644 index 000000000..ec42e0843 --- /dev/null +++ b/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.scss @@ -0,0 +1,24 @@ +@import '../../../../shared/src/lib/theme/common-style.scss'; + +:host { + display: block; /* Important for host to take up space */ + height: 100%; /* Allows the component to fill its parent */ +} + +.selectors { + height: calc(100vh + 100px); + overflow-y: auto; + //padding: -10px; +} + +.dim-selector { + display: flex; + flex-direction: row; + justify-content: end; + width: 100%; +} + +.form-buttons { + margin-top: 0px; + margin-bottom: 10px; +} diff --git a/gui-js/package-lock.json b/gui-js/package-lock.json index d6215ba8e..1a36c7227 100644 --- a/gui-js/package-lock.json +++ b/gui-js/package-lock.json @@ -77,13 +77,13 @@ } }, "node_modules/@angular-devkit/architect": { - "version": "0.2001.0", - "resolved": "https://registry.npmjs.org/@angular-devkit/architect/-/architect-0.2001.0.tgz", - "integrity": "sha512-IDBG+YP0nPaA/tIjtJ1ZPh0VEfbxSn0yCvbS7dTfqyrnmanPUFpU5qsT9vJTU6yzkuzBEhNFRzkUCQaUAziLRA==", + "version": "0.2001.2", + "resolved": "https://registry.npmjs.org/@angular-devkit/architect/-/architect-0.2001.2.tgz", + "integrity": "sha512-n6F9VMJXbesgzV4aQEhqoT83irJw+RBbo/V6F8uHilDF3bC4jHBgFhcLkajNAg6i3gLcQb6BpResO7vqQ5MsaQ==", "dev": true, "license": "MIT", "dependencies": { - "@angular-devkit/core": "20.1.0", + "@angular-devkit/core": "20.1.2", "rxjs": "7.8.2" }, "engines": { @@ -93,17 +93,17 @@ } }, "node_modules/@angular-devkit/build-angular": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular-devkit/build-angular/-/build-angular-20.1.0.tgz", - "integrity": "sha512-u0v5X5djZnW7K9HW+tsroyYVNnoX9Q2fCw9+kTBo7kOppM1p+bQ/krLWE2joWhgC++TZV1q0y/T/uEbAP0wyMg==", + "version": "20.1.2", + "resolved": "https://registry.npmjs.org/@angular-devkit/build-angular/-/build-angular-20.1.2.tgz", + "integrity": "sha512-WSkpgMiEryJdCsmbOjx6NUff1RNrZVUneKtYR2cp0AwbkBV3+RaLkQJGtJwd60hn+4OtB6HhDJT3EWigja4yTA==", "dev": true, "license": "MIT", "dependencies": { "@ampproject/remapping": "2.3.0", - "@angular-devkit/architect": "0.2001.0", - "@angular-devkit/build-webpack": "0.2001.0", - "@angular-devkit/core": "20.1.0", - "@angular/build": "20.1.0", + "@angular-devkit/architect": "0.2001.2", + "@angular-devkit/build-webpack": "0.2001.2", + "@angular-devkit/core": "20.1.2", + "@angular/build": "20.1.2", "@babel/core": "7.27.7", "@babel/generator": "7.27.5", "@babel/helper-annotate-as-pure": "7.27.3", @@ -114,7 +114,7 @@ "@babel/preset-env": "7.27.2", "@babel/runtime": "7.27.6", "@discoveryjs/json-ext": "0.6.3", - "@ngtools/webpack": "20.1.0", + "@ngtools/webpack": "20.1.2", "ansi-colors": "4.1.3", "autoprefixer": "10.4.21", "babel-loader": "10.0.0", @@ -169,7 +169,7 @@ "@angular/platform-browser": "^20.0.0", "@angular/platform-server": "^20.0.0", "@angular/service-worker": "^20.0.0", - "@angular/ssr": "^20.1.0", + "@angular/ssr": "^20.1.2", "@web/test-runner": "^0.20.0", "browser-sync": "^3.0.2", "jest": "^29.5.0", @@ -226,13 +226,13 @@ } }, "node_modules/@angular-devkit/build-webpack": { - "version": "0.2001.0", - "resolved": "https://registry.npmjs.org/@angular-devkit/build-webpack/-/build-webpack-0.2001.0.tgz", - "integrity": "sha512-41dGClWoMAL+SoEazyw7AghvVHhbxF6LRSMjlgEiFmSy0aGVyEsYTeH+TlBwClS0KUKXtGx16C5cKch21CuAXA==", + "version": "0.2001.2", + "resolved": "https://registry.npmjs.org/@angular-devkit/build-webpack/-/build-webpack-0.2001.2.tgz", + "integrity": "sha512-JrirWgiauncSeydGkFC0DSYJcyukVeYP8wNxM9IPHf9Yv3E1v83VZRAX4R77lDVzVNK2IMWLhmpWXi49cZWDRQ==", "dev": true, "license": "MIT", "dependencies": { - "@angular-devkit/architect": "0.2001.0", + "@angular-devkit/architect": "0.2001.2", "rxjs": "7.8.2" }, "engines": { @@ -246,9 +246,9 @@ } }, "node_modules/@angular-devkit/core": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-20.1.0.tgz", - "integrity": "sha512-i2t22bklvKsqdwmUtjXltRyxmJ+lJW8isrdc7XeN0N6VW/lDHSJqFlucT1+pO9+FxXJQyz3Hc1dpRd6G65mGyw==", + "version": "20.1.2", + "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-20.1.2.tgz", + "integrity": "sha512-GBZoc5VxgY0xnXVwC715ubcWpVKc2m1H63Nv/msw5mmnfkjgOyG2lo4vA5VzLYVvptc8hwUhX9rsLN/C340rDg==", "dev": true, "license": "MIT", "dependencies": { @@ -274,9 +274,9 @@ } }, "node_modules/@angular/animations": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular/animations/-/animations-20.1.0.tgz", - "integrity": "sha512-5ILngsvu5VPQYaIm7lRyegZaDaAEtLUIPSS8h1dzWPaCxBIJ4uwzx9RDMiF32zhbxi+q0mAO2w2FdDlzWTT3og==", + "version": "20.1.3", + "resolved": "https://registry.npmjs.org/@angular/animations/-/animations-20.1.3.tgz", + "integrity": "sha512-3mkWhcHw2CbfvvjfJYMWjXbTtNHAtZDiVuaqQX4r9i0rPbQ7DqoM1zSgC6XWainWqxnfCHZIZFoI6PKEBVKSrg==", "license": "MIT", "dependencies": { "tslib": "^2.3.0" @@ -285,19 +285,19 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" }, "peerDependencies": { - "@angular/common": "20.1.0", - "@angular/core": "20.1.0" + "@angular/common": "20.1.3", + "@angular/core": "20.1.3" } }, "node_modules/@angular/build": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular/build/-/build-20.1.0.tgz", - "integrity": "sha512-Sl4rkq5PQIrbVNk8cXx2JQhQ156H4bXLvfAYpgXPHAfSfbIIzaV25LJIfTdWSEjMzBGdIX5E0Vpi0SGwcNS7Uw==", + "version": "20.1.2", + "resolved": "https://registry.npmjs.org/@angular/build/-/build-20.1.2.tgz", + "integrity": "sha512-QCzXl/+nnlU7e6hTqWK5dkeUbZWAy/n5trbkIzBLiVQj6j1iTDoF3ABkS76jn5LUKB0Fx1AJVCSAqdxHqMHjDQ==", "dev": true, "license": "MIT", "dependencies": { "@ampproject/remapping": "2.3.0", - "@angular-devkit/architect": "0.2001.0", + "@angular-devkit/architect": "0.2001.2", "@babel/core": "7.27.7", "@babel/helper-annotate-as-pure": "7.27.3", "@babel/helper-split-export-declaration": "7.24.7", @@ -339,7 +339,7 @@ "@angular/platform-browser": "^20.0.0", "@angular/platform-server": "^20.0.0", "@angular/service-worker": "^20.0.0", - "@angular/ssr": "^20.1.0", + "@angular/ssr": "^20.1.2", "karma": "^6.4.0", "less": "^4.2.0", "ng-packagr": "^20.0.0", @@ -389,14 +389,14 @@ } }, "node_modules/@angular/cdk": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular/cdk/-/cdk-20.1.0.tgz", - "integrity": "sha512-JhgbSOv7xZqWNZjuCh8A3A7pGv0mhtmGjHo36157LrxRO6R7x2yJJjxC5nQeroKZWhgN+X/jG/EJlzEvl9PxTw==", + "version": "20.1.3", + "resolved": "https://registry.npmjs.org/@angular/cdk/-/cdk-20.1.3.tgz", + "integrity": "sha512-TO/OBOPWIDJe+0g4S+ye6hewnWOhgWGa4iygvAlmQ77nyqhioHT60puyaDZRATxKh9k6KVmg9cPAk1lYbOFvaA==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "parse5": "^7.1.2", + "parse5": "^8.0.0", "tslib": "^2.3.0" }, "peerDependencies": { @@ -406,9 +406,9 @@ } }, "node_modules/@angular/common": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular/common/-/common-20.1.0.tgz", - "integrity": "sha512-RsHClHJux+4lXrHdGHVw22wekRbSjYtx6Xwjox2S+IRPP51CbX0KskAALZ9ZmtCttkYSFVtvr0S+SQrU2cu5WA==", + "version": "20.1.3", + "resolved": "https://registry.npmjs.org/@angular/common/-/common-20.1.3.tgz", + "integrity": "sha512-h2eQfbx6kYw69xpEHtwZ3XbtWinGa6f8sXj7k9di1/xVAxqtbf+9OcBhYYY++oR1QqDeRghNYNblNNt0H9zKzQ==", "license": "MIT", "dependencies": { "tslib": "^2.3.0" @@ -417,14 +417,14 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" }, "peerDependencies": { - "@angular/core": "20.1.0", + "@angular/core": "20.1.3", "rxjs": "^6.5.3 || ^7.4.0" } }, "node_modules/@angular/compiler": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-20.1.0.tgz", - "integrity": "sha512-sM8H3dJotIDDmI1u8qGuAn16XVfR7A4+/5s5cKLI/osnnIjafi5HHqAf76R5IlGoIv0ZHVQIYaJ/Qdvfyvdhfg==", + "version": "20.1.3", + "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-20.1.3.tgz", + "integrity": "sha512-NGMFLymImIdvjLSoH+pasgtJxKynDHX9COBU6T5LP7qi5kf6eR829Zrf7650R3K+uERqwz5PTLg8Kwa4aY7I9w==", "license": "MIT", "dependencies": { "tslib": "^2.3.0" @@ -434,9 +434,9 @@ } }, "node_modules/@angular/compiler-cli": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular/compiler-cli/-/compiler-cli-20.1.0.tgz", - "integrity": "sha512-ajbCmvYYFxeXRdKSfdHjp62MZ2lCMUS0UzswBDAbT9sPd/ThppbvLXLsMBj8SlwaXSSBeTAa1oSHEO1MeuVvGQ==", + "version": "20.1.3", + "resolved": "https://registry.npmjs.org/@angular/compiler-cli/-/compiler-cli-20.1.3.tgz", + "integrity": "sha512-NT7+vtwABtvVj2NLL7KvRzSsa5hgro23AvkAvg6A5sdfWzYDRXovI0YILlTIx1oEA8rupTPu/39gStW5k8XZqg==", "license": "MIT", "dependencies": { "@babel/core": "7.28.0", @@ -456,7 +456,7 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" }, "peerDependencies": { - "@angular/compiler": "20.1.0", + "@angular/compiler": "20.1.3", "typescript": ">=5.8 <5.9" }, "peerDependenciesMeta": { @@ -527,9 +527,9 @@ } }, "node_modules/@angular/core": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular/core/-/core-20.1.0.tgz", - "integrity": "sha512-/dJooZi+OAACkjWgGMPrOOGikdtlTJXwdeXPJTgZSUD5L8oQMbhZFG0XW/1Hldvsti87wPjZPz67ivB7zR86VA==", + "version": "20.1.3", + "resolved": "https://registry.npmjs.org/@angular/core/-/core-20.1.3.tgz", + "integrity": "sha512-haQypZGbKKsClDbR0I4eK+PmKGaZ8b/9QDwNYzInaEqHrTX/rkFXu0L0ejTTznElutQuMM6OPh6aVfnJ9nRr2g==", "license": "MIT", "dependencies": { "tslib": "^2.3.0" @@ -538,7 +538,7 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" }, "peerDependencies": { - "@angular/compiler": "20.1.0", + "@angular/compiler": "20.1.3", "rxjs": "^6.5.3 || ^7.4.0", "zone.js": "~0.15.0" }, @@ -552,9 +552,9 @@ } }, "node_modules/@angular/forms": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular/forms/-/forms-20.1.0.tgz", - "integrity": "sha512-NgQxowyyG2yiSOXxtQS1xK1vAQT+4GRoMFuzmS3uBshIifgCgFckSxJHQXhlQOInuv2NsZ1Q0HuCvao+yZfIow==", + "version": "20.1.3", + "resolved": "https://registry.npmjs.org/@angular/forms/-/forms-20.1.3.tgz", + "integrity": "sha512-q2Lbz65mqk/Xmp3qvFSZyUJRKeah3jtfSRxJlHC63utG5WdGl7gN7xRy2dydarRKToWyXqMsjoSlh1YIrUIAng==", "license": "MIT", "dependencies": { "tslib": "^2.3.0" @@ -563,23 +563,23 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" }, "peerDependencies": { - "@angular/common": "20.1.0", - "@angular/core": "20.1.0", - "@angular/platform-browser": "20.1.0", + "@angular/common": "20.1.3", + "@angular/core": "20.1.3", + "@angular/platform-browser": "20.1.3", "rxjs": "^6.5.3 || ^7.4.0" } }, "node_modules/@angular/material": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular/material/-/material-20.1.0.tgz", - "integrity": "sha512-LfGz/V/kZwRIhzIZBiurM4Wc5CQiiJkiOChUfoEOvQLN2hckPFZbbvtg6JwxxA6nhzsDhuGHbj7Xj5dNsLfZLw==", + "version": "20.1.3", + "resolved": "https://registry.npmjs.org/@angular/material/-/material-20.1.3.tgz", + "integrity": "sha512-W6/XJ2mih70b+PJUEAbI3mC415/SNY06nMBKcjWjRSth0jHe5/ujqIj0WkygkpDz34HEa11vV/0BgSpdS2FT5g==", "dev": true, "license": "MIT", "dependencies": { "tslib": "^2.3.0" }, "peerDependencies": { - "@angular/cdk": "20.1.0", + "@angular/cdk": "20.1.3", "@angular/common": "^20.0.0 || ^21.0.0", "@angular/core": "^20.0.0 || ^21.0.0", "@angular/forms": "^20.0.0 || ^21.0.0", @@ -588,9 +588,9 @@ } }, "node_modules/@angular/platform-browser": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-20.1.0.tgz", - "integrity": "sha512-l3+Ijq5SFxT0v10DbOyMc7NzGdbK76yot2i8pXyArlPSPmpWvbbjXbiBqzrv3TSTrksHBhG3mMvyhTmHQ1cQFA==", + "version": "20.1.3", + "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-20.1.3.tgz", + "integrity": "sha512-58iwj2LXdvwr4DG5tAiA2vj9bm/fhBWaR5JWvn3fJEAdW8fnT2gpjpfdBJTMcqg7Qfpx0ZhFsRxH2EUGEV6mvw==", "license": "MIT", "peer": true, "dependencies": { @@ -600,9 +600,9 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" }, "peerDependencies": { - "@angular/animations": "20.1.0", - "@angular/common": "20.1.0", - "@angular/core": "20.1.0" + "@angular/animations": "20.1.3", + "@angular/common": "20.1.3", + "@angular/core": "20.1.3" }, "peerDependenciesMeta": { "@angular/animations": { @@ -611,9 +611,9 @@ } }, "node_modules/@angular/platform-browser-dynamic": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular/platform-browser-dynamic/-/platform-browser-dynamic-20.1.0.tgz", - "integrity": "sha512-s+Rm2akzYTE2UFdXZPvf02TxDCDskGdUxAxa/jmJbVuOpniuY0RlbnxIKDUD0qj3bYMUkbr7f2KJwHVldqJP6w==", + "version": "20.1.3", + "resolved": "https://registry.npmjs.org/@angular/platform-browser-dynamic/-/platform-browser-dynamic-20.1.3.tgz", + "integrity": "sha512-y8m+HNHTYfgyQ/Mtku6+NOvlrD54oaj5cTnr382MVc692r+FuBkI9jMI1oZCqNTdv9cFK6Opj5Ie6A7ZxAfGVA==", "license": "MIT", "dependencies": { "tslib": "^2.3.0" @@ -622,16 +622,16 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" }, "peerDependencies": { - "@angular/common": "20.1.0", - "@angular/compiler": "20.1.0", - "@angular/core": "20.1.0", - "@angular/platform-browser": "20.1.0" + "@angular/common": "20.1.3", + "@angular/compiler": "20.1.3", + "@angular/core": "20.1.3", + "@angular/platform-browser": "20.1.3" } }, "node_modules/@angular/router": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@angular/router/-/router-20.1.0.tgz", - "integrity": "sha512-fuUX1+AhcVSDgSSx85o6VOtXKM3oXAza+44jQ+nJGf316P0xpLKA586DKRNPjS4sRsWM7otKuOOTXXc4AMUHpQ==", + "version": "20.1.3", + "resolved": "https://registry.npmjs.org/@angular/router/-/router-20.1.3.tgz", + "integrity": "sha512-ELJyzFJ2JeJkuVpv3kte4AwGBd/zuB5H/wv4+9gcmf6exxO5xH2/PbbLDGs+rWwHkCUcoRHFVyUPqk9yuRq/XA==", "license": "MIT", "dependencies": { "tslib": "^2.3.0" @@ -640,9 +640,9 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" }, "peerDependencies": { - "@angular/common": "20.1.0", - "@angular/core": "20.1.0", - "@angular/platform-browser": "20.1.0", + "@angular/common": "20.1.3", + "@angular/core": "20.1.3", + "@angular/platform-browser": "20.1.3", "rxjs": "^6.5.3 || ^7.4.0" } }, @@ -3287,28 +3287,28 @@ } }, "node_modules/@emnapi/core": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.4.4.tgz", - "integrity": "sha512-A9CnAbC6ARNMKcIcrQwq6HeHCjpcBZ5wSx4U01WXCqEKlrzB9F9315WDNHkrs2xbx7YjjSxbUYxuN6EQzpcY2g==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.4.5.tgz", + "integrity": "sha512-XsLw1dEOpkSX/WucdqUhPWP7hDxSvZiY+fsUC14h+FtQ2Ifni4znbBt8punRX+Uj2JG/uDb8nEHVKvrVlvdZ5Q==", "license": "MIT", "dependencies": { - "@emnapi/wasi-threads": "1.0.3", + "@emnapi/wasi-threads": "1.0.4", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.4.4.tgz", - "integrity": "sha512-hHyapA4A3gPaDCNfiqyZUStTMqIkKRshqPIuDOXv1hcBnD4U3l8cP0T1HMCfGRxQ6V64TGCcoswChANyOAwbQg==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.4.5.tgz", + "integrity": "sha512-++LApOtY0pEEz1zrd9vy1/zXVaVJJ/EbAF3u0fXIzPJEDtnITsBGbbK0EkM72amhl/R5b+5xx0Y/QhcVOpuulg==", "license": "MIT", "dependencies": { "tslib": "^2.4.0" } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.0.3.tgz", - "integrity": "sha512-8K5IFFsQqF9wQNJptGbS6FNKgUTsSRYnTqNCG1vPP8jFdjSv18n2mQfJpkt2Oibo9iBEzcDnDxNwKTzC7svlJw==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.0.4.tgz", + "integrity": "sha512-PJR+bOmMOPH8AtcTGAyYNiuJ3/Fcoj2XN/gBEWzDIKh254XO+mM9XoXHk5GNEhodxeMznbg7BlRojVbKN+gC6g==", "license": "MIT", "dependencies": { "tslib": "^2.4.0" @@ -3786,14 +3786,14 @@ } }, "node_modules/@inquirer/core": { - "version": "10.1.14", - "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.1.14.tgz", - "integrity": "sha512-Ma+ZpOJPewtIYl6HZHZckeX1STvDnHTCB2GVINNUlSEn2Am6LddWwfPkIGY0IUFVjUUrr/93XlBwTK6mfLjf0A==", + "version": "10.1.15", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.1.15.tgz", + "integrity": "sha512-8xrp836RZvKkpNbVvgWUlxjT4CraKk2q+I3Ksy+seI2zkcE+y6wNs1BVhgcv8VyImFecUhdQrYLdW32pAjwBdA==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/figures": "^1.0.12", - "@inquirer/type": "^3.0.7", + "@inquirer/figures": "^1.0.13", + "@inquirer/type": "^3.0.8", "ansi-escapes": "^4.3.2", "cli-width": "^4.1.0", "mute-stream": "^2.0.0", @@ -3814,9 +3814,9 @@ } }, "node_modules/@inquirer/figures": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.12.tgz", - "integrity": "sha512-MJttijd8rMFcKJC8NYmprWr6hD3r9Gd9qUC0XwPNwoEPWSMVJwA2MlXxF+nhZZNMY+HXsWa+o7KY2emWYIn0jQ==", + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.13.tgz", + "integrity": "sha512-lGPVU3yO9ZNqA7vTYz26jny41lE7yoQansmqdMLBEfqaGsmdg7V3W9mK9Pvb5IL4EVZ9GnSDGMO/cJXud5dMaw==", "dev": true, "license": "MIT", "engines": { @@ -3824,9 +3824,9 @@ } }, "node_modules/@inquirer/type": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.7.tgz", - "integrity": "sha512-PfunHQcjwnju84L+ycmcMKB/pTPIngjUJvfnRhKY6FKPuYXlM4aQCb/nIdTFR6BEhMjFvngzvng/vBAJMZpLSA==", + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.8.tgz", + "integrity": "sha512-lg9Whz8onIHRthWaN1Q9EGLa/0LFJjyM8mEUbL1eTi6yMGvBf8gvyDLtxSXztQsxMvhxxNpJYrwa1YHdq+w4Jw==", "dev": true, "license": "MIT", "engines": { @@ -4107,6 +4107,17 @@ } } }, + "node_modules/@jest/diff-sequences": { + "version": "30.0.1", + "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.0.1.tgz", + "integrity": "sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, "node_modules/@jest/environment": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", @@ -4168,6 +4179,17 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/@jest/get-type": { + "version": "30.0.1", + "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.0.1.tgz", + "integrity": "sha512-AyYdemXCptSRFirI5EPazNxyPwAL0jXt3zceFjaj8NFiKP9pOi0bfXonf6qkf82z2t3QWPeLCWWw4stPBzctLw==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, "node_modules/@jest/globals": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", @@ -5072,9 +5094,9 @@ } }, "node_modules/@ngtools/webpack": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/@ngtools/webpack/-/webpack-20.1.0.tgz", - "integrity": "sha512-v+Mdg+NIvkWJYWcuHCQeRC4/Wov8RxNEF8eiCPFmQGmXJllIWUybY/o9lysG1TY4j/2H56VinIBYbeK/VIBYvg==", + "version": "20.1.2", + "resolved": "https://registry.npmjs.org/@ngtools/webpack/-/webpack-20.1.2.tgz", + "integrity": "sha512-1kN6o/JGevLY9d89qyRtr5bKkRMBUwH2/6wIquZgkcK2jPYs0Cmm0N7kV0eDL7yBFLI4RtyV6IHligPmdCHpeA==", "dev": true, "license": "MIT", "engines": { @@ -7076,27 +7098,41 @@ ] }, "node_modules/@nx/workspace": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/@nx/workspace/-/workspace-21.2.3.tgz", - "integrity": "sha512-bC3J6pgXvL9JWyYmP7AOGCIZhtI6vmY1YLan1T+FFkSr7yyKvIwnnL9E68whQD5jcbJl1Mvu9l0lVlsVdQYF/g==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/@nx/workspace/-/workspace-21.3.5.tgz", + "integrity": "sha512-jvA/wVOzMANHIvyz+9ACkK/twUaYHWgzDbES+xnQDdNMnjzkbgHPWdeTnjkyu2dspTm4c5nVMHH6ErMv6IjWeQ==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@nx/devkit": "21.2.3", + "@nx/devkit": "21.3.5", "@zkochan/js-yaml": "0.0.7", "chalk": "^4.1.0", "enquirer": "~2.3.6", - "nx": "21.2.3", + "nx": "21.3.5", "picomatch": "4.0.2", "tslib": "^2.3.0", "yargs-parser": "21.1.1" } }, + "node_modules/@nx/workspace/node_modules/@jest/schemas": { + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.5.tgz", + "integrity": "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@sinclair/typebox": "^0.34.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, "node_modules/@nx/workspace/node_modules/@nx/devkit": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/@nx/devkit/-/devkit-21.2.3.tgz", - "integrity": "sha512-H5Hk0qeZwqhxQmqcWaLpMc+otU4TroUzDYoV6kFpZdvcwGnXQKHCuGzZoI18kh9wPXvKFmb1BWmr9as3lHUw3Q==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/@nx/devkit/-/devkit-21.3.5.tgz", + "integrity": "sha512-YTmjD+kqDUapfcV37IgVddLZL4oOoFVXO0dFKsYSkx/FNmhccTbeXxgsdcyRTJY6gCwsFJ+4X0aIv8NxebWYaw==", "dev": true, "license": "MIT", "peer": true, @@ -7111,13 +7147,13 @@ "yargs-parser": "21.1.1" }, "peerDependencies": { - "nx": "21.2.3" + "nx": "21.3.5" } }, "node_modules/@nx/workspace/node_modules/@nx/nx-darwin-arm64": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/@nx/nx-darwin-arm64/-/nx-darwin-arm64-21.2.3.tgz", - "integrity": "sha512-5WgOjoX4vqG286A8abYoLCScA2ZF5af/2ZBjaM5EHypgbJLGQuMcP2ahzX66FYohT4wdAej1D0ILkEax71fAKw==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/@nx/nx-darwin-arm64/-/nx-darwin-arm64-21.3.5.tgz", + "integrity": "sha512-QKWtKjIdYS2foDo/4ojvVr5NjrtY8IcHPyFFATAk7v5BWe2tEGh6pPDj8GRqvSqZPWSBZNDcJ6efovJyka6yzw==", "cpu": [ "arm64" ], @@ -7130,9 +7166,9 @@ "peer": true }, "node_modules/@nx/workspace/node_modules/@nx/nx-darwin-x64": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/@nx/nx-darwin-x64/-/nx-darwin-x64-21.2.3.tgz", - "integrity": "sha512-aSaK8Ic9nHTwSuNZZtaKCPIXgD6+Ss9UwkNMIXPLYiYLF+EdSDORHnHutmajZZ8HakoWCQPWvxfWv30zre6iqw==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/@nx/nx-darwin-x64/-/nx-darwin-x64-21.3.5.tgz", + "integrity": "sha512-X06eb6lzuln9ZHh7L8430s4Cc7pi5mihU3IlJsN0rbgdCp2PMlOsJ/8P/zw/DBwq6qmTuVwZ8Xk01VOVtWZT0w==", "cpu": [ "x64" ], @@ -7145,9 +7181,9 @@ "peer": true }, "node_modules/@nx/workspace/node_modules/@nx/nx-freebsd-x64": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/@nx/nx-freebsd-x64/-/nx-freebsd-x64-21.2.3.tgz", - "integrity": "sha512-hFSbtaYM1gL+XQq88CkmwqeeabmFsLjpsBF+HFIv1UMAjb02ObrYHVVICmmin5c1NkBsEJcQzh3mf8PBSOHW8A==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/@nx/nx-freebsd-x64/-/nx-freebsd-x64-21.3.5.tgz", + "integrity": "sha512-+ZWvAn/1UD4Wwduv5nhcXWIUcev1JDEEsNBesGvFWS4c19dBk8vaUpUv3YQspwRUgAGdfYw1CWqDDOYvGFrZ3w==", "cpu": [ "x64" ], @@ -7160,9 +7196,9 @@ "peer": true }, "node_modules/@nx/workspace/node_modules/@nx/nx-linux-arm-gnueabihf": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/@nx/nx-linux-arm-gnueabihf/-/nx-linux-arm-gnueabihf-21.2.3.tgz", - "integrity": "sha512-yRzt8dLwTpRP7655We9/ol+Ol+n52R9wsRRnxJFdWHyLrHguZF0dqiZ5rAFFzyvywaDP6CRoPuS7wqFT7K14bw==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/@nx/nx-linux-arm-gnueabihf/-/nx-linux-arm-gnueabihf-21.3.5.tgz", + "integrity": "sha512-JeSw0/WdVo4AxCKWRrH686rxu9jHzKnl/IY1m+/jiIPq2yUPUUxqSj9+Xesvp9K3plAmZFlSulbfd8BX15/cEw==", "cpu": [ "arm" ], @@ -7175,9 +7211,9 @@ "peer": true }, "node_modules/@nx/workspace/node_modules/@nx/nx-linux-arm64-gnu": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/@nx/nx-linux-arm64-gnu/-/nx-linux-arm64-gnu-21.2.3.tgz", - "integrity": "sha512-5u8mmUogvrNn1xlJk8Y6AJg/g1h2bKxYSyWfxR2mazKj5wI/VgbHuxHAgMXB7WDW2tK5bEcrUTvO8V0DjZQhNA==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/@nx/nx-linux-arm64-gnu/-/nx-linux-arm64-gnu-21.3.5.tgz", + "integrity": "sha512-TlMhwwj75cP67m/qYuSAmvdMMW6oASELo3uxRJ9PbpyTRiCmQZoqjZqALL/48rAEk1Ig69o83RI4pIMRkGMQUw==", "cpu": [ "arm64" ], @@ -7190,9 +7226,9 @@ "peer": true }, "node_modules/@nx/workspace/node_modules/@nx/nx-linux-arm64-musl": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/@nx/nx-linux-arm64-musl/-/nx-linux-arm64-musl-21.2.3.tgz", - "integrity": "sha512-4huuq2iuCBOWmJQw60gk5g3yjeHxFzwdDZJPM0680fZ7Pa/haPwamkR6kE2U6aFtFMhi1QVGPEoj4v4vE4ZS5g==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/@nx/nx-linux-arm64-musl/-/nx-linux-arm64-musl-21.3.5.tgz", + "integrity": "sha512-GZBMLTJFP9H9/o26jSfqxwlBoZ4c0FNBl8rJ3tOC9jCNHn7wcMJKaVbp0dUKDgUtyzATa5poJsdClG84zMnHdA==", "cpu": [ "arm64" ], @@ -7205,9 +7241,9 @@ "peer": true }, "node_modules/@nx/workspace/node_modules/@nx/nx-linux-x64-gnu": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/@nx/nx-linux-x64-gnu/-/nx-linux-x64-gnu-21.2.3.tgz", - "integrity": "sha512-qWpJXpF8vjOrZTkgSC8kQAnIh0pIFbsisePicYWj5U9szJYyTUvVbjMAvdUPH4Z3bnrUtt+nzf9mpFCJRLjsOQ==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/@nx/nx-linux-x64-gnu/-/nx-linux-x64-gnu-21.3.5.tgz", + "integrity": "sha512-rNZ2X+h3AbF+vM3zKcpv122Eu5fyYS0079iNiYAHNknwLPJUlvDEQU3nu6ts8Hw1zSjxzibHbWZghSfZRAIHZA==", "cpu": [ "x64" ], @@ -7220,9 +7256,9 @@ "peer": true }, "node_modules/@nx/workspace/node_modules/@nx/nx-linux-x64-musl": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/@nx/nx-linux-x64-musl/-/nx-linux-x64-musl-21.2.3.tgz", - "integrity": "sha512-JZHlovF9uzvN3blImysYJmG90/8ookr3jOmEFxmP4RfMUl6EdN9yBLBdx0zIG2ulh7+WQrR3eQ1qrmsWFb6oiw==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/@nx/nx-linux-x64-musl/-/nx-linux-x64-musl-21.3.5.tgz", + "integrity": "sha512-LtsDUhrH0sVl7gBSsJ+cy/cKH71PorysOhJqTrE7Z5UVpnWu+1djiOsbEDRAheyUf80QfFA8xC239Pi+QG3T/w==", "cpu": [ "x64" ], @@ -7235,9 +7271,9 @@ "peer": true }, "node_modules/@nx/workspace/node_modules/@nx/nx-win32-arm64-msvc": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/@nx/nx-win32-arm64-msvc/-/nx-win32-arm64-msvc-21.2.3.tgz", - "integrity": "sha512-8Q1ljgFle6F2ZGSe6dLBItSdvYXjO0n2ovZI0zIih9+5OGLdN8wf6iONQJT7he2YST1dowIDPNWdeKiuOzPo6w==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/@nx/nx-win32-arm64-msvc/-/nx-win32-arm64-msvc-21.3.5.tgz", + "integrity": "sha512-lmhzLTVXzQNa0em6v0gBCfswpD5vdgtcjUxr5flR6ylWYo0hVYD4w/EqoymqXq0rU94lx28lksmKX0vhNH5aiw==", "cpu": [ "arm64" ], @@ -7250,9 +7286,9 @@ "peer": true }, "node_modules/@nx/workspace/node_modules/@nx/nx-win32-x64-msvc": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/@nx/nx-win32-x64-msvc/-/nx-win32-x64-msvc-21.2.3.tgz", - "integrity": "sha512-qJpHIZU/D48+EZ2bH02/LIFIkANYryGbcbNQUqC+pYA8ZPCU0wMqZVn4UcNMoI9K4YtXe/SvSBdjiObDuRb8yw==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/@nx/nx-win32-x64-msvc/-/nx-win32-x64-msvc-21.3.5.tgz", + "integrity": "sha512-8ljAlmV96WUK7NbXCL96HDxP7Qm6MDKgz/8mD4XtMJIWvZ098VDv4+1Ce2lK366grQRmNQfYVUuhwPppnJfqug==", "cpu": [ "x64" ], @@ -7264,6 +7300,28 @@ ], "peer": true }, + "node_modules/@nx/workspace/node_modules/@sinclair/typebox": { + "version": "0.34.38", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.38.tgz", + "integrity": "sha512-HpkxMmc2XmZKhvaKIZZThlHmx1L0I/V1hWK1NubtlFnr6ZqdiOpV72TKudZUNQjZNsyDBay72qFEhEvb+bcwcA==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/@nx/workspace/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/@nx/workspace/node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -7372,6 +7430,23 @@ "node": ">=8" } }, + "node_modules/@nx/workspace/node_modules/jest-diff": { + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.0.5.tgz", + "integrity": "sha512-1UIqE9PoEKaHcIKvq2vbibrCog4Y8G0zmOxgQUVEiTqwR5hJVMCoDsN1vFvI5JvwD37hjueZ1C4l2FyGnfpE0A==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@jest/diff-sequences": "30.0.1", + "@jest/get-type": "30.0.1", + "chalk": "^4.1.2", + "pretty-format": "30.0.5" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, "node_modules/@nx/workspace/node_modules/jsonc-parser": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", @@ -7399,9 +7474,9 @@ } }, "node_modules/@nx/workspace/node_modules/nx": { - "version": "21.2.3", - "resolved": "https://registry.npmjs.org/nx/-/nx-21.2.3.tgz", - "integrity": "sha512-2wL/2fSmIbRWn6zXaQ/g3kj5DfEaTw/aJkPr6ozJh8BUq5iYKE+tS9oh0PjsVVwN6Pybe80Lu+mn9RgWyeV3xw==", + "version": "21.3.5", + "resolved": "https://registry.npmjs.org/nx/-/nx-21.3.5.tgz", + "integrity": "sha512-iRAO7D7SkjhIM6y5xH8GO+ojTJB2QSIzG2xNBgbRwuTV6AxLBkq6sU5hFA0wNzD/LncUeEoJmRtHfbGXfQQORQ==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -7423,7 +7498,7 @@ "flat": "^5.0.2", "front-matter": "^4.0.2", "ignore": "^5.0.4", - "jest-diff": "^29.4.1", + "jest-diff": "^30.0.2", "jsonc-parser": "3.2.0", "lines-and-columns": "2.0.3", "minimatch": "9.0.3", @@ -7448,16 +7523,16 @@ "nx-cloud": "bin/nx-cloud.js" }, "optionalDependencies": { - "@nx/nx-darwin-arm64": "21.2.3", - "@nx/nx-darwin-x64": "21.2.3", - "@nx/nx-freebsd-x64": "21.2.3", - "@nx/nx-linux-arm-gnueabihf": "21.2.3", - "@nx/nx-linux-arm64-gnu": "21.2.3", - "@nx/nx-linux-arm64-musl": "21.2.3", - "@nx/nx-linux-x64-gnu": "21.2.3", - "@nx/nx-linux-x64-musl": "21.2.3", - "@nx/nx-win32-arm64-msvc": "21.2.3", - "@nx/nx-win32-x64-msvc": "21.2.3" + "@nx/nx-darwin-arm64": "21.3.5", + "@nx/nx-darwin-x64": "21.3.5", + "@nx/nx-freebsd-x64": "21.3.5", + "@nx/nx-linux-arm-gnueabihf": "21.3.5", + "@nx/nx-linux-arm64-gnu": "21.3.5", + "@nx/nx-linux-arm64-musl": "21.3.5", + "@nx/nx-linux-x64-gnu": "21.3.5", + "@nx/nx-linux-x64-musl": "21.3.5", + "@nx/nx-win32-arm64-msvc": "21.3.5", + "@nx/nx-win32-x64-msvc": "21.3.5" }, "peerDependencies": { "@swc-node/register": "^1.8.0", @@ -7515,6 +7590,22 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@nx/workspace/node_modules/pretty-format": { + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.5.tgz", + "integrity": "sha512-D1tKtYvByrBkFLe2wHJl2bwMJIiT8rW+XA+TiataH79/FszLQMrpGEvzUVkzPau7OCO0Qnrhpe87PqtOAIB8Yw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@jest/schemas": "30.0.5", + "ansi-styles": "^5.2.0", + "react-is": "^18.3.1" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, "node_modules/@nx/workspace/node_modules/readable-stream": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", @@ -8595,6 +8686,32 @@ "parse5": "^7.0.0" } }, + "node_modules/@types/jsdom/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/@types/jsdom/node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -8627,9 +8744,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "24.0.13", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.0.13.tgz", - "integrity": "sha512-Qm9OYVOFHFYg3wJoTSrz80hoec5Lia/dPp84do3X7dZvLikQvM1YpmvTBEdIr/e+U8HTkFjLHLnl78K/qjf+jQ==", + "version": "24.1.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.1.0.tgz", + "integrity": "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w==", "dev": true, "license": "MIT", "dependencies": { @@ -9854,13 +9971,13 @@ } }, "node_modules/axios": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.10.0.tgz", - "integrity": "sha512-/1xYAC4MP/HEG+3duIhFr4ZQXR4sQXOIe+o6sdqzeykGLx6Upp/1p8MHqhINOvGeP7xyNHe7tsiJByc4SSVUxw==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.11.0.tgz", + "integrity": "sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.6", - "form-data": "^4.0.0", + "form-data": "^4.0.4", "proxy-from-env": "^1.1.0" } }, @@ -12712,9 +12829,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.182", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.182.tgz", - "integrity": "sha512-Lv65Btwv9W4J9pyODI6EWpdnhfvrve/us5h1WspW8B2Fb0366REPtY3hX7ounk1CkV/TBjWCEvCBBbYbmV0qCA==", + "version": "1.5.190", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.190.tgz", + "integrity": "sha512-k4McmnB2091YIsdCgkS0fMVMPOJgxl93ltFzaryXqwip1AaxeDqKCGLxkXODDA5Ab/D+tV5EL5+aTx76RvLRxw==", "license": "ISC" }, "node_modules/electron-winstaller": { @@ -12756,9 +12873,9 @@ } }, "node_modules/electron/node_modules/@types/node": { - "version": "20.19.7", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.7.tgz", - "integrity": "sha512-1GM9z6BJOv86qkPvzh2i6VW5+VVrXxCLknfmTkWEqz+6DqosiY28XUWCTmBcJ0ACzKqx/iwdIREfo1fwExIlkA==", + "version": "20.19.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.9.tgz", + "integrity": "sha512-cuVNgarYWZqxRJDQHEB58GEONhOK79QVR/qYx4S7kcUObQvUwvFnYxJuuHUKm2aieN9X3yZB4LZsuYNU1Qphsw==", "dev": true, "license": "MIT", "dependencies": { @@ -14006,9 +14123,9 @@ } }, "node_modules/form-data": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.3.tgz", - "integrity": "sha512-qsITQPfmvMOSAdeyZ+12I1c+CKSstAFAwu+97zrnWAbIr5u8wfsExUzCesVLC8NgHuRUqNN4Zy6UPWUTRGslcA==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", + "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -14119,9 +14236,9 @@ "license": "ISC" }, "node_modules/fs-monkey": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-1.0.6.tgz", - "integrity": "sha512-b1FMfwetIKymC0eioW7mTywihSQE4oLzQn1dB6rZB5fx/3NpNEdAWeCSMB+60/AeT0TCXsxzAlcYVEFCTAksWg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-1.1.0.tgz", + "integrity": "sha512-QMUezzXWII9EV5aTFXW1UBVUO77wYPpjqIF8/AviUCThNeSYZykpoTixUeaNNBwmCev0AMDWMAni+f8Hxb1IFw==", "dev": true, "license": "Unlicense" }, @@ -16343,9 +16460,9 @@ } }, "node_modules/jest-preset-angular": { - "version": "14.6.0", - "resolved": "https://registry.npmjs.org/jest-preset-angular/-/jest-preset-angular-14.6.0.tgz", - "integrity": "sha512-LGSKLCsUhtrs2dw6f7ega/HOS8/Ni/1gV+oXmxPHmJDLHFpM6cI78Monmz8Z1P87a/A4OwnKilxgPRr+6Pzmgg==", + "version": "14.6.1", + "resolved": "https://registry.npmjs.org/jest-preset-angular/-/jest-preset-angular-14.6.1.tgz", + "integrity": "sha512-7q5x42wKrsF2ykOwGVzcXpr9p1X4FQJMU/DnH1tpvCmeOm5XqENdwD/xDZug+nP6G8SJPdioauwdsK/PMY/MpQ==", "dev": true, "license": "MIT", "dependencies": { @@ -16825,6 +16942,19 @@ "node": ">= 6.0.0" } }, + "node_modules/jsdom/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/jsdom/node_modules/http-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", @@ -16854,6 +16984,19 @@ "node": ">= 6" } }, + "node_modules/jsdom/node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/jsesc": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", @@ -18421,9 +18564,9 @@ "license": "MIT" }, "node_modules/msgpackr": { - "version": "1.11.4", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.4.tgz", - "integrity": "sha512-uaff7RG9VIC4jacFW9xzL3jc0iM32DNHe4jYVycBcjUePT/Klnfj7pqtWJt9khvDFizmjN2TlYniYmSS2LIaZg==", + "version": "1.11.5", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.5.tgz", + "integrity": "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==", "dev": true, "license": "MIT", "optional": true, @@ -20266,11 +20409,12 @@ } }, "node_modules/parse5": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", - "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.0.tgz", + "integrity": "sha512-9m4m5GSgXjL4AjumKzq1Fgfp3Z8rsvjRNbnkVwfu2ImRqE5D0LnY2QfDen18FSY9C573YU5XxSapdHZTZ2WolA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "entities": "^6.0.0" }, @@ -20306,6 +20450,19 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/parse5-html-rewriting-stream/node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/parse5-sax-parser": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/parse5-sax-parser/-/parse5-sax-parser-7.0.0.tgz", @@ -20319,12 +20476,39 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/parse5-sax-parser/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/parse5-sax-parser/node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/parse5/node_modules/entities": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", "dev": true, "license": "BSD-2-Clause", + "peer": true, "engines": { "node": ">=0.12" }, @@ -24384,13 +24568,13 @@ } }, "node_modules/wait-on": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/wait-on/-/wait-on-8.0.3.tgz", - "integrity": "sha512-nQFqAFzZDeRxsu7S3C7LbuxslHhk+gnJZHyethuGKAn2IVleIbTB9I3vJSQiSR+DifUqmdzfPMoMPJfLqMF2vw==", + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/wait-on/-/wait-on-8.0.4.tgz", + "integrity": "sha512-8f9LugAGo4PSc0aLbpKVCVtzayd36sSCp4WLpVngkYq6PK87H79zt77/tlCU6eKCLqR46iFvcl0PU5f+DmtkwA==", "dev": true, "license": "MIT", "dependencies": { - "axios": "^1.8.2", + "axios": "^1.11.0", "joi": "^17.13.3", "lodash": "^4.17.21", "minimist": "^1.2.8", From 3f14fb79899e5115dd86913561e1ba62928bbdac Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Thu, 24 Jul 2025 18:20:32 +1000 Subject: [PATCH 35/41] Miscellanea from PR code self-review. --- engine/CSVParser.cc | 2 -- .../minsky-electron/src/app/managers/RecordingManager.ts | 4 ++-- gui-js/apps/minsky-electron/src/app/managers/StoreManager.ts | 5 +++-- .../ui-components/src/lib/import-csv/import-csv.component.ts | 1 - 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/engine/CSVParser.cc b/engine/CSVParser.cc index 4800faad2..922f83fb1 100644 --- a/engine/CSVParser.cc +++ b/engine/CSVParser.cc @@ -46,8 +46,6 @@ using ravel::getWholeLine; #include #include -//typedef boost::tokenizer Tokenizer; - namespace { /// An any with cached hash diff --git a/gui-js/apps/minsky-electron/src/app/managers/RecordingManager.ts b/gui-js/apps/minsky-electron/src/app/managers/RecordingManager.ts index 493a6464d..327560eff 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/RecordingManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/RecordingManager.ts @@ -24,7 +24,7 @@ export class RecordingManager { this.stopRecording(); const replayRecordingDialog = await WindowManager.showOpenDialog({ - defaultPath: 'models', + defaultPath: ':models', filters: [ { extensions: ['json'], name: 'JSON' }, { extensions: ['*'], name: 'All Files' }, @@ -48,7 +48,7 @@ export class RecordingManager { const index = dialog.showMessageBoxSync(options); if (options.buttons[index] === positiveResponseText) { - const saveDialog = await WindowManager.showSaveDialog({defaultPath: 'models'}); + const saveDialog = await WindowManager.showSaveDialog({defaultPath: ':models'}); const { canceled, filePath } = saveDialog; diff --git a/gui-js/apps/minsky-electron/src/app/managers/StoreManager.ts b/gui-js/apps/minsky-electron/src/app/managers/StoreManager.ts index ceb99eee6..2447ab5f8 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/StoreManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/StoreManager.ts @@ -1,5 +1,6 @@ import { defaultBackgroundColor } from '@minsky/shared'; import Store from 'electron-store'; +import {homedir} from 'node:os'; interface MinskyPreferences { godleyTableShowValues: boolean; @@ -26,8 +27,8 @@ class StoreManager { defaults: { recentFiles: [], backgroundColor: defaultBackgroundColor, - defaultModelDirectory: "", - defaultDataDirectory: "", + defaultModelDirectory: homedir(), + defaultDataDirectory: homedir(), preferences: { godleyTableShowValues: false, godleyTableOutputStyle: 'sign', diff --git a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts index 28a7eeea0..49ee78a72 100644 --- a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts +++ b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts @@ -191,7 +191,6 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni ) { super(); this.route.queryParams.pipe(takeUntil(this.destroy$)).subscribe((params) => { - electronService.log(JSON5.stringify(params)); this.csvDialog = new CSVDialog(params.csvDialog); this.systemWindowId = params.systemWindowId; this.isInvokedUsingToolbar = params.isInvokedUsingToolbar==="true"; From d68edc92bb29ebbb6c362562d01f90383eaf4c7c Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Fri, 25 Jul 2025 09:08:58 +1000 Subject: [PATCH 36/41] Code review nits. --- .../src/app/managers/WindowManager.ts | 14 ++++++++------ .../connect-database/connect-database.component.ts | 12 ++++++------ .../src/lib/connect-database/connect-database.html | 1 - .../src/lib/new-database/new-database.component.ts | 6 +++--- .../src/lib/new-database/new-database.html | 1 - .../ravel-select-horizontal-dim.component.ts | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/gui-js/apps/minsky-electron/src/app/managers/WindowManager.ts b/gui-js/apps/minsky-electron/src/app/managers/WindowManager.ts index 38f34eef5..771ceddd5 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/WindowManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/WindowManager.ts @@ -138,7 +138,7 @@ export class WindowManager { /// returns the directory key for the StoreManager. static processDefaultDirectory(options: OpenDialogOptions|SaveDialogOptions) { let splitDefaultPath=/([^\/]*)\/?(.*)/.exec(options.defaultPath); - let defaultType=splitDefaultPath[0]; + let defaultType=splitDefaultPath[1]; let defaultDirectoryKey=""; switch (defaultType) { @@ -153,7 +153,7 @@ export class WindowManager { if (defaultDirectoryKey) { let defaultDirectory=StoreManager.store.get(defaultDirectoryKey) as string; if (defaultDirectory) - options['defaultPath']=defaultDirectory; + options['defaultPath']=defaultDirectory+'/'+splitDefaultPath[2]; } return defaultDirectoryKey; } @@ -165,10 +165,11 @@ export class WindowManager { let options=args[args.length-1] as OpenDialogOptions; let defaultDirectoryKey=this.processDefaultDirectory(options); + let res: Electron.OpenDialogReturnValue; if (args.length>1) - var res=await dialog.showOpenDialog(args[0],options); + res=await dialog.showOpenDialog(args[0],options); else - var res=await dialog.showOpenDialog(options); + res=await dialog.showOpenDialog(options); if (!res.canceled && defaultDirectoryKey) { StoreManager.store.set(defaultDirectoryKey,dirname(res.filePaths[0])); } @@ -182,10 +183,11 @@ export class WindowManager { let options=args[args.length-1] as SaveDialogOptions; let defaultDirectoryKey=this.processDefaultDirectory(options); + let res: Electron.SaveDialogReturnValue; if (args.length>1) - var res=await dialog.showSaveDialog(args[0], options); + res=await dialog.showSaveDialog(args[0], options); else - var res=await dialog.showSaveDialog(options); + res=await dialog.showSaveDialog(options); if (!res.canceled && defaultDirectoryKey) { StoreManager.store.set(defaultDirectoryKey,dirname(res.filePath)); } diff --git a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts index 5db755a91..912f05fa1 100644 --- a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts +++ b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts @@ -20,10 +20,10 @@ import JSON5 from 'json5'; ], }) export class ConnectDatabaseComponent implements OnInit { - dbType="sqlite3"; + dbType: string="sqlite3"; connection: string; - table=""; - tables=[]; + table: string=""; + tables: string[]=[]; ravel: Ravel; constructor( private route: ActivatedRoute, @@ -38,7 +38,7 @@ export class ConnectDatabaseComponent implements OnInit { }); } - setDbType(event) { + setDbType(event: Event) { const target = event.target as HTMLSelectElement; this.dbType=target.value; } @@ -49,12 +49,12 @@ export class ConnectDatabaseComponent implements OnInit { this.tables=await this.ravel.db.tableNames(); } - setTable(event) { + setTable(event: Event) { const target = event.target as HTMLSelectElement; this.table=target.value; } - setConnection(event) { + setConnection(event: Event) { const target = event.target as HTMLSelectElement; this.connection=target.value; } diff --git a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html index e4520a3d0..86704b2f7 100644 --- a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html +++ b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.html @@ -5,7 +5,6 @@ - diff --git a/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts b/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts index 0adff31ca..6313bd29c 100644 --- a/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts +++ b/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts @@ -40,12 +40,12 @@ export class NewDatabaseComponent implements OnInit { }); } - setDbType(event) { + setDbType(event: Event) { const target = event.target as HTMLSelectElement; this.dbType=target.value; } - setConnection(event) { + setConnection(event: Event) { const target = event.target as HTMLSelectElement; this.connection=target.value; } @@ -56,7 +56,7 @@ export class NewDatabaseComponent implements OnInit { this.tables=await this.electronService.minsky.databaseIngestor.db.tableNames(); } - setTable(event) { + setTable(event: Event) { const target = event.target as HTMLSelectElement; this.table=target.value; } diff --git a/gui-js/libs/ui-components/src/lib/new-database/new-database.html b/gui-js/libs/ui-components/src/lib/new-database/new-database.html index b3398c7a9..2da0faac9 100644 --- a/gui-js/libs/ui-components/src/lib/new-database/new-database.html +++ b/gui-js/libs/ui-components/src/lib/new-database/new-database.html @@ -5,7 +5,6 @@ - diff --git a/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.component.ts b/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.component.ts index 1331a3436..59e614118 100644 --- a/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.component.ts +++ b/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.component.ts @@ -41,7 +41,7 @@ export class RavelSelectHorizontalDimComponent implements OnInit { this.dataCols=await this.ravel.db.numericalColumnNames(); } - clickDim(event) { + clickDim(event: Event) { const target = event.target as HTMLInputElement; if (target.checked) this.horizontalDimCols.add(target.name); From 52d31f9a4350e2ccfb819b599ade7c7eaed5f616 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Fri, 25 Jul 2025 09:42:41 +1000 Subject: [PATCH 37/41] Codereview nits. --- .../new-database/new-database.component.ts | 27 ++++++------------- .../src/lib/new-database/new-database.html | 14 ++++++++-- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts b/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts index 6313bd29c..81cbe1919 100644 --- a/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts +++ b/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts @@ -23,11 +23,11 @@ import JSON5 from 'json5'; MatOptionModule, ], }) -export class NewDatabaseComponent implements OnInit { - dbType="sqlite3"; +export class NewDatabaseComponent { + dbType: string="sqlite3"; connection: string; - table=""; - tables=[]; + table: string=""; + tables: string[]=[]; constructor( private route: ActivatedRoute, private electronService: ElectronService, @@ -35,11 +35,6 @@ export class NewDatabaseComponent implements OnInit { ) { } - ngOnInit(): void { - this.route.queryParams.subscribe((params) => { - }); - } - setDbType(event: Event) { const target = event.target as HTMLSelectElement; this.dbType=target.value; @@ -62,8 +57,7 @@ export class NewDatabaseComponent implements OnInit { } setTableInput(event) { - let input=document.getElementById("table") as HTMLInputElement; - this.table=input.value=event?.option?.value; + this.table=event?.option?.value; } async selectFile() { @@ -86,12 +80,8 @@ export class NewDatabaseComponent implements OnInit { if (filePath) this.connection=`db=${filePath}`; } - if (this.connection) { - let connectionInput=document.getElementById("connection") as HTMLInputElement; - connectionInput.hidden=false; - connectionInput.value=this.connection; - this.getTables(); - } + if (this.connection) + await this.getTables(); } connect() { @@ -100,8 +90,7 @@ export class NewDatabaseComponent implements OnInit { return; } this.electronService.minsky.databaseIngestor.db.connect(this.dbType,this.connection,this.table); - // TODO - set dropTable according to whether an existing table is selected, or a new given - let dropTable=!this.tables.includes(this.tables); + let dropTable=!this.tables.includes(this.table); this.electronService.invoke(events.IMPORT_CSV_TO_DB, {dropTable}); this.closeWindow(); } diff --git a/gui-js/libs/ui-components/src/lib/new-database/new-database.html b/gui-js/libs/ui-components/src/lib/new-database/new-database.html index 2da0faac9..5a71b2fa1 100644 --- a/gui-js/libs/ui-components/src/lib/new-database/new-database.html +++ b/gui-js/libs/ui-components/src/lib/new-database/new-database.html @@ -11,14 +11,24 @@ - + + {{table}} From d89ca34a5af66f261df6e1fa86c4fa38c540d6b0 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Fri, 25 Jul 2025 10:06:21 +1000 Subject: [PATCH 38/41] Codereview nits. --- .../minsky-electron/src/app/managers/ContextMenuManager.ts | 4 ++-- gui-js/libs/shared/src/lib/constants/constants.ts | 2 +- .../ui-components/src/lib/import-csv/import-csv.component.ts | 2 +- .../ravel-select-horizontal-dim.scss | 3 +-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts b/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts index 47b9bbbd1..7a25fb957 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/ContextMenuManager.ts @@ -919,7 +919,7 @@ export class ContextMenuManager { WindowManager.createPopupWindowWithRouting({ title: 'Set numerical axes', url: '#/headless/ravel-select-horizontal-dim', - height: 180, + height: 400, width: 400, }) }, @@ -1388,7 +1388,7 @@ export class ContextMenuManager { private static async initContextMenuForCSVImport(event: IpcMainEvent, variableValue: string, row: number, col: number) { - const refresh=()=>event.sender.send(events.CSV_IMPORT_REFRESH); + const refresh=()=>event.sender.send(events.REFRESH_CSV_IMPORT); const value=new VariableValue(variableValue); var menu=Menu.buildFromTemplate([ new MenuItem({ diff --git a/gui-js/libs/shared/src/lib/constants/constants.ts b/gui-js/libs/shared/src/lib/constants/constants.ts index f67456708..f3248810e 100644 --- a/gui-js/libs/shared/src/lib/constants/constants.ts +++ b/gui-js/libs/shared/src/lib/constants/constants.ts @@ -24,7 +24,6 @@ export const events = { CLOSE_WINDOW: 'close-window', CONTEXT_MENU: 'context-menu', CREATE_MENU_POPUP: 'create-menu-popup', - CSV_IMPORT_REFRESH: 'csv-import-refresh', CURRENT_TAB_MOVE_TO: 'current-tab-move-to', CURRENT_TAB_POSITION: 'current-tab-position', CURSOR_BUSY: 'cursor-busy', @@ -52,6 +51,7 @@ export const events = { RECORD: 'record', RECORDING_REPLAY: 'recording-replay', RECORDING_STATUS_CHANGED: 'recording-status-changed', + REFRESH_CSV_IMPORT: 'refresh-csv-import', REPLAY_RECORDING: 'replay-recording', RESET_ZOOM: 'reset-zoom', RESET_SCROLL: 'reset-scroll', diff --git a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts index 49ee78a72..4c67485a0 100644 --- a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts +++ b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts @@ -222,7 +222,7 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni }), }); - this.electronService.on(events.CSV_IMPORT_REFRESH, async e => { + this.electronService.on(events.REFRESH_CSV_IMPORT, async e => { await this.getCSVDialogSpec(); this.updateColumnTypes(); this.updateForm(); diff --git a/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.scss b/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.scss index ec42e0843..908ca4bf0 100644 --- a/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.scss +++ b/gui-js/libs/ui-components/src/lib/ravel-select-horizontal-dim/ravel-select-horizontal-dim.scss @@ -6,9 +6,8 @@ } .selectors { - height: calc(100vh + 100px); + height: 100vh; //calc(100vh + 100px); overflow-y: auto; - //padding: -10px; } .dim-selector { From 45394722f9342d07a8a89e0f2c9272539a6aab6e Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Fri, 25 Jul 2025 10:13:42 +1000 Subject: [PATCH 39/41] Codereview nits. --- .../src/app/managers/CommandsManager.ts | 7 ++++++- .../lib/connect-database/connect-database.component.ts | 10 ++-------- .../src/lib/new-database/new-database.component.ts | 3 +-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts b/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts index 97512285b..622de2e1e 100644 --- a/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts +++ b/gui-js/apps/minsky-electron/src/app/managers/CommandsManager.ts @@ -913,7 +913,12 @@ export class CommandsManager { this.setLogSimulationCheckmark(true); } - /// @param dropTable whether to create a new table if loading to a database + /** + * Opens CSV import dialog + * @param csvDialog - CSV dialog configuration object + * @param isInvokedUsingToolbar - Whether invoked from toolbar (affects cleanup) + * @param dropTable - Whether to drop existing table (for database imports) + */ static async importCSV(csvDialog: CSVDialog, isInvokedUsingToolbar = false, dropTable=false) { const itemInfo: CanvasItem={classType: ClassType.Variable, id: csvDialog.$prefix(), displayContents: false}; if (!WindowManager.focusIfWindowIsPresent(itemInfo.id)) { diff --git a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts index 912f05fa1..4f5dbea84 100644 --- a/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts +++ b/gui-js/libs/ui-components/src/lib/connect-database/connect-database.component.ts @@ -1,4 +1,4 @@ -import { ChangeDetectorRef, Component, OnInit, SimpleChanges } from '@angular/core'; +import { ChangeDetectorRef, Component, SimpleChanges } from '@angular/core'; import { FormsModule, } from '@angular/forms'; import { ActivatedRoute } from '@angular/router'; import { ElectronService } from '@minsky/core'; @@ -6,7 +6,6 @@ import { Ravel} from '@minsky/shared'; import { MatButtonModule } from '@angular/material/button'; import { OpenDialogOptions } from 'electron'; import { CommonModule } from '@angular/common'; // Often useful for ngIf, ngFor -import JSON5 from 'json5'; @Component({ selector: 'connect-database', @@ -19,7 +18,7 @@ import JSON5 from 'json5'; MatButtonModule, ], }) -export class ConnectDatabaseComponent implements OnInit { +export class ConnectDatabaseComponent { dbType: string="sqlite3"; connection: string; table: string=""; @@ -33,11 +32,6 @@ export class ConnectDatabaseComponent implements OnInit { this.ravel=new Ravel(this.electronService.minsky.canvas.item); } - ngOnInit(): void { - this.route.queryParams.subscribe((params) => { - }); - } - setDbType(event: Event) { const target = event.target as HTMLSelectElement; this.dbType=target.value; diff --git a/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts b/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts index 81cbe1919..b64007b32 100644 --- a/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts +++ b/gui-js/libs/ui-components/src/lib/new-database/new-database.component.ts @@ -1,4 +1,4 @@ -import { ChangeDetectorRef, Component, OnInit, SimpleChanges } from '@angular/core'; +import { ChangeDetectorRef, Component, SimpleChanges } from '@angular/core'; import { FormsModule, } from '@angular/forms'; import { ActivatedRoute } from '@angular/router'; import { ElectronService } from '@minsky/core'; @@ -8,7 +8,6 @@ import { MatButtonModule } from '@angular/material/button'; import { MatOptionModule } from '@angular/material/core'; import { OpenDialogOptions, SaveDialogOptions } from 'electron'; import { CommonModule } from '@angular/common'; // Often useful for ngIf, ngFor -import JSON5 from 'json5'; @Component({ selector: 'new-database', From 7973ad92d40878f2a89de54f91b211390302b1a7 Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Fri, 25 Jul 2025 11:51:50 +1000 Subject: [PATCH 40/41] Update RavelCAPI ref. --- RavelCAPI | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RavelCAPI b/RavelCAPI index d3326d2ec..755b1c5b6 160000 --- a/RavelCAPI +++ b/RavelCAPI @@ -1 +1 @@ -Subproject commit d3326d2ecc55ff4e3693b4d565c96b575c95b47b +Subproject commit 755b1c5b68eb9798fe003c7ec3e261735e3e1448 From 81c02d4f6536efd72545f1ba84e910d113d45b7e Mon Sep 17 00:00:00 2001 From: Russell Standish Date: Fri, 25 Jul 2025 16:29:02 +1000 Subject: [PATCH 41/41] Import-csv needs to set spec before calling importFromCSV --- .../ui-components/src/lib/import-csv/import-csv.component.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts index 4c67485a0..997e79cdb 100644 --- a/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts +++ b/gui-js/libs/ui-components/src/lib/import-csv/import-csv.component.ts @@ -552,6 +552,8 @@ export class ImportCsvComponent extends Zoomable implements OnInit, AfterViewIni if (this.dialogState.spec.dataCols.length === 0) this.dialogState.spec.counter = true; + this.csvDialog.spec.$properties(this.dialogState.spec); + if (!this.files || !this.files[0]) this.files=[this.url.value]; if (this.files && (this.dropTable.value || this.newTable)) this.electronService.minsky.databaseIngestor.createTable(this.files[0]);