From 122bf196294aedabc039bb789d032592fbc79d92 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Mon, 1 Dec 2025 22:17:22 +0000 Subject: [PATCH 01/13] WIP - write file --- .../Standard/Database/0.0.0-dev/src/SQL.enso | 16 +++ .../0.0.0-dev/src/DuckDB_Connection.enso | 115 +++++++++++++++++- 2 files changed, 129 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/SQL.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/SQL.enso index 88fbe7248ba8..32457d21f772 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/SQL.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/SQL.enso @@ -241,6 +241,22 @@ type SQL_Statement - `internal_fragments`: A vector of SQL code fragments. private Value (internal_fragments:(Vector SQL_Fragment)) + ## --- + private: true + advanced: true + --- + Adds a prefix and suffix to the SQL statement. + + ## Arguments + - `prefix`: The prefix to add. + - `suffix`: The suffix to add. + surround self (prefix : Text = "") (suffix : Text = "") -> SQL_Statement = + if prefix=="" && suffix=="" then self else + prefix_fragment = if prefix=="" then [] else [SQL_Fragment.Code_Part prefix] + suffix_fragment = if suffix=="" then [] else [SQL_Fragment.Code_Part suffix] + new_fragments = prefix_fragment + self.internal_fragments + suffix_fragment + SQL_Statement.Value new_fragments + ## --- private: true advanced: true diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso index e0f7cbfb0dc3..9314aa0839ce 100644 --- a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso @@ -2,6 +2,7 @@ from Standard.Base import all import Standard.Base.Enso_Cloud.Data_Link_Helpers import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.Runtime.Context +import Standard.Base.System.File.Generic.Writable_File.Writable_File import Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data import Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Header from Standard.Base.Metadata.Choice import Option @@ -25,6 +26,7 @@ from Standard.Database.Errors import SQL_Error, Table_Already_Exists, Table_Not_ from Standard.Database.Internal.Upload.Helpers.Default_Arguments import first_column_name_in_structure from Standard.Database.Internal.Upload.Operations.Create import create_table_implementation +import project.DuckDB.DuckDB import project.Internal.DuckDB_Dialect import project.Internal.DuckDB_Entity_Naming_Properties import project.Internal.DuckDB_Type_Mapping.DuckDB_Type_Mapping @@ -264,7 +266,8 @@ type DuckDB_Connection Nothing escaped_as = as.replace '"' '""' Data_Link_Helpers.as_file path file-> - query = 'CREATE' + (if replace_if_present then ' OR REPLACE' else "") + ' TABLE "' + escaped_as + '" AS SELECT * FROM \'' + file.path + '\';' + escaped_path = file.path.replace "'" "''" + query = 'CREATE' + (if replace_if_present then ' OR REPLACE' else "") + ' TABLE "' + escaped_as + '" AS SELECT * FROM \'' + escaped_path + '\';' Context.Output.with_enabled <| create_table = self.execute_update query create_table.if_not_error <| @@ -301,7 +304,8 @@ type DuckDB_Connection Nothing escaped_as = as.replace '"' '""' Data_Link_Helpers.as_file path file-> - query = 'CREATE' + (if replace_if_present then ' OR REPLACE' else "") + ' TABLE "' + escaped_as + '" AS SELECT * FROM ST_Read(\'' + file.path + '\');' + escaped_path = file.path.replace "'" "''" + query = 'CREATE' + (if replace_if_present then ' OR REPLACE' else "") + ' TABLE "' + escaped_as + '" AS SELECT * FROM ST_Read(\'' + escaped_path + '\');' Context.Output.with_enabled <| create_table = self.execute_update query create_table.if_not_error <| @@ -648,6 +652,108 @@ type DuckDB_Connection to_js_object self = JS_Object.from_pairs <| [["type", "DuckDB_Connection"], ["links", self.connection.tables.at "Name" . to_vector]] + ## --- + icon: data_output + --- + Writes the current database to a new database file to the specified path + and then returns a new connection to the written database. + + ## Arguments + - `path`: The path to write the database to. + - `schema_only`: If set to `True`, only the schema will be written, + without any data. Defaults to `False`. + + ## Returns + A new DuckDB_Connection connected to the written database. + write_database : Text -> Boolean -> DuckDB_Connection + write_database self (path:Writable_File=Missing_Argument.throw "path") schema_only:Boolean=False = + ## Check a local file (to do add support for non-local files later) + if path.is_local.not then Error.throw (Illegal_Argument.Error "Only local files are currently supported for writing DuckDB databases. Write to a local file and then copy it to the desired location.") + + ## Check if it exists + if path.exists then Error.throw (Illegal_Argument.Error "The specified file already exists. Please provide a different path or delete the existing file.") + + ## Check Execution Context + Context.Output.if_enabled disabled_message="As writing is disabled, cannot write the database. Press the Write button ▶ to perform the operation." panic=False <| + Nothing + + ## Write the database + escaped_path = path.path.replace "'" "''" + query = "ATTACH '" + escaped_path + "' AS copy_to_database; COPY FROM DATABASE " + self.database + " TO copy_to_database" + (if schema_only then " (SCHEMA);" else ";") + " DETACH copy_to_database;" + result = self.execute_update query + result.if_not_error <| + ## Return a new connection + DuckDB.From_File path schema=self.schema . connect + + ## --- + icon: data_output + --- + Writes a table to a file using DuckDB's writing capabilities. + + ## Arguments + - `table`: The input table name or `DB_Table` to write. + - `path`: The path to write the database to. + - `on_existing_file`: The behavior to use if the file already exists. + Defaults to `Backup`, which will create a backup of the existing file. + + ## Returns + A file pointing at the written file. + @query make_table_name_with_schema_selector schema_black_list=schema_black_list + @on_existing_file Existing_File_Behavior.widget include_append=False + write_file : (SQL_Query_With_Schema | SQL_Query | Table) -> Text -> Existing_File_Behavior -> DuckDB_Connection + write_file self (query:SQL_Query_With_Schema | SQL_Query | Table=Missing_Argument.throw "query") (path:Writable_File=Missing_Argument.throw "path") on_existing_file:Existing_File_Behavior=..Backup = + ## Check a local file (to do add support for non-local files later) + if path.is_local.not then Error.throw (Illegal_Argument.Error "Only local files are currently supported for writing via DuckDB. Write to a local file and then copy it to the desired location.") + + ## Check Execution Context + Context.Output.if_enabled disabled_message="As writing is disabled, cannot write the file. Press the Write button ▶ to perform the operation." panic=False <| + Nothing + + ## Rationalise the query to a DB_Table + if table.is_error then table + table = case query of + _ : In_Memory_Table -> Error.throw (Illegal_Argument.Error "Load the table into DuckDB first using `bulk_load`.") + _ : DB_Table -> if Meta.is_same_object self query.connection then query else + Error.throw (Illegal_Argument.Error "Cannot write a DB_Table from a different connection. Load the table into this connection first.") + _ -> query.to_db_table + + ## Check Existing_File_Behavior + if path.exists then case on_existing_file of + Existing_File_Behavior.Append -> Error.throw (Illegal_Argument.Error "DuckDB does not support appending to existing files.") + Existing_File_Behavior.Error -> Error.throw (File_Error.Already_Exists path) + _ -> Nothing + write_action = if path.exists.not || on_existing_file != Existing_File_Behavior then file->action-> action file else + file-> action-> + tmp_file = File.create_temporary_file prefix="duckdb_write_" suffix=".tmp" + path.copy_to tmp_file overwrite=True + path.delete + + ## Convert any Panic to an Error allowing us to restore the original file on failure + result = Panic.catch Any (action path) caught_panic-> Error.throw caught_panic.payload + + case result.is_error then + True -> tmp_file.copy_to path overwrite=True + False -> tmp_file.copy_to (path.parent / (path.name+".bak")) overwrite=True + + result + + ## Write the file + write_action path file-> + escaped_path = file.path.replace "'" "''" + sql_query = query.to_sql.surround 'COPY (' ') TO \'' + escaped_path + '\';' + result = self.execute_update sql_query + result.if_not_error <| + path + + ## --- + group: Standard.Base.Metadata + icon: drive + --- + Gets a list of the available drivers for writing spatial data. + spatial_drivers : Table + spatial_drivers self = + self.connection.execute_query "SELECT short_name, long_name, help_url FROM ST_Drivers() WHERE can_create=True" limit=..All_Rows write_operation=False + ## --- private: true --- @@ -682,3 +788,8 @@ private _column_names_widget connection single_choice:Boolean=False cache=Nothin col_names = db_table.column_names.map n-> Option n n.pretty if single_choice then Single_Choice [Option "" '"Column"']+col_names display=..Always else Multiple_Choice col_names display=..Always + +private _spatial_driver_widget connection = + names = connection.spatial_drivers.at "short_name" . to_vector + options = names.map n-> Option n n.pretty + Single_Choice options+[Option "" '"DriverName"'] display=..Always From 04e3c25f71a15d5ee2339945038ce55f989c8187 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Tue, 2 Dec 2025 11:18:08 +0000 Subject: [PATCH 02/13] Add missing conversion for OneDrive_File. --- .../lib/Standard/Microsoft/0.0.0-dev/src/OneDrive.enso | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/OneDrive.enso b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/OneDrive.enso index 4ebb3ba41cde..8b0c4adea435 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/OneDrive.enso +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/OneDrive.enso @@ -1,5 +1,6 @@ from Standard.Base import all import Standard.Base.Enso_Cloud.Data_Link.Data_Link +import Standard.Base.Enso_Cloud.Data_Link.Data_Link_From_File import Standard.Base.Enso_Cloud.Data_Link_Helpers import Standard.Base.Errors.File_Error.File_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument @@ -360,6 +361,11 @@ type OneDrive_File --- File_Like.from (that : OneDrive_File) = File_Like.Value that +## --- + private: true + --- +Data_Link_From_File.from (that : OneDrive_File) = Data_Link_From_File.Value that + private _check_directory file:OneDrive_File ~action = if file.is_directory then action else Error.throw <| Illegal_Argument.Error "The OneDrive_File must represent a folder: "+file.onedrive_path.path.to_text From e56e6dea5bc956557e2ffba31642ca38e3753ec8 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Tue, 2 Dec 2025 14:52:35 +0000 Subject: [PATCH 03/13] Rename other to passthru for clarity. --- .../Standard/Base/0.0.0-dev/src/Error.enso | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Error.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Error.enso index 95d39c2732b9..2e2d8556a01c 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Error.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Error.enso @@ -211,16 +211,23 @@ type Error advanced: true icon: warning --- - Returns the provided `other` value, unless `self` is an error. + Returns the provided `passthru` value, unless `self` is an error. The primary application of this function is to introduce a dataflow dependency between two otherwise unrelated operations. Very useful if one of the operations is performing a side-effect. + Aside from ensuring that any dataflow errors are propagated, the result will also inherit any warnings attached to any of the two inputs. ## Arguments - - `other`: The value to return if `self` is not an error. + - `passthru`: The value to return if `self` is not an error. + + ## Returns + `passthru` if `self` is not an error; otherwise, returns the error. + + ## Returns + `passthru` if `self` is not an error; otherwise, returns the error. ## Examples ### Writing to a file and returning the file object if all went well, or an @@ -230,8 +237,8 @@ type Error file.write "foo" . if_not_error file ``` if_not_error : Any -> Any - if_not_error self ~other = - const self other + if_not_error self ~passthru = + const self passthru ## --- private: true @@ -343,16 +350,20 @@ Any.is_error self -> Boolean = False advanced: true icon: warning --- - Returns the provided `other` value, unless `self` is an error. + Returns the provided `passthru` value, unless `self` is an error. The primary application of this function is to introduce a dataflow dependency between two otherwise unrelated operations. Very useful if one of the operations is performing a side-effect. + Aside from ensuring that any dataflow errors are propagated, the result will also inherit any warnings attached to any of the two inputs. ## Arguments - - `other`: The value to return if `self` is not an error. + - `passthru`: The value to return if `self` is not an error. + + ## Returns + `passthru` if `self` is not an error; otherwise, returns the error. ## Examples ### Writing to a file and returning the file object if all went well, or an @@ -361,4 +372,4 @@ Any.is_error self -> Boolean = False ``` file.write "foo" . if_not_error file ``` -Any.if_not_error self ~other = other +Any.if_not_error self ~passthru = passthru From 4df77d6e5ac9076aa00fd330cce1db5846a5a574 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Tue, 2 Dec 2025 18:49:41 +0000 Subject: [PATCH 04/13] Allow URIs to actually work with a space or an escaped space. --- .../Base/0.0.0-dev/src/Network/URI.enso | 4 +-- .../org/enso/base/net/URITransformer.java | 29 +++++++++++++++++++ test/Base_Tests/src/Network/URI_Spec.enso | 5 +++- test/Cloud_Tests/src/Network/URI_Spec.enso | 5 +++- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso index 6af014f79cbd..e7db8dac94eb 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso @@ -71,12 +71,12 @@ type URI ``` parse : Text -> URI ! Syntax_Error parse uri:Text = if uri == "" then Error.throw (Syntax_Error.Error "URI cannot be empty.") else - raw_uri = Panic.catch URISyntaxException (URI.Value (Java_URI.new uri) []) caught_panic-> + raw_uri = Panic.catch URISyntaxException (URI.Value (URITransformer.parse uri) []) caught_panic-> query_index = uri.index_of '?' result = if query_index.is_nothing then Nothing else new_uri = (uri.take query_index+1) + (URITransformer.encodeQuery (uri.drop query_index+1)) warning = Invalid_Query_String.Warning (uri.drop query_index+1) - Panic.catch URISyntaxException (Warning.attach warning URI.Value (Java_URI.new new_uri) []) _->Nothing + Panic.catch URISyntaxException (Warning.attach warning URI.Value (URITransformer.parse new_uri) []) _->Nothing if result.is_nothing.not then result else message = caught_panic.payload.getMessage diff --git a/std-bits/base/src/main/java/org/enso/base/net/URITransformer.java b/std-bits/base/src/main/java/org/enso/base/net/URITransformer.java index 14c366935c1e..1f4e3a90a284 100644 --- a/std-bits/base/src/main/java/org/enso/base/net/URITransformer.java +++ b/std-bits/base/src/main/java/org/enso/base/net/URITransformer.java @@ -1,12 +1,41 @@ package org.enso.base.net; import java.net.URI; +import java.net.URISyntaxException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; /** Utilities for building and transforming URIs. */ public class URITransformer { + /** + * Parses a URI from a string. Replaces spaces with %20 to ensure valid encoding. + * + * @param uri the URI string to parse + * @return the parsed URI + */ + public static URI parse(String uri) throws URISyntaxException { + int index = uri.indexOf('?'); + if (index >= 0) { + String uriWithoutQuery = uri.substring(0, index + 1).replaceAll(" ", "%20"); + uri = uriWithoutQuery + uri.substring(index + 1); + } else { + uri = uri.replaceAll(" ", "%20"); + } + var url = new URI(uri); + // URI class does not encode the path if single argument constructor is used. + if (url.getPath().equals(url.getRawPath())) { + return url; + } + + index = uri.indexOf(url.getRawPath()); + var newUriString = + uri.substring(0, index) + + url.getPath().replace(" ", "%20") + + uri.substring(index + url.getRawPath().length()); + return new URI(newUriString); + } + /** Removes query parameters from the given URI. */ public static URI removeQueryParameters(URI uri) { return buildUriFromParts( diff --git a/test/Base_Tests/src/Network/URI_Spec.enso b/test/Base_Tests/src/Network/URI_Spec.enso index 69b9d8de7675..1928e2e6d7d8 100644 --- a/test/Base_Tests/src/Network/URI_Spec.enso +++ b/test/Base_Tests/src/Network/URI_Spec.enso @@ -70,10 +70,13 @@ add_specs suite_builder = addr.fragment.should_equal Nothing addr.raw_user_info.should_equal "%D0%9B%D0%B8%D0%BD%D1%83%D1%81:pass" addr.raw_authority.should_equal "%D0%9B%D0%B8%D0%BD%D1%83%D1%81:pass@ru.wikipedia.org" - addr.raw_path.should_equal "/wiki/%D0%AF%D0%B4%D1%80%D0%BE_Linux" addr.raw_query.should_equal "%D0%9A%D0%BE%D0%B4" addr.raw_fragment.should_equal Nothing + group_builder.specify "should cope with spaces in URIs" <| + addr = URI.parse "https://graph.microsoft.com/v1.0/me/drive/root:/My Documents/Enso Project Plan.docx:/content" + addr.path.should_equal "/v1.0/me/drive/root:/My Documents/Enso Project Plan.docx:/content" + group_builder.specify "should automatically URI encode the query string if needed with a warning" <| addr = URI.parse 'https://httpbin.org/get?foo=bar&baz="a&c=d"%20?' addr.to_text . should_equal "https://httpbin.org/get?foo=bar&baz=%22a&c=d%22%2520%3F" diff --git a/test/Cloud_Tests/src/Network/URI_Spec.enso b/test/Cloud_Tests/src/Network/URI_Spec.enso index cc3e6e44e353..d485a2bccca5 100644 --- a/test/Cloud_Tests/src/Network/URI_Spec.enso +++ b/test/Cloud_Tests/src/Network/URI_Spec.enso @@ -76,10 +76,13 @@ add_specs suite_builder = addr.fragment.should_equal Nothing addr.raw_user_info.should_equal "%D0%9B%D0%B8%D0%BD%D1%83%D1%81:pass" addr.raw_authority.should_equal "%D0%9B%D0%B8%D0%BD%D1%83%D1%81:pass@ru.wikipedia.org" - addr.raw_path.should_equal "/wiki/%D0%AF%D0%B4%D1%80%D0%BE_Linux" addr.raw_query.should_equal "%D0%9A%D0%BE%D0%B4" addr.raw_fragment.should_equal Nothing + group_builder.specify "should cope with spaces in URIs" <| + addr = URI.parse "https://graph.microsoft.com/v1.0/me/drive/root:/My Documents/Enso Project Plan.docx:/content" + addr.path.should_equal "/v1.0/me/drive/root:/My Documents/Enso Project Plan.docx:/content" + group_builder.specify "should automatically URI encode the query string if needed with a warning" <| addr = URI.parse 'https://httpbin.org/get?foo=bar&baz="a&c=d"%20?' addr.to_text . should_equal "https://httpbin.org/get?foo=bar&baz=%22a&c=d%22%2520%3F" From 8379d0c63d5b9b37a639fc41a9343b65212a334e Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Tue, 2 Dec 2025 19:11:47 +0000 Subject: [PATCH 05/13] Working write file. Need to add format control and write_spatial_file. --- .../0.0.0-dev/src/Connection/Connection.enso | 11 ++++++++++ .../0.0.0-dev/src/DuckDB_Connection.enso | 22 +++++++++---------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso index ba825aa00b68..e3001166dcb4 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso @@ -584,6 +584,17 @@ type Connection key_column_names = selected.sort 1 . at 0 . to_vector if key_column_names.is_empty then Nothing else key_column_names + ## --- + private: true + --- + Checks that the DB_Table is using a specified connection.DB_Table + + ## Arguments: + - `connection`: Connection to check against + - `db_table`: DB_Table to check + check_on_connection connection db_table:DB_Table = + Meta.is_same_object connection db_table.connection + ## --- private: true --- diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso index 9314aa0839ce..c8077db2fab8 100644 --- a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso @@ -1,5 +1,6 @@ from Standard.Base import all import Standard.Base.Enso_Cloud.Data_Link_Helpers +import Standard.Base.Errors.File_Error.File_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.Runtime.Context import Standard.Base.System.File.Generic.Writable_File.Writable_File @@ -701,7 +702,7 @@ type DuckDB_Connection @query make_table_name_with_schema_selector schema_black_list=schema_black_list @on_existing_file Existing_File_Behavior.widget include_append=False write_file : (SQL_Query_With_Schema | SQL_Query | Table) -> Text -> Existing_File_Behavior -> DuckDB_Connection - write_file self (query:SQL_Query_With_Schema | SQL_Query | Table=Missing_Argument.throw "query") (path:Writable_File=Missing_Argument.throw "path") on_existing_file:Existing_File_Behavior=..Backup = + write_file self (query:(SQL_Query_With_Schema | SQL_Query | Table)=Missing_Argument.throw "query") (path:Writable_File=Missing_Argument.throw "path") on_existing_file:Existing_File_Behavior=..Backup = ## Check a local file (to do add support for non-local files later) if path.is_local.not then Error.throw (Illegal_Argument.Error "Only local files are currently supported for writing via DuckDB. Write to a local file and then copy it to the desired location.") @@ -710,10 +711,10 @@ type DuckDB_Connection Nothing ## Rationalise the query to a DB_Table - if table.is_error then table + if query.is_error then query table = case query of _ : In_Memory_Table -> Error.throw (Illegal_Argument.Error "Load the table into DuckDB first using `bulk_load`.") - _ : DB_Table -> if Meta.is_same_object self query.connection then query else + db_table : DB_Table -> if Connection.check_on_connection self db_table then query else Error.throw (Illegal_Argument.Error "Cannot write a DB_Table from a different connection. Load the table into this connection first.") _ -> query.to_db_table @@ -723,24 +724,23 @@ type DuckDB_Connection Existing_File_Behavior.Error -> Error.throw (File_Error.Already_Exists path) _ -> Nothing write_action = if path.exists.not || on_existing_file != Existing_File_Behavior then file->action-> action file else - file-> action-> + file -> action -> tmp_file = File.create_temporary_file prefix="duckdb_write_" suffix=".tmp" - path.copy_to tmp_file overwrite=True - path.delete + file.move_to tmp_file replace_existing=True ## Convert any Panic to an Error allowing us to restore the original file on failure - result = Panic.catch Any (action path) caught_panic-> Error.throw caught_panic.payload + result = Panic.catch Any (action file) caught_panic-> Error.throw caught_panic.payload - case result.is_error then - True -> tmp_file.copy_to path overwrite=True - False -> tmp_file.copy_to (path.parent / (path.name+".bak")) overwrite=True + case result.is_error of + True -> tmp_file.move_to file replace_existing=True + False -> tmp_file.move_to (file.parent / (file.name+".bak")) replace_existing=True result ## Write the file write_action path file-> escaped_path = file.path.replace "'" "''" - sql_query = query.to_sql.surround 'COPY (' ') TO \'' + escaped_path + '\';' + sql_query = table.to_sql.surround "COPY (" ") TO '" + escaped_path + "';'" result = self.execute_update sql_query result.if_not_error <| path From 59e2e75f7ad412241268c83eede57f1c2eccc444 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 3 Dec 2025 10:45:34 +0000 Subject: [PATCH 06/13] Add new Error.return_if_error causing function to exit if the input is an error. --- .../Standard/Base/0.0.0-dev/src/Error.enso | 10 ++++++++ .../0.0.0-dev/src/DuckDB_Connection.enso | 25 +++++++++++-------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Error.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Error.enso index 2e2d8556a01c..69eb4cc027e7 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Error.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Error.enso @@ -5,6 +5,7 @@ import project.Data.Vector.Vector import project.Errors.Common.No_Such_Conversion import project.Errors.Wrapped_Error.Wrapped_Error import project.Meta +import project.Nothing.Nothing import project.Panic.Panic import project.Runtime.Stack_Trace_Element from project.Data.Boolean import Boolean, False, True @@ -207,6 +208,15 @@ type Error is_error : Boolean is_error self = True + ## --- + private: true + --- + A function that will cause an error to be thrown in a Java style + return_if_error : Any -> Nothing + return_if_error flag = + if flag.is_error then flag + Nothing + ## --- advanced: true icon: warning diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso index c8077db2fab8..ab72c1047f0c 100644 --- a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso @@ -308,16 +308,21 @@ type DuckDB_Connection escaped_path = file.path.replace "'" "''" query = 'CREATE' + (if replace_if_present then ' OR REPLACE' else "") + ' TABLE "' + escaped_as + '" AS SELECT * FROM ST_Read(\'' + escaped_path + '\');' Context.Output.with_enabled <| - create_table = self.execute_update query - create_table.if_not_error <| - ## Get the spatial columns and add RTree indexes on them - describe = self.execute_query ('DESCRIBE "' + escaped_as + '";') - spatial_columns = describe.filter "column_type" (..Equal "GEOMETRY") . at "column_name" . to_vector - with_spatial_indexes = spatial_columns.fold as table->column-> - if table.is_error then table else - self.create_spatial_index table column (as+"_"+column+"_RTREE_INDEX") replace_if_present - with_spatial_indexes.if_not_error <| - self.create_index as key_columns (escaped_as + '_INDEX') replace_if_present + created_table = self.execute_update query + Error.return_if_error created_table + + ## Get the spatial columns and add RTree indexes on them + describe = self.execute_query ('DESCRIBE "' + escaped_as + '";') + spatial_columns = describe.filter "column_type" (..Equal "GEOMETRY") . at "column_name" . to_vector + + ## Create spatial indices + with_spatial_indexes = spatial_columns.fold as table->column-> + Error.return_if_error table + self.create_spatial_index table column (as+"_"+column+"_RTREE_INDEX") replace_if_present + Error.return_if_error with_spatial_indexes + + ## Create any other user requested index. + self.create_index as key_columns (escaped_as + '_INDEX') replace_if_present ## --- group: Standard.Base.Database From cd5b5786c6d2b4b1b8386ae8489ff39536d9a1e5 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 3 Dec 2025 11:35:48 +0000 Subject: [PATCH 07/13] Docs. Fix Writable_File issue. --- .../Standard/Base/0.0.0-dev/docs/api/Error.md | 5 ++-- .../docs/api/Connection/Connection.md | 1 + .../Database/0.0.0-dev/docs/api/SQL.md | 1 + .../0.0.0-dev/docs/api/DuckDB_Connection.md | 3 +++ .../0.0.0-dev/src/DuckDB_Connection.enso | 26 ++++++++++++------- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/docs/api/Error.md b/distribution/lib/Standard/Base/0.0.0-dev/docs/api/Error.md index 2b901fad6ff6..a7c5b0995616 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/docs/api/Error.md +++ b/distribution/lib/Standard/Base/0.0.0-dev/docs/api/Error.md @@ -3,9 +3,10 @@ - type Error - catch self error_type:Standard.Base.Any.Any= handler:Standard.Base.Any.Any= -> Standard.Base.Any.Any - get_stack_trace_text self -> Standard.Base.Any.Any - - if_not_error self ~other:Standard.Base.Any.Any -> Standard.Base.Any.Any + - if_not_error self ~passthru:Standard.Base.Any.Any -> Standard.Base.Any.Any - is_error self -> Standard.Base.Any.Any - map_error self f:Standard.Base.Any.Any -> Standard.Base.Any.Any + - return_if_error flag:Standard.Base.Any.Any -> Standard.Base.Any.Any - stack_trace self -> Standard.Base.Any.Any - throw payload:Standard.Base.Any.Any -> Standard.Base.Any.Any - to_display_text self -> Standard.Base.Any.Any @@ -14,6 +15,6 @@ - look_for_wrapped_error error_type:Standard.Base.Any.Any= error_value:Standard.Base.Any.Any -> Standard.Base.Any.Any - Standard.Base.Any.Any.catch self error_type:Standard.Base.Any.Any= handler:Standard.Base.Any.Any= -> Standard.Base.Any.Any - Standard.Base.Any.Any.catch_primitive self handler:Standard.Base.Any.Any -> Standard.Base.Any.Any -- Standard.Base.Any.Any.if_not_error self ~other:Standard.Base.Any.Any -> Standard.Base.Any.Any +- Standard.Base.Any.Any.if_not_error self ~passthru:Standard.Base.Any.Any -> Standard.Base.Any.Any - Standard.Base.Any.Any.is_error self -> Standard.Base.Data.Boolean.Boolean - Standard.Base.Any.Any.map_error self ~f:Standard.Base.Any.Any -> Standard.Base.Any.Any diff --git a/distribution/lib/Standard/Database/0.0.0-dev/docs/api/Connection/Connection.md b/distribution/lib/Standard/Database/0.0.0-dev/docs/api/Connection/Connection.md index 0c5387b366c5..49a0e4bbba8e 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/docs/api/Connection/Connection.md +++ b/distribution/lib/Standard/Database/0.0.0-dev/docs/api/Connection/Connection.md @@ -3,6 +3,7 @@ - type Connection - Value jdbc_connection:Standard.Base.Any.Any dialect:Standard.Base.Any.Any type_mapping:Standard.Base.Any.Any entity_naming_properties:Standard.Database.Internal.Connection.Entity_Naming_Properties.Entity_Naming_Properties supports_large_update:(Standard.Base.Runtime.Ref.Ref Standard.Base.Data.Boolean.Boolean) hidden_table_registry:Standard.Database.Internal.Hidden_Table_Registry.Hidden_Table_Registry data_link_setup:(Standard.Database.Internal.Data_Link_Setup.Data_Link_Setup|Standard.Base.Nothing.Nothing)= - base_connection self -> Standard.Base.Any.Any + - check_on_connection connection:Standard.Base.Any.Any db_table:Standard.Database.DB_Table.DB_Table -> Standard.Base.Any.Any - close self -> Standard.Base.Any.Any - column_naming_helper self -> Standard.Base.Any.Any - create_literal_table self source:Standard.Table.Table.Table alias:Standard.Base.Data.Text.Text -> (Standard.Table.Table.Table&Standard.Database.DB_Table.DB_Table&Standard.Base.Any.Any) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/docs/api/SQL.md b/distribution/lib/Standard/Database/0.0.0-dev/docs/api/SQL.md index 0353626197cb..57f79ccde414 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/docs/api/SQL.md +++ b/distribution/lib/Standard/Database/0.0.0-dev/docs/api/SQL.md @@ -19,6 +19,7 @@ - fragments self -> Standard.Base.Any.Any - prepare self -> Standard.Base.Any.Any - serialize self ensure_roundtrip:Standard.Base.Data.Boolean.Boolean -> Standard.Base.Data.Json.JS_Object!Standard.Database.SQL.Unable_To_Serialize_SQL_Statement + - surround self prefix:Standard.Base.Data.Text.Text= suffix:Standard.Base.Data.Text.Text= -> Standard.Database.SQL.SQL_Statement - to_js_object self -> Standard.Base.Any.Any - to_text self -> Standard.Base.Data.Text.Text - unsafe_to_raw_sql self -> Standard.Base.Any.Any diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md b/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md index 903b6576704f..62d037929a24 100644 --- a/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md @@ -27,11 +27,14 @@ - schemas self -> Standard.Base.Any.Any - set_database self database:Standard.Base.Any.Any -> Standard.Base.Any.Any - set_schema self schema:Standard.Base.Any.Any -> Standard.Base.Any.Any + - spatial_drivers self -> Standard.Base.Any.Any - table_types self -> Standard.Base.Any.Any - tables self name_like:Standard.Base.Data.Text.Text= database:Standard.Base.Data.Text.Text= schema:Standard.Base.Data.Text.Text= types:Standard.Base.Any.Any= all_fields:Standard.Base.Any.Any= -> Standard.Base.Any.Any - to_js_object self -> Standard.Base.Any.Any - truncate_table self table_name:Standard.Base.Any.Any -> Standard.Base.Any.Any - type_mapping self -> Standard.Base.Any.Any - version self -> Standard.Base.Data.Text.Text + - write_database self path:Standard.Base.System.File.Generic.Writable_File.Writable_File= schema_only:Standard.Base.Data.Boolean.Boolean= -> Standard.Base.Any.Any + - write_file self query:(Standard.Database.SQL_Query.SQL_Query_With_Schema|Standard.Database.SQL_Query.SQL_Query|Standard.Table.Table.Table)= path:Standard.Base.System.File.Generic.Writable_File.Writable_File= on_existing_file:Standard.Base.System.File.Existing_File_Behavior.Existing_File_Behavior= -> Standard.Base.Any.Any - schema_black_list -> Standard.Base.Any.Any - Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data.from that:Standard.DuckDB.DuckDB_Connection.DuckDB_Connection -> Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso index ab72c1047f0c..6fe1be479f1a 100644 --- a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso @@ -675,21 +675,25 @@ type DuckDB_Connection write_database self (path:Writable_File=Missing_Argument.throw "path") schema_only:Boolean=False = ## Check a local file (to do add support for non-local files later) if path.is_local.not then Error.throw (Illegal_Argument.Error "Only local files are currently supported for writing DuckDB databases. Write to a local file and then copy it to the desired location.") + path_as_file = path.file ## Check if it exists - if path.exists then Error.throw (Illegal_Argument.Error "The specified file already exists. Please provide a different path or delete the existing file.") + if path_as_file.exists then Error.throw (Illegal_Argument.Error "The specified file already exists. Please provide a different path or delete the existing file.") ## Check Execution Context Context.Output.if_enabled disabled_message="As writing is disabled, cannot write the database. Press the Write button ▶ to perform the operation." panic=False <| Nothing ## Write the database - escaped_path = path.path.replace "'" "''" + escaped_path = path_as_file.path.replace "'" "''" query = "ATTACH '" + escaped_path + "' AS copy_to_database; COPY FROM DATABASE " + self.database + " TO copy_to_database" + (if schema_only then " (SCHEMA);" else ";") + " DETACH copy_to_database;" + + ## Copy the database result = self.execute_update query - result.if_not_error <| - ## Return a new connection - DuckDB.From_File path schema=self.schema . connect + Error.return_if_error result + + ## Return a new connection + DuckDB.From_File path_as_file schema=self.schema . connect ## --- icon: data_output @@ -708,15 +712,17 @@ type DuckDB_Connection @on_existing_file Existing_File_Behavior.widget include_append=False write_file : (SQL_Query_With_Schema | SQL_Query | Table) -> Text -> Existing_File_Behavior -> DuckDB_Connection write_file self (query:(SQL_Query_With_Schema | SQL_Query | Table)=Missing_Argument.throw "query") (path:Writable_File=Missing_Argument.throw "path") on_existing_file:Existing_File_Behavior=..Backup = + Error.return_if_error query + ## Check a local file (to do add support for non-local files later) if path.is_local.not then Error.throw (Illegal_Argument.Error "Only local files are currently supported for writing via DuckDB. Write to a local file and then copy it to the desired location.") + path_as_file = path.file ## Check Execution Context Context.Output.if_enabled disabled_message="As writing is disabled, cannot write the file. Press the Write button ▶ to perform the operation." panic=False <| Nothing ## Rationalise the query to a DB_Table - if query.is_error then query table = case query of _ : In_Memory_Table -> Error.throw (Illegal_Argument.Error "Load the table into DuckDB first using `bulk_load`.") db_table : DB_Table -> if Connection.check_on_connection self db_table then query else @@ -724,11 +730,11 @@ type DuckDB_Connection _ -> query.to_db_table ## Check Existing_File_Behavior - if path.exists then case on_existing_file of + if path_as_file.exists then case on_existing_file of Existing_File_Behavior.Append -> Error.throw (Illegal_Argument.Error "DuckDB does not support appending to existing files.") - Existing_File_Behavior.Error -> Error.throw (File_Error.Already_Exists path) + Existing_File_Behavior.Error -> Error.throw (File_Error.Already_Exists path_as_file) _ -> Nothing - write_action = if path.exists.not || on_existing_file != Existing_File_Behavior then file->action-> action file else + write_action = if path_as_file.exists.not || on_existing_file != Existing_File_Behavior then file->action-> action file else file -> action -> tmp_file = File.create_temporary_file prefix="duckdb_write_" suffix=".tmp" file.move_to tmp_file replace_existing=True @@ -743,7 +749,7 @@ type DuckDB_Connection result ## Write the file - write_action path file-> + write_action path_as_file file-> escaped_path = file.path.replace "'" "''" sql_query = table.to_sql.surround "COPY (" ") TO '" + escaped_path + "';'" result = self.execute_update sql_query From 401c00ece3b7dfb5d25227e6aff50b234cc61bf5 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 3 Dec 2025 11:39:08 +0000 Subject: [PATCH 08/13] Changelog --- .../lib/Standard/Microsoft/0.0.0-dev/docs/api/OneDrive.md | 1 + project/Dependencies.scala | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/docs/api/OneDrive.md b/distribution/lib/Standard/Microsoft/0.0.0-dev/docs/api/OneDrive.md index a8eccb57a7b9..b0b46a0f94e7 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/docs/api/OneDrive.md +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/docs/api/OneDrive.md @@ -26,3 +26,4 @@ - uri self -> Standard.Base.Any.Any - with_input_stream self open_options:Standard.Base.Data.Vector.Vector action:Standard.Base.Any.Any -> Standard.Base.Any.Any - Standard.Base.System.File.Generic.File_Like.File_Like.from that:Standard.Microsoft.OneDrive.OneDrive_File -> Standard.Base.System.File.Generic.File_Like.File_Like +- Standard.Base.Enso_Cloud.Data_Link.Data_Link_From_File.from that:Standard.Microsoft.OneDrive.OneDrive_File -> Standard.Base.Enso_Cloud.Data_Link.Data_Link_From_File diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 4bb4e4847adc..b3ccca1b6f51 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -61,7 +61,7 @@ object Dependencies { // Keep in sync with GraalVM.version. Do not change the name of this variable, // it is used by the Rust build script via regex matching. val graalMavenPackagesVersion = "25.0.1" - val targetJavaVersion = "17" + val targetJavaVersion = "24" val defaultDevEnsoVersion = "0.0.0-dev" val ensoVersion = sys.env.getOrElse( "ENSO_VERSION", From 60fe30704e70775e8ff60c4e535313ffa05eeb3f Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 3 Dec 2025 11:40:08 +0000 Subject: [PATCH 09/13] Changelog --- CHANGELOG.md | 2 ++ project/Dependencies.scala | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b20b588dc3e6..d8c5f0acc882 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ - [Read files into DuckDB both spatial and not.][14367] - [Implement Text_Column to_case for DB backends][14386] - [Implement bulk loading to DuckDB][14402] +- [Initial file writing from DuckDB][14421] [13769]: https://github.com/enso-org/enso/pull/13769 [14026]: https://github.com/enso-org/enso/pull/14026 @@ -100,6 +101,7 @@ [14367]: https://github.com/enso-org/enso/pull/14367 [14386]: https://github.com/enso-org/enso/pull/14386 [14402]: https://github.com/enso-org/enso/pull/14402 +[14421]: https://github.com/enso-org/enso/pull/14421 #### Enso Language & Runtime diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b3ccca1b6f51..4bb4e4847adc 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -61,7 +61,7 @@ object Dependencies { // Keep in sync with GraalVM.version. Do not change the name of this variable, // it is used by the Rust build script via regex matching. val graalMavenPackagesVersion = "25.0.1" - val targetJavaVersion = "24" + val targetJavaVersion = "17" val defaultDevEnsoVersion = "0.0.0-dev" val ensoVersion = sys.env.getOrElse( "ENSO_VERSION", From 31e9b94ce26c9ccbc8d534aedf10d71be0e3fb2e Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 3 Dec 2025 13:03:04 +0000 Subject: [PATCH 10/13] Fully functional. --- .../Standard/Base/0.0.0-dev/src/Data/XML.enso | 4 +- .../0.0.0-dev/src/Network/HTTP/Response.enso | 2 +- .../src/Network/HTTP/Response_Body.enso | 2 +- .../System/File/Existing_File_Behavior.enso | 10 +-- .../src/System/File/Write_Extensions.enso | 4 +- .../0.0.0-dev/src/DuckDB_Connection.enso | 83 +++++++++++-------- 6 files changed, 59 insertions(+), 46 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/XML.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/XML.enso index 04a85727edd8..8a3967598892 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/XML.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/XML.enso @@ -337,7 +337,7 @@ type XML_Document @path (Widget.Text_Input display=Display.Always) @encoding Encoding.default_widget write : Writable_File -> Encoding -> Existing_File_Behavior -> Boolean -> Problem_Behavior -> File - write self path:Writable_File (encoding : Encoding = Encoding.utf_8) (on_existing_file : Existing_File_Behavior = Existing_File_Behavior.Backup) (include_xml_declaration : Boolean = Boolean.True) (on_problems : Problem_Behavior = ..Report_Warning) = + write self path:Writable_File (encoding : Encoding = Encoding.utf_8) (on_existing_file : Existing_File_Behavior = ..Backup) (include_xml_declaration : Boolean = Boolean.True) (on_problems : Problem_Behavior = ..Report_Warning) = write_impl self.java_document path encoding on_existing_file include_xml_declaration on_problems ## --- @@ -783,7 +783,7 @@ type XML_Element @path (Widget.Text_Input display=Display.Always) @encoding Encoding.default_widget write : Writable_File -> Encoding -> Existing_File_Behavior -> Boolean -> Problem_Behavior -> File - write self path:Writable_File (encoding : Encoding = Encoding.utf_8) (on_existing_file : Existing_File_Behavior = Existing_File_Behavior.Backup) (include_xml_declaration : Boolean = Boolean.True) (on_problems : Problem_Behavior = ..Report_Warning) = + write self path:Writable_File (encoding : Encoding = Encoding.utf_8) (on_existing_file : Existing_File_Behavior = ..Backup) (include_xml_declaration : Boolean = Boolean.True) (on_problems : Problem_Behavior = ..Report_Warning) = write_impl self.java_element path encoding on_existing_file include_xml_declaration on_problems ## --- diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso index 2f4c0f82d4ef..cd57f4d4cc0a 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso @@ -235,7 +235,7 @@ type Response ``` @file (Widget.Text_Input display=Display.Always) write : Writable_File -> Existing_File_Behavior -> File - write self file:Writable_File on_existing_file=Existing_File_Behavior.Backup = + write self file:Writable_File on_existing_file:Existing_File_Behavior=..Backup = self.body.write file on_existing_file ## --- diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response_Body.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response_Body.enso index e9fce286cc0d..de7ef49ec7b9 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response_Body.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response_Body.enso @@ -206,7 +206,7 @@ type Response_Body ``` @file (Widget.Text_Input display=Display.Always) write : Writable_File -> Existing_File_Behavior -> File - write self file:Writable_File on_existing_file=Existing_File_Behavior.Backup = + write self file:Writable_File on_existing_file:Existing_File_Behavior=..Backup = self.with_stream body_stream-> file.write on_existing_file output_stream-> r = output_stream.write_stream body_stream diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/Existing_File_Behavior.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/Existing_File_Behavior.enso index 9c3903ec8192..53c31ac6b809 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/Existing_File_Behavior.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/Existing_File_Behavior.enso @@ -1,5 +1,4 @@ import project.Data.Vector.Vector -import project.Meta import project.Metadata.Display import project.Metadata.Widget from project.Data.Boolean import Boolean, True @@ -30,14 +29,13 @@ type Existing_File_Behavior private: true --- widget (include_overwrite:Boolean = True) (include_backup:Boolean = True) (include_append:Boolean = True) (include_error:Boolean = True) -> Widget = - fqn = Existing_File_Behavior.to Meta.Type . qualified_name options = Vector.build builder-> if include_overwrite then - builder.append (Option "Overwrite" fqn+".Overwrite") + builder.append (Option "Overwrite" "..Overwrite") if include_backup then - builder.append (Option "Backup" fqn+".Backup") + builder.append (Option "Backup" "..Backup") if include_append then - builder.append (Option "Append" fqn+".Append") + builder.append (Option "Append" "..Append") if include_error then - builder.append (Option "Error" fqn+".Error") + builder.append (Option "Error" "..Error") Single_Choice display=Display.Always values=options diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/Write_Extensions.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/Write_Extensions.enso index 8c37bc869279..9b1cb43a9b80 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/Write_Extensions.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/Write_Extensions.enso @@ -53,7 +53,7 @@ from project.Metadata import Display, Widget @path (Widget.Text_Input display=Display.Always) @encoding Encoding.default_widget Text.write : Writable_File -> Encoding -> Existing_File_Behavior -> Problem_Behavior -> File ! Encoding_Error | Illegal_Argument | File_Error -Text.write self (path : Writable_File) (encoding : Encoding = Encoding.utf_8) (on_existing_file : Existing_File_Behavior = Existing_File_Behavior.Backup) (on_problems : Problem_Behavior = ..Report_Warning) = +Text.write self (path : Writable_File) (encoding : Encoding = Encoding.utf_8) (on_existing_file : Existing_File_Behavior = ..Backup) (on_problems : Problem_Behavior = ..Report_Warning) = bytes = self.bytes encoding on_problems bytes.if_not_error <| path.write_handling_dry_run on_existing_file effective_file-> stream-> @@ -97,7 +97,7 @@ Text.write self (path : Writable_File) (encoding : Encoding = Encoding.utf_8) (o [36, -62, -93, -62, -89, -30, -126, -84, -62, -94].write_bytes Examples.scratch_file.write_bytes Examples.scratch_file Existing_File_Behavior.Append ``` Vector.write_bytes : Writable_File -> Existing_File_Behavior -> Any ! Illegal_Argument | File_Error -Vector.write_bytes self (path : Writable_File) (on_existing_file : Existing_File_Behavior = Existing_File_Behavior.Backup) = +Vector.write_bytes self (path : Writable_File) (on_existing_file : Existing_File_Behavior = ..Backup) = Panic.catch Unsupported_Argument_Types handler=(_ -> Error.throw (Illegal_Argument.Error "Only Vectors consisting of bytes (integers in the range from -128 to 127) are supported by the `write_bytes` method.")) <| ## Output_Stream does this conversion, but we do it here as well so that opening a file for a write that will fail won't overwrite existing diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso index 6fe1be479f1a..8904c53ef786 100644 --- a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso @@ -17,6 +17,7 @@ from Standard.Table.Internal.Storage import from_value_type import Standard.Database.Bulk_Load_Exists.Bulk_Load_Exists import Standard.Database.Connection.Connection.Connection +import Standard.Database.Connection.Connection_Options.Connection_Options import Standard.Database.DB_Table as DB_Table_Module import Standard.Database.Dialects.Dialect.Dialect import Standard.Database.Internal.JDBC_Connection @@ -668,32 +669,38 @@ type DuckDB_Connection - `path`: The path to write the database to. - `schema_only`: If set to `True`, only the schema will be written, without any data. Defaults to `False`. + - `on_existing_file`: The behavior to use if the file already exists. + Defaults to `Backup`, which will create a backup of the existing file. ## Returns A new DuckDB_Connection connected to the written database. - write_database : Text -> Boolean -> DuckDB_Connection - write_database self (path:Writable_File=Missing_Argument.throw "path") schema_only:Boolean=False = + write_database : Text -> Boolean -> Existing_File_Behavior -> DuckDB_Connection + write_database self (path:Writable_File=Missing_Argument.throw "path") schema_only:Boolean=False on_existing_file:Existing_File_Behavior=..Backup = ## Check a local file (to do add support for non-local files later) if path.is_local.not then Error.throw (Illegal_Argument.Error "Only local files are currently supported for writing DuckDB databases. Write to a local file and then copy it to the desired location.") path_as_file = path.file - ## Check if it exists - if path_as_file.exists then Error.throw (Illegal_Argument.Error "The specified file already exists. Please provide a different path or delete the existing file.") + ## Check if it exists and then follow on_existing_file behavior + write_action = _handle_existing_file path_as_file on_existing_file allow_append=True + Error.return_if_error write_action ## Check Execution Context Context.Output.if_enabled disabled_message="As writing is disabled, cannot write the database. Press the Write button ▶ to perform the operation." panic=False <| Nothing - ## Write the database - escaped_path = path_as_file.path.replace "'" "''" - query = "ATTACH '" + escaped_path + "' AS copy_to_database; COPY FROM DATABASE " + self.database + " TO copy_to_database" + (if schema_only then " (SCHEMA);" else ";") + " DETACH copy_to_database;" - ## Copy the database - result = self.execute_update query - Error.return_if_error result + write_action file-> + ## Write the database + escaped_path = file.path.replace "'" "''" + query = "ATTACH '" + escaped_path + "' AS copy_to_database; COPY FROM DATABASE " + self.database + " TO copy_to_database" + (if schema_only then " (SCHEMA);" else ";") + " DETACH copy_to_database;" - ## Return a new connection - DuckDB.From_File path_as_file schema=self.schema . connect + result = self.execute_update query + if result.is_error then + _ = self.execute_update "DETACH copy_to_database;" + Error.return_if_error result + + ## Return a new connection + DuckDB.From_File file schema=self.schema . connect Connection_Options.Value ## --- icon: data_output @@ -718,6 +725,10 @@ type DuckDB_Connection if path.is_local.not then Error.throw (Illegal_Argument.Error "Only local files are currently supported for writing via DuckDB. Write to a local file and then copy it to the desired location.") path_as_file = path.file + ## Check if it exists and then follow on_existing_file behavior + write_action = _handle_existing_file path_as_file on_existing_file + Error.return_if_error write_action + ## Check Execution Context Context.Output.if_enabled disabled_message="As writing is disabled, cannot write the file. Press the Write button ▶ to perform the operation." panic=False <| Nothing @@ -725,33 +736,14 @@ type DuckDB_Connection ## Rationalise the query to a DB_Table table = case query of _ : In_Memory_Table -> Error.throw (Illegal_Argument.Error "Load the table into DuckDB first using `bulk_load`.") - db_table : DB_Table -> if Connection.check_on_connection self db_table then query else + db_table : DB_Table -> if Connection.check_on_connection self db_table then db_table else Error.throw (Illegal_Argument.Error "Cannot write a DB_Table from a different connection. Load the table into this connection first.") _ -> query.to_db_table - ## Check Existing_File_Behavior - if path_as_file.exists then case on_existing_file of - Existing_File_Behavior.Append -> Error.throw (Illegal_Argument.Error "DuckDB does not support appending to existing files.") - Existing_File_Behavior.Error -> Error.throw (File_Error.Already_Exists path_as_file) - _ -> Nothing - write_action = if path_as_file.exists.not || on_existing_file != Existing_File_Behavior then file->action-> action file else - file -> action -> - tmp_file = File.create_temporary_file prefix="duckdb_write_" suffix=".tmp" - file.move_to tmp_file replace_existing=True - - ## Convert any Panic to an Error allowing us to restore the original file on failure - result = Panic.catch Any (action file) caught_panic-> Error.throw caught_panic.payload - - case result.is_error of - True -> tmp_file.move_to file replace_existing=True - False -> tmp_file.move_to (file.parent / (file.name+".bak")) replace_existing=True - - result - ## Write the file - write_action path_as_file file-> + write_action file-> escaped_path = file.path.replace "'" "''" - sql_query = table.to_sql.surround "COPY (" ") TO '" + escaped_path + "';'" + sql_query = table.to_sql.surround "COPY (" (") TO '" + escaped_path + "';") result = self.execute_update sql_query result.if_not_error <| path @@ -804,3 +796,26 @@ private _spatial_driver_widget connection = names = connection.spatial_drivers.at "short_name" . to_vector options = names.map n-> Option n n.pretty Single_Choice options+[Option "" '"DriverName"'] display=..Always + +private _handle_existing_file path on_existing_file allow_append:Boolean=False = + if path.exists then case on_existing_file of + Existing_File_Behavior.Append -> if allow_append then Nothing else + Error.throw (Illegal_Argument.Error "DuckDB does not support appending to existing files.") + Existing_File_Behavior.Error -> Error.throw (File_Error.Already_Exists path) + _ -> Nothing + + if path.exists.not || on_existing_file==Existing_File_Behavior.Append then action-> action path else + action -> + tmp_file = File.create_temporary_file prefix="duckdb_write_" suffix=".tmp" + path.move_to tmp_file replace_existing=True + + ## Convert any Panic to an Error allowing us to restore the original file on failure + result = Panic.catch Any (action path) caught_panic-> Error.throw caught_panic.payload + + case result.is_error of + True -> tmp_file.move_to path replace_existing=True + False -> + if on_existing_file==Existing_File_Behavior.Overwrite then tmp_file.delete else + tmp_file.move_to (path.parent / (path.name+".bak")) replace_existing=True + + result From c40c95563650b60920ebb52740f6e8dab85235ea Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 3 Dec 2025 13:43:22 +0000 Subject: [PATCH 11/13] Docs --- .../lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md b/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md index 62d037929a24..03bee2c92949 100644 --- a/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md @@ -34,7 +34,7 @@ - truncate_table self table_name:Standard.Base.Any.Any -> Standard.Base.Any.Any - type_mapping self -> Standard.Base.Any.Any - version self -> Standard.Base.Data.Text.Text - - write_database self path:Standard.Base.System.File.Generic.Writable_File.Writable_File= schema_only:Standard.Base.Data.Boolean.Boolean= -> Standard.Base.Any.Any + - write_database self path:Standard.Base.System.File.Generic.Writable_File.Writable_File= schema_only:Standard.Base.Data.Boolean.Boolean= on_existing_file:Standard.Base.System.File.Existing_File_Behavior.Existing_File_Behavior= -> Standard.Base.Any.Any - write_file self query:(Standard.Database.SQL_Query.SQL_Query_With_Schema|Standard.Database.SQL_Query.SQL_Query|Standard.Table.Table.Table)= path:Standard.Base.System.File.Generic.Writable_File.Writable_File= on_existing_file:Standard.Base.System.File.Existing_File_Behavior.Existing_File_Behavior= -> Standard.Base.Any.Any - schema_black_list -> Standard.Base.Any.Any - Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data.from that:Standard.DuckDB.DuckDB_Connection.DuckDB_Connection -> Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data From 825b3966001a4573ef6bdf5d49c5e75889528453 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 3 Dec 2025 16:15:14 +0000 Subject: [PATCH 12/13] Fix failing tests and widget for OneDrive. --- .../lib/Standard/Microsoft/0.0.0-dev/src/OneDrive.enso | 3 +++ test/Base_Tests/src/Network/Http/Request_Spec.enso | 2 +- test/Base_Tests/src/Network/URI_Spec.enso | 8 ++++---- test/Cloud_Tests/src/Network/Http/Request_Spec.enso | 2 +- test/Cloud_Tests/src/Network/URI_Spec.enso | 8 ++++---- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/OneDrive.enso b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/OneDrive.enso index 8b0c4adea435..2e800b21fefa 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/OneDrive.enso +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/OneDrive.enso @@ -11,6 +11,7 @@ import Standard.Base.System.File.Generic.File_Like.File_Like import Standard.Base.System.File.Generic.Writable_File.Writable_File import Standard.Base.System.File_Format_Metadata.File_Format_Metadata import Standard.Base.System.Input_Stream.Input_Stream +from Standard.Base.Metadata.Widget import Secret_Browse from Standard.Base.System.File import find_extension_from_name from Standard.Base.System.File.Generic.File_Write_Strategy import generic_copy @@ -26,6 +27,7 @@ type OneDrive ## Arguments - `credential`: A Cloud secret or a file containing Microsoft365 credentials. + @credential Secret_Browse root : File | Enso_Secret -> OneDrive_File root (credential : File | Enso_Secret | Microsoft365_Credential = _default_credential) -> OneDrive_File = ms_cred = Microsoft365_Credential.initialize credential @@ -39,6 +41,7 @@ type OneDrive ## Arguments - `path`: A path to a file or folder in OneDrive. - `credential`: A Cloud secret or a file containing Microsoft365 credentials. + @credential Secret_Browse new : Text -> File | Enso_Secret -> OneDrive_File new path:Text="" (credential : File | Enso_Secret | Microsoft365_Credential = _default_credential) -> OneDrive_File = used_path = (if (path.starts_with "onedrive://" ..Insensitive) then path.drop (..First 11) else path) . trim ..Left '/' diff --git a/test/Base_Tests/src/Network/Http/Request_Spec.enso b/test/Base_Tests/src/Network/Http/Request_Spec.enso index a3b80d2a076a..3d6f719ba6d7 100644 --- a/test/Base_Tests/src/Network/Http/Request_Spec.enso +++ b/test/Base_Tests/src/Network/Http/Request_Spec.enso @@ -12,7 +12,7 @@ add_specs suite_builder = test_headers = [Header.application_json, Header.new "X-Foo-Id" "0123456789"] suite_builder.group "Request" group_builder-> group_builder.specify "should return error when creating request from invalid URI" <| - Request.new HTTP_Method.Post "invalid uri" . should_fail_with Syntax_Error + Request.new HTTP_Method.Post "a b://invalid uri" . should_fail_with Syntax_Error group_builder.specify "should get method" <| req = Request.new HTTP_Method.Post test_uri req.method.should_equal HTTP_Method.Post diff --git a/test/Base_Tests/src/Network/URI_Spec.enso b/test/Base_Tests/src/Network/URI_Spec.enso index 1928e2e6d7d8..fc7805837a9b 100644 --- a/test/Base_Tests/src/Network/URI_Spec.enso +++ b/test/Base_Tests/src/Network/URI_Spec.enso @@ -87,10 +87,10 @@ add_specs suite_builder = addr2.to_text . should_equal 'https://httpbin.org/get?c=d+d%3De%3Df&g=h' group_builder.specify "should return Syntax_Error when parsing invalid URI" <| - r = URI.parse "a b c" + r = URI.parse "a b://c d e" r.should_fail_with Syntax_Error - r.catch.to_display_text . should_contain "a b c" - URI.from "a b c" . should_fail_with Syntax_Error + r.catch.to_display_text . should_contain "a%20b://c%20d%20e" + URI.from "a b://c d e" . should_fail_with Syntax_Error group_builder.specify "should return Syntax_Error when parsing empty URI" <| r = URI.parse "" @@ -249,7 +249,7 @@ add_specs suite_builder = r = uri1.fetch r.at "path" . should_equal "/get/a b c/d+e/f%20g/ś🚧:@" - ext = "a [!who puts that//// into URI??!!] ; --- ### a:=b" + ext = "a !who puts that//// into URI??!! ; --- a:=b" uri3 = uri0 / ext r2 = uri3.fetch r2.at "path" . should_equal "/get/"+ext diff --git a/test/Cloud_Tests/src/Network/Http/Request_Spec.enso b/test/Cloud_Tests/src/Network/Http/Request_Spec.enso index a3b80d2a076a..3d6f719ba6d7 100644 --- a/test/Cloud_Tests/src/Network/Http/Request_Spec.enso +++ b/test/Cloud_Tests/src/Network/Http/Request_Spec.enso @@ -12,7 +12,7 @@ add_specs suite_builder = test_headers = [Header.application_json, Header.new "X-Foo-Id" "0123456789"] suite_builder.group "Request" group_builder-> group_builder.specify "should return error when creating request from invalid URI" <| - Request.new HTTP_Method.Post "invalid uri" . should_fail_with Syntax_Error + Request.new HTTP_Method.Post "a b://invalid uri" . should_fail_with Syntax_Error group_builder.specify "should get method" <| req = Request.new HTTP_Method.Post test_uri req.method.should_equal HTTP_Method.Post diff --git a/test/Cloud_Tests/src/Network/URI_Spec.enso b/test/Cloud_Tests/src/Network/URI_Spec.enso index d485a2bccca5..201c36011ee9 100644 --- a/test/Cloud_Tests/src/Network/URI_Spec.enso +++ b/test/Cloud_Tests/src/Network/URI_Spec.enso @@ -93,10 +93,10 @@ add_specs suite_builder = addr2.to_text . should_equal 'https://httpbin.org/get?c=d+d%3De%3Df&g=h' group_builder.specify "should return Syntax_Error when parsing invalid URI" <| - r = URI.parse "a b c" + r = URI.parse "a b://c d e" r.should_fail_with Syntax_Error - r.catch.to_display_text . should_contain "a b c" - URI.from "a b c" . should_fail_with Syntax_Error + r.catch.to_display_text . should_contain "a%20b://c%20d%20e" + URI.from "a b://c d e" . should_fail_with Syntax_Error group_builder.specify "should return Syntax_Error when parsing empty URI" <| r = URI.parse "" @@ -255,7 +255,7 @@ add_specs suite_builder = r = uri1.fetch r.at "path" . should_equal "/get/a b c/d+e/f%20g/ś🚧:@" - ext = "a [!who puts that//// into URI??!!] ; --- ### a:=b" + ext = "a !who puts that//// into URI??!! ; --- a:=b" uri3 = uri0 / ext r2 = uri3.fetch r2.at "path" . should_equal "/get/"+ext From b8b4cbb519b42c0a83976ea28fc331f65f591ec9 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 3 Dec 2025 16:53:53 +0000 Subject: [PATCH 13/13] Docs --- .../Standard/Base/0.0.0-dev/docs/api/Network/HTTP/Response.md | 2 +- .../Base/0.0.0-dev/docs/api/Network/HTTP/Response_Body.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/docs/api/Network/HTTP/Response.md b/distribution/lib/Standard/Base/0.0.0-dev/docs/api/Network/HTTP/Response.md index 9bc5a43b2281..46cf913c2ccf 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/docs/api/Network/HTTP/Response.md +++ b/distribution/lib/Standard/Base/0.0.0-dev/docs/api/Network/HTTP/Response.md @@ -16,6 +16,6 @@ - to_js_object self -> Standard.Base.Any.Any - uri self -> Standard.Base.Any.Any - with_materialized_body self -> Standard.Base.Any.Any - - write self file:Standard.Base.System.File.Generic.Writable_File.Writable_File on_existing_file:Standard.Base.Any.Any= -> Standard.Base.Any.Any + - write self file:Standard.Base.System.File.Generic.Writable_File.Writable_File on_existing_file:Standard.Base.System.File.Existing_File_Behavior.Existing_File_Behavior= -> Standard.Base.Any.Any - filename_from_content_disposition content_disposition:Standard.Base.Any.Any -> Standard.Base.Any.Any - resolve_file_metadata_for_response response:Standard.Base.Any.Any -> Standard.Base.Any.Any diff --git a/distribution/lib/Standard/Base/0.0.0-dev/docs/api/Network/HTTP/Response_Body.md b/distribution/lib/Standard/Base/0.0.0-dev/docs/api/Network/HTTP/Response_Body.md index 4b6525dafaf0..fdfb0a2a2082 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/docs/api/Network/HTTP/Response_Body.md +++ b/distribution/lib/Standard/Base/0.0.0-dev/docs/api/Network/HTTP/Response_Body.md @@ -12,7 +12,7 @@ - new input_stream:Standard.Base.Any.Any metadata:Standard.Base.System.File_Format_Metadata.File_Format_Metadata uri:Standard.Base.Network.URI.URI -> Standard.Base.Any.Any - to_text self -> Standard.Base.Any.Any - with_stream self action:Standard.Base.Any.Any -> Standard.Base.Any.Any - - write self file:Standard.Base.System.File.Generic.Writable_File.Writable_File on_existing_file:Standard.Base.Any.Any= -> Standard.Base.Any.Any + - write self file:Standard.Base.System.File.Generic.Writable_File.Writable_File on_existing_file:Standard.Base.System.File.Existing_File_Behavior.Existing_File_Behavior= -> Standard.Base.Any.Any - can_decode type:Standard.Base.Any.Any -> Standard.Base.Any.Any - decode_format_selector -> Standard.Base.Any.Any - delete_file file:Standard.Base.Any.Any -> Standard.Base.Any.Any