Update Core to 2.24. (#433)

* Regenerate bindings for 2.24.

No new APIs were added; some just moved to stable from experimental.

* Add two now-stable APIs.

* Update documentation of `ls_recursive`.

* Bump Core to 2.24.0.

* Run nightly builds on release-2.24.

* Bump version to 5.14.0.

* Add tests.

* Improve array upgrade test.

Author: teo-tsirpanis

.github/workflows/nightly.yml

@@ -27,7 +27,7 @@ jobs:
           - os: windows-latest
             platform: windows-x86_64
           - tag: dev
-        tag: [release-2.23, dev]
+        tag: [release-2.24, dev]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout TileDB
@@ -71,7 +71,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        tag: [release-2.23, dev]
+        tag: [release-2.24, dev]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout TileDB-CSharp
@@ -112,7 +112,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        tag: [release-2.23, dev]
+        tag: [release-2.24, dev]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout TileDB-CSharp

Directory.Packages.props

@@ -3,7 +3,7 @@
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
     <TileDBNativePackageName>TileDB.Native</TileDBNativePackageName>
     <TileDBNativeVersionMajor>2</TileDBNativeVersionMajor>
-    <TileDBNativeVersionMinor>23</TileDBNativeVersionMinor>
+    <TileDBNativeVersionMinor>24</TileDBNativeVersionMinor>
     <TileDBNativePackageVersion>[$(TileDBNativeVersionMajor).$(TileDBNativeVersionMinor).0,$(TileDBNativeVersionMajor).$([MSBuild]::Add($(TileDBNativeVersionMinor), 1)).0)</TileDBNativePackageVersion>
 
     <!-- The DevelopmentBuild property switches to the locally built native packages.

sources/TileDB.CSharp/Array.cs

@@ -329,6 +329,18 @@ public void Create(ArraySchema schema)
         Create(_ctx, _uri, schema);
     }
 
+    /// <summary>
+    /// Deletes an array from storage.
+    /// </summary>
+    /// <param name="ctx">The <see cref="CSharp.Context"/> to use.</param>
+    /// <param name="uri">The array's URI.</param>
+    public static void Delete(Context ctx, string uri)
+    {
+        using var msUri = new MarshaledString(uri);
+        using var ctxHandle = ctx.Handle.Acquire();
+        ctx.handle_error(Methods.tiledb_array_delete(ctxHandle, msUri));
+    }
+
     /// <summary>
     /// Applies an <see cref="ArraySchemaEvolution"/> to the schema of an array.
     /// </summary>
@@ -367,6 +379,20 @@ public void Evolve(ArraySchemaEvolution schemaEvolution)
         Evolve(_ctx, _uri, schemaEvolution);
     }
 
+    /// <summary>
+    /// Upgrades the storage format version of an array.
+    /// </summary>
+    /// <param name="ctx">The <see cref="CSharp.Context"/> to use.</param>
+    /// <param name="uri">The array's URI.</param>
+    /// <param name="config">Optional <see cref="CSharp.Config"/> to customize the process.</param>
+    public static void UpgradeVersion(Context ctx, string uri, Config? config = null)
+    {
+        using var msUri = new MarshaledString(uri);
+        using var ctxHandle = ctx.Handle.Acquire();
+        using var configHandle = config?.Handle.Acquire() ?? default;
+        ctx.handle_error(Methods.tiledb_array_upgrade_version(ctxHandle, msUri, configHandle));
+    }
+
     /// <summary>
     /// Consolidates an array.
     /// </summary>

sources/TileDB.CSharp/Interop/Methods.cs

@@ -6,12 +6,12 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Nullable>enable</Nullable>
     <RootNamespace>TileDB.CSharp</RootNamespace>
-    <Version>5.13.0</Version>
+    <Version>5.14.0</Version>
     <Description>C# wrapper of the TileDB Embedded universal data engine.</Description>
     <PackageReadmeFile>README.md</PackageReadmeFile>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <EnablePackageValidation>true</EnablePackageValidation>
-    <PackageValidationBaselineVersion>5.12.0</PackageValidationBaselineVersion>
+    <PackageValidationBaselineVersion>5.13.0</PackageValidationBaselineVersion>
     <NoWarn>$(NoWarn);TILEDB0012;TILEDB0013;TILEDB0014</NoWarn>
   </PropertyGroup>
 

sources/TileDB.CSharp/TileDB.CSharp.csproj

@@ -381,7 +381,7 @@ public List<string> GetChildren(string uri)
     /// <returns>A <see cref="List{T}"/> containing the files of the directory in <paramref name="uri"/>
     /// and their sizes.</returns>
     /// <remarks>
-    /// This operation is supported only on URIs to AWS S3.
+    /// This operation is supported only on URIs to local file system, S3, Azure and GCS.
     /// </remarks>
     public List<(string Uri, ulong Size)> GetChildrenRecursive(string uri)
     {
@@ -439,7 +439,7 @@ public void VisitChildren<T>(string uri, Func<string, T, bool> callback, T callb
     /// <param name="callbackArg">An argument that will be passed to <paramref name="callback"/>.</param>
     /// <typeparam name="T">The type of <paramref name="callbackArg"/>.</typeparam>
     /// <remarks>
-    /// This operation is supported only on local filesystem and AWS S3 URIs.
+    /// This operation is supported only on URIs to local file system, S3, Azure and GCS.
     /// </remarks>
     public void VisitChildrenRecursive<T>(string uri, Func<string, ulong, T, bool> callback, T callbackArg)
     {

sources/TileDB.CSharp/VFS.cs

@@ -23,7 +23,7 @@ public void TestDenseArray()
         var array = new Array(context, tmpArrayPath);
         Assert.IsNotNull(array);
 
-        var array_schema = BuildDenseArraySchema(context);
+        var array_schema = ArrayTest.BuildDenseArraySchema(context);
         Assert.IsNotNull(array_schema);
 
         array.Create(array_schema);
@@ -75,7 +75,7 @@ public void TestSparseArray()
         var array = new Array(context, tmpArrayPath);
         Assert.IsNotNull(array);
 
-        var array_schema = BuildSparseArraySchema(context);
+        var array_schema = ArrayTest.BuildSparseArraySchema(context);
         Assert.IsNotNull(array_schema);
 
         array.Create(array_schema);
@@ -133,7 +133,7 @@ public void TestConsolidateFragments()
 
         using var uri = new TemporaryDirectory("array_consolidate_fragments");
 
-        using (var schema = BuildDenseArraySchema(context))
+        using (var schema = ArrayTest.BuildDenseArraySchema(context))
         using (var array = new Array(context, uri))
         {
             array.Create(schema);
@@ -170,7 +170,102 @@ public void TestConsolidateFragments()
         Assert.AreEqual(FragmentCount, fragmentInfo.FragmentToVacuumCount);
     }
 
-    private ArraySchema BuildDenseArraySchema(Context context)
+    [TestMethod]
+    public void TestDelete()
+    {
+        var context = Context.GetDefault();
+
+        using var uri = new TemporaryDirectory("array_delete");
+
+        using (var schema = BuildDenseArraySchema(context))
+        {
+            Array.Create(context, uri, schema);
+        }
+
+        Assert.AreEqual(ObjectType.Array, context.GetObjectType(uri));
+
+        Array.Delete(context, uri);
+
+        Assert.AreEqual(ObjectType.Invalid, context.GetObjectType(uri));
+    }
+
+    /// <summary>
+    /// Base-64 encoded zip file of an array with an old format version.
+    /// </summary>
+    /// <remarks>
+    /// The array was taken from <see href="https://github.com/TileDB-Inc/TileDB-Py/blob/3ea00dbc2eabce972bea8af1ea8a4dc420e00a72/tiledb/tests/test_libtiledb.py#L311-L341"/>
+    /// and repackaged to a zip file, because <c>System.Formats.Tar</c> is available only in .NET 7+.
+    /// </remarks>
+    const string OldVersionArrayZip = """
+        UEsDBBQAAAAAAAQHtFIAAAAAAAAAAAAAAABDAAAAX18xNjIxNDYxMzY3OTcyXzE2MjE0NjEzNjc5
+        NzJfMWJlOWNlMDM3NDI1NGMyNGI2NjBlZDE3OGU3OWQ1ZTdfNS5va1BLAwQUAAAACAAEB7RSkZnF
+        iBEAAAAsAAAARQAAAF9fMTYyMTQ2MTM2Nzk3Ml8xNjIxNDYxMzY3OTcyXzFiZTljZTAzNzQyNTRj
+        MjRiNjYwZWQxNzhlNzlkNWU3XzUvLnRkYmNkgAAJKAYBRijNBKVZoTQAUEsDBBQAAAAIAAQHtFIu
+        OY4+jQAAAI0EAABYAAAAX18xNjIxNDYxMzY3OTcyXzE2MjE0NjEzNjc5NzJfMWJlOWNlMDM3NDI1
+        NGMyNGI2NjBlZDE3OGU3OWQ1ZTdfNS9fX2ZyYWdtZW50X21ldGFkYXRhLnRkYmNlYGCwYIAABSjN
+        wghlMAiBSUYGkAAjK4gAs5DUiwCxAJTPiCRWwfg4KSEhoSVJgZXdJ0CMgXEJAz/IAH2oWpgeYu0C
+        qedmQLULJlbBmJziwAqym2nUCphxMH2jVoxaAbECAZiQFIMAM5TWYUAFDgzEgRwofR5KG0ENngql
+        f0DpaKjF+6C0ItTiFqgDAFBLAwQUAAAACAAEB7RSiHDSEyYAAABAAAAARgAAAF9fMTYyMTQ2MTM2
+        Nzk3Ml8xNjIxNDYxMzY3OTcyXzFiZTljZTAzNzQyNTRjMjRiNjYwZWQxNzhlNzlkNWU3XzUvZC50
+        ZGJjZIAACSCWAWIBKJ8RSUxjq/5fBYm5DAwRQHEmqAImBu1IHgZuAFBLAwQUAAAAAACkCbRSAAAA
+        AAAAAAAAAAAAQwAAAF9fMTYyMTQ2MjM4Nzg5N18xNjIxNDYyMzg3ODk3Xzk5ZWQxNDgxMzc5ZTQ0
+        ZTk4N2U0NTM1YWIyYWExYjBjXzUub2tQSwMEFAAAAAgApAm0UpGZxYgRAAAALAAAAEUAAABfXzE2
+        MjE0NjIzODc4OTdfMTYyMTQ2MjM4Nzg5N185OWVkMTQ4MTM3OWU0NGU5ODdlNDUzNWFiMmFhMWIw
+        Y181Ly50ZGJjZIAACSgGAUYozQSlWaE0AFBLAwQUAAAACACkCbRSLjmOPo0AAACNBAAAWAAAAF9f
+        MTYyMTQ2MjM4Nzg5N18xNjIxNDYyMzg3ODk3Xzk5ZWQxNDgxMzc5ZTQ0ZTk4N2U0NTM1YWIyYWEx
+        YjBjXzUvX19mcmFnbWVudF9tZXRhZGF0YS50ZGJjZWBgsGCAAAUozcIIZTAIgUlGBpAAIyuIALOQ
+        1IsAsQCUz4gkVsH4OCkhIaElSYGV3SdAjIFxCQM/yAB9qFqYHmLtAqnnZkC1CyZWwZic4sAKsptp
+        1AqYcTB9o1aMWgGxAgGYkBSDADOU1mFABQ4MxIEcKH0eShtBDZ4KpX9A6WioxfugtCLU4haoAwBQ
+        SwMEFAAAAAgApAm0Uohw0hMmAAAAQAAAAEYAAABfXzE2MjE0NjIzODc4OTdfMTYyMTQ2MjM4Nzg5
+        N185OWVkMTQ4MTM3OWU0NGU5ODdlNDUzNWFiMmFhMWIwY181L2QudGRiY2SAAAkglgFiASifEUlM
+        Y6v+XwWJuQwMEUBxJqgCJgbtSB4GbgBQSwMEFAAAAAgAAwe0UsP6HbxgAAAAkAAAABIAAABfX2Fy
+        cmF5X3NjaGVtYS50ZGJjZWBgiGGAgAYozcIIZTAIgUlGBpAAIyuIALOQ1FsAsQCUz4gkVsGYnMrA
+        5MbGxlgqwMwi4RjDphYm8FGew8DV4ZPCnEOsSoo5CzgTzJoSmgwTeTqSFhSlxDIwsnNcAwBQSwME
+        FAAAAAAAAwe0UgAAAAAAAAAAAAAAAAoAAABfX2xvY2sudGRiUEsBAhQAFAAAAAAABAe0UgAAAAAA
+        AAAAAAAAAEMAAAAAAAAAAAAgAAAAAAAAAF9fMTYyMTQ2MTM2Nzk3Ml8xNjIxNDYxMzY3OTcyXzFi
+        ZTljZTAzNzQyNTRjMjRiNjYwZWQxNzhlNzlkNWU3XzUub2tQSwECFAAUAAAACAAEB7RSkZnFiBEA
+        AAAsAAAARQAAAAAAAAAAACAAAABhAAAAX18xNjIxNDYxMzY3OTcyXzE2MjE0NjEzNjc5NzJfMWJl
+        OWNlMDM3NDI1NGMyNGI2NjBlZDE3OGU3OWQ1ZTdfNS8udGRiUEsBAhQAFAAAAAgABAe0Ui45jj6N
+        AAAAjQQAAFgAAAAAAAAAAAAgAAAA1QAAAF9fMTYyMTQ2MTM2Nzk3Ml8xNjIxNDYxMzY3OTcyXzFi
+        ZTljZTAzNzQyNTRjMjRiNjYwZWQxNzhlNzlkNWU3XzUvX19mcmFnbWVudF9tZXRhZGF0YS50ZGJQ
+        SwECFAAUAAAACAAEB7RSiHDSEyYAAABAAAAARgAAAAAAAAAAACAAAADYAQAAX18xNjIxNDYxMzY3
+        OTcyXzE2MjE0NjEzNjc5NzJfMWJlOWNlMDM3NDI1NGMyNGI2NjBlZDE3OGU3OWQ1ZTdfNS9kLnRk
+        YlBLAQIUABQAAAAAAKQJtFIAAAAAAAAAAAAAAABDAAAAAAAAAAAAIAAAAGICAABfXzE2MjE0NjIz
+        ODc4OTdfMTYyMTQ2MjM4Nzg5N185OWVkMTQ4MTM3OWU0NGU5ODdlNDUzNWFiMmFhMWIwY181Lm9r
+        UEsBAhQAFAAAAAgApAm0UpGZxYgRAAAALAAAAEUAAAAAAAAAAAAgAAAAwwIAAF9fMTYyMTQ2MjM4
+        Nzg5N18xNjIxNDYyMzg3ODk3Xzk5ZWQxNDgxMzc5ZTQ0ZTk4N2U0NTM1YWIyYWExYjBjXzUvLnRk
+        YlBLAQIUABQAAAAIAKQJtFIuOY4+jQAAAI0EAABYAAAAAAAAAAAAIAAAADcDAABfXzE2MjE0NjIz
+        ODc4OTdfMTYyMTQ2MjM4Nzg5N185OWVkMTQ4MTM3OWU0NGU5ODdlNDUzNWFiMmFhMWIwY181L19f
+        ZnJhZ21lbnRfbWV0YWRhdGEudGRiUEsBAhQAFAAAAAgApAm0Uohw0hMmAAAAQAAAAEYAAAAAAAAA
+        AAAgAAAAOgQAAF9fMTYyMTQ2MjM4Nzg5N18xNjIxNDYyMzg3ODk3Xzk5ZWQxNDgxMzc5ZTQ0ZTk4
+        N2U0NTM1YWIyYWExYjBjXzUvZC50ZGJQSwECFAAUAAAACAADB7RSw/odvGAAAACQAAAAEgAAAAAA
+        AAAAACAAAADEBAAAX19hcnJheV9zY2hlbWEudGRiUEsBAhQAFAAAAAAAAwe0UgAAAAAAAAAAAAAA
+        AAoAAAAAAAAAAAAgAAAAVAUAAF9fbG9jay50ZGJQSwUGAAAAAAoACgA0BAAAfAUAAAAA
+        """;
+
+    [TestMethod]
+    public void TestUpgradeVersion()
+    {
+        var context = Context.GetDefault();
+
+        using var uri = new TemporaryDirectory("array_upgrade_version");
+
+        TestUtil.UnzipBase64String(uri, OldVersionArrayZip);
+
+        using (var schema = Array.LoadArraySchema(context, uri))
+        {
+            Assert.AreEqual(5u, schema.FormatVersion());
+        }
+
+        Array.UpgradeVersion(context, uri);
+
+        using (var schema = Array.LoadArraySchema(context, uri))
+        {
+            Assert.IsTrue(schema.FormatVersion() >= 15u, "Array was not upgraded.");
+        }
+    }
+
+    private static ArraySchema BuildDenseArraySchema(Context context)
     {
         var dimension = Dimension.Create<short>(context, "dim1", 1, 10, 5);
         Assert.IsNotNull(dimension);
@@ -199,7 +294,7 @@ private ArraySchema BuildDenseArraySchema(Context context)
         return array_schema;
     }
 
-    private ArraySchema BuildSparseArraySchema(Context context)
+    private static ArraySchema BuildSparseArraySchema(Context context)
     {
         var dim1 = Dimension.Create<short>(context, "dim1", 1, 10, 5);
         Assert.IsNotNull(dim1);

tests/TileDB.CSharp.Test/ArrayTest.cs

@@ -2,6 +2,7 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.IO;
+using System.IO.Compression;
 using System.IO.Enumeration;
 using System.Linq;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
@@ -244,4 +245,12 @@ public static void PrintLocalSchema(ArraySchema schema)
                               $"Nullable: {pair.Value.Nullable()};");
         }
     }
+
+    public static void UnzipBase64String(string path, string data)
+    {
+        byte[] bytes = Convert.FromBase64String(data);
+        using var stream = new MemoryStream(bytes, writable: false);
+        using var zipFile = new ZipArchive(stream, ZipArchiveMode.Read);
+        zipFile.ExtractToDirectory(path, overwriteFiles: false);
+    }
 }