Handling datetime types (#1636)

## Why make this change?
Currently, we are incorrectly handling the datetime datatypes. A snippet
from `BaseSqlQueryStructure` class:

<img width="509" alt="image"
src="https://github.com/Azure/data-api-builder/assets/34566234/bc126221-f392-41cf-b1e3-767eefb2a54a">

As evident from the highlighted portion, even for `DateTime` .NET type,
we are parsing it with `DateTimeOffset` which is wrong. Just for an
example, if you execute a GET request on the `stocks_price` table which
has `(categoryid,pieceid,instant)` as PK, you would not get any result
back even when the record exists for that PK in the table. Why? Because
even though instant is a `datetime` column, it gets parsed as
`datetimeoffset` column. This is shown in the next 2 images.

<img width="642" alt="image"
src="https://github.com/Azure/data-api-builder/assets/34566234/24cd9003-1d30-4713-aa54-5953584ff574">
<img width="323" alt="image"
src="https://github.com/Azure/data-api-builder/assets/34566234/bac63094-adba-4544-814d-43e3661536d3">

Also, the existing tests (like the ones fixed in
`MsSqlGraphQLPaginationTests` and `GraphQLSupportedTypesTestsBase`) were
wrongly written, in which values in the range of `datetime2` were
supplied for `datetime` column.

## What is this change?
1. Correctly return value using `DateTime` field of the `DateTimeOffset`
parsed value in `BaseSqlQueryStructure` class,
2. Populate the DbType of the datetime parameters when they are sent to
the database so that the database knows the exact type of the parameter.
Eg. by default, if you send a `datetime2` param without a DbType, it is
assumed to be a `datetime` and an exception is thrown incorrectly by the
database. This is done using the map
`TypeHelper._timeSqlDbTypeToDbType`.
It should be noted 4 sql server types - `date`, `smalldatetime`,
`datetime`, `datetime2` map to the same .NET system type
`System.DateTime`. The existing `TypeHelper._systemTypeToDbTypeMap`
would not be of any use to us because it has mapping from system type to
DbType and all the above 3 sql server types have the same system type.
So, we need to rely on the actual mapping from sql server type to DbType
which is:
`datetime -> DbType.DateTime, smalldatetime -> DbType.DateTime,
datetime2 -> DbType.DateTime2`
`date -> DbType.Date`

3. Fix the existing tests.

## How was this tested?

- [x] Integration Tests - Added test for getting single record by PK
having a `datetime` column as a PK column to `FindApiTestBase`.
- [x] Unit Tests - Added filter/orderby and insert tests to
`GraphQLSupportedTypesTestsBase.QueryTypeColumnFilterAndOrderByDateTime`
and `GraphQLSupportedTypesTestsBase.InsertIntoTypeColumn`for datetime
types - `datetime2`, `smalldatetime`, `date`.

## Sample Request(s)

- Example REST request to demonstrate modifications
<img width="651" alt="image"
src="https://github.com/Azure/data-api-builder/assets/34566234/9e150a5b-63ce-4c44-b674-e2e4cb2f88b8">

- Example GQL request to demonstrate modifications
<img width="850" alt="image"
src="https://github.com/Azure/data-api-builder/assets/34566234/28c70b5a-fb2b-4dbe-9641-bab25823425e">

---------

Co-authored-by: Aniruddh Munde <anmunde@microsoft.com>
Co-authored-by: Abhishek Kumar <abhishekkuma@microsoft.com>
Co-authored-by: abhishekkumams <102276754+abhishekkumams@users.noreply.github.com>
Co-authored-by: Sean Leonard <sean.leonard@microsoft.com>

Author: 4 people

config-generators/mssql-commands.txt

@@ -11,6 +11,7 @@ add Broker --config "dab-config.MsSql.json" --source brokers --permissions "anon
 add WebsiteUser --config "dab-config.MsSql.json" --source website_users --permissions "anonymous:create,read,delete,update"
 add SupportedType --config "dab-config.MsSql.json" --source type_table --permissions "anonymous:create,read,delete,update"
 add stocks_price --config "dab-config.MsSql.json" --source stocks_price --permissions "authenticated:create,read,update,delete"
+update stocks_price --config "dab-config.MsSql.json" --permissions "anonymous:read"
 update stocks_price --config "dab-config.MsSql.json" --permissions "TestNestedFilterFieldIsNull_ColumnForbidden:read" --fields.exclude "price"
 update stocks_price --config "dab-config.MsSql.json" --permissions "TestNestedFilterFieldIsNull_EntityReadForbidden:create"
 add Tree --config "dab-config.MsSql.json" --source trees --permissions "anonymous:create,read,update,delete"

config-generators/mysql-commands.txt

@@ -98,7 +98,7 @@ update BookWebsitePlacement --config "dab-config.MySql.json" --permissions "auth
 update BookWebsitePlacement --config "dab-config.MySql.json" --permissions "authenticated:delete" --fields.include "*" --policy-database "@claims.userId eq @item.id"
 update Author --config "dab-config.MySql.json"  --permissions "authenticated:create,read,update,delete" --rest true --graphql true
 update WebsiteUser --config "dab-config.MySql.json" --permissions "authenticated:create,read,delete,update" --rest false --graphql "websiteUser:websiteUsers"
-update stocks_price --config "dab-config.MySql.json" --permissions "authenticated:create,read,delete,update" --rest false
+update stocks_price --config "dab-config.MySql.json" --permissions "authenticated:create,read,delete,update" --rest true
 update Comic --config "dab-config.MySql.json" --permissions "authenticated:create,read,update,delete" --rest true --graphql true  --relationship myseries --target.entity series --cardinality one
 update series --config "dab-config.MySql.json" --relationship comics --target.entity Comic --cardinality many
 update Broker --config "dab-config.MySql.json" --permissions "authenticated:create,update,read,delete" --graphql false

config-generators/postgresql-commands.txt

@@ -10,6 +10,7 @@ add Broker --config "dab-config.PostgreSql.json" --source brokers --permissions
 add WebsiteUser --config "dab-config.PostgreSql.json" --source website_users --permissions "anonymous:create,read,delete,update"
 add SupportedType --config "dab-config.PostgreSql.json" --source type_table --permissions "anonymous:create,read,delete,update"
 add stocks_price --config "dab-config.PostgreSql.json" --source stocks_price --permissions "authenticated:create,read,update,delete"
+update stocks_price --config "dab-config.PostgreSql.json" --permissions "anonymous:read"
 update stocks_price --config "dab-config.PostgreSql.json" --permissions "TestNestedFilterFieldIsNull_ColumnForbidden:read" --fields.exclude "price"
 update stocks_price --config "dab-config.PostgreSql.json" --permissions "TestNestedFilterFieldIsNull_EntityReadForbidden:create"
 add Tree --config "dab-config.PostgreSql.json" --source trees --permissions "anonymous:create,read,update,delete"

src/Core/Models/GraphQLFilterParsers.cs

@@ -564,7 +564,7 @@ public static Predicate Parse(
                 new PredicateOperand(column),
                 op,
                 new PredicateOperand(processLiteral ? $"{processLiterals(value, column.ColumnName)}" : value.ToString()))
-            ));
+                ));
         }
 
         return GQLFilterParser.MakeChainPredicate(predicates, PredicateOperation.AND);

src/Core/Resolvers/Sql Query Structures/BaseSqlQueryStructure.cs

@@ -361,7 +361,7 @@ protected static object ParseParamAsSystemType(string param, Type systemType)
                 "Double" => double.Parse(param),
                 "Decimal" => decimal.Parse(param),
                 "Boolean" => bool.Parse(param),
-                "DateTime" => DateTimeOffset.Parse(param, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal),
+                "DateTime" => DateTimeOffset.Parse(param, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal).DateTime,
                 "DateTimeOffset" => DateTimeOffset.Parse(param, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal),
                 "Date" => DateOnly.Parse(param),
                 "Guid" => Guid.Parse(param),

src/Core/Resolvers/Sql Query Structures/SqlQueryStructure.cs

@@ -475,7 +475,7 @@ public void AddPaginationPredicate(IEnumerable<PaginationColumn> afterJsonValues
             {
                 column.TableAlias = SourceAlias;
                 column.ParamName = column.Value is not null ?
-                     MakeDbConnectionParam(GetParamAsSystemType(column.Value!.ToString()!, column.ColumnName, GetColumnSystemType(column.ColumnName))) :
+                     MakeDbConnectionParam(GetParamAsSystemType(column.Value!.ToString()!, column.ColumnName, GetColumnSystemType(column.ColumnName)), column.ColumnName) :
                      MakeDbConnectionParam(null, column.ColumnName);
             }
 

src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs

@@ -2,15 +2,20 @@
 // Licensed under the MIT License.
 
 using System.Data;
+using System.Data.Common;
+using System.Net;
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using Azure.DataApiBuilder.Config.DatabasePrimitives;
+using Azure.DataApiBuilder.Config.ObjectModel;
 using Azure.DataApiBuilder.Core.Configurations;
 using Azure.DataApiBuilder.Core.Models;
 using Azure.DataApiBuilder.Core.Resolvers;
 using Azure.DataApiBuilder.Core.Resolvers.Factories;
+using Azure.DataApiBuilder.Service.Exceptions;
 using Microsoft.Data.SqlClient;
 using Microsoft.Extensions.Logging;
+using static Azure.DataApiBuilder.Service.GraphQLBuilder.GraphQLNaming;
 
 namespace Azure.DataApiBuilder.Core.Services
 {
@@ -80,5 +85,156 @@ public override async Task PopulateTriggerMetadataForTable(string entityName, st
                 }
             }
         }
+
+        /// <inheritdoc/>
+        protected override void PopulateColumnDefinitionWithHasDefaultAndDbType(
+            SourceDefinition sourceDefinition,
+            DataTable allColumnsInTable)
+        {
+            foreach (DataRow columnInfo in allColumnsInTable.Rows)
+            {
+                string columnName = (string)columnInfo["COLUMN_NAME"];
+                bool hasDefault =
+                    Type.GetTypeCode(columnInfo["COLUMN_DEFAULT"].GetType()) != TypeCode.DBNull;
+                if (sourceDefinition.Columns.TryGetValue(columnName, out ColumnDefinition? columnDefinition))
+                {
+                    columnDefinition.HasDefault = hasDefault;
+
+                    if (hasDefault)
+                    {
+                        columnDefinition.DefaultValue = columnInfo["COLUMN_DEFAULT"];
+                    }
+
+                    columnDefinition.DbType = TypeHelper.GetDbTypeFromSystemType(columnDefinition.SystemType);
+                    if (columnDefinition.SystemType == typeof(DateTime) || columnDefinition.SystemType == typeof(DateTimeOffset))
+                    {
+                        // MsSql types like date,smalldatetime,datetime,datetime2 are mapped to the same .NET type of DateTime.
+                        // Thus to determine the actual dbtype, we use the underlying MsSql type instead of the .NET type.
+                        DbType dbType;
+                        string sqlType = (string)columnInfo["DATA_TYPE"];
+                        if (TryResolveDbType(sqlType, out dbType))
+                        {
+                            columnDefinition.DbType = dbType;
+                        }
+                    }
+                }
+            }
+        }
+
+        /// <inheritdoc/>
+        protected override async Task FillSchemaForStoredProcedureAsync(
+            Entity procedureEntity,
+            string entityName,
+            string schemaName,
+            string storedProcedureSourceName,
+            StoredProcedureDefinition storedProcedureDefinition)
+        {
+            using DbConnection conn = new SqlConnection();
+            conn.ConnectionString = ConnectionString;
+            await QueryExecutor.SetManagedIdentityAccessTokenIfAnyAsync(conn, _dataSourceName);
+            await conn.OpenAsync();
+
+            string[] procedureRestrictions = new string[NUMBER_OF_RESTRICTIONS];
+
+            // To restrict the parameters for the current stored procedure, specify its name
+            procedureRestrictions[0] = conn.Database;
+            procedureRestrictions[1] = schemaName;
+            procedureRestrictions[2] = storedProcedureSourceName;
+
+            DataTable procedureMetadata = await conn.GetSchemaAsync(collectionName: "Procedures", restrictionValues: procedureRestrictions);
+
+            // Stored procedure does not exist in DB schema
+            if (procedureMetadata.Rows.Count == 0)
+            {
+                throw new DataApiBuilderException(
+                    message: $"No stored procedure definition found for the given database object {storedProcedureSourceName}",
+                    statusCode: HttpStatusCode.ServiceUnavailable,
+                    subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization);
+            }
+
+            // Each row in the procedureParams DataTable corresponds to a single parameter
+            DataTable parameterMetadata = await conn.GetSchemaAsync(collectionName: "ProcedureParameters", restrictionValues: procedureRestrictions);
+
+            // For each row/parameter, add an entry to StoredProcedureDefinition.Parameters dictionary
+            foreach (DataRow row in parameterMetadata.Rows)
+            {
+                // row["DATA_TYPE"] has value type string so a direct cast to System.Type is not supported.
+                // See https://learn.microsoft.com/en-us/dotnet/framework/data/adonet/sql-server-data-type-mappings
+                string sqlType = (string)row["DATA_TYPE"];
+                Type systemType = SqlToCLRType(sqlType);
+                ParameterDefinition paramDefinition = new()
+                {
+                    SystemType = systemType,
+                    DbType = TypeHelper.GetDbTypeFromSystemType(systemType)
+                };
+
+                if (paramDefinition.SystemType == typeof(DateTime) || paramDefinition.SystemType == typeof(DateTimeOffset))
+                {
+                    // MsSql types like date,smalldatetime,datetime,datetime2 are mapped to the same .NET type of DateTime.
+                    // Thus to determine the actual dbtype, we use the underlying MsSql type instead of the .NET type.
+                    DbType dbType;
+                    if (TryResolveDbType(sqlType, out dbType))
+                    {
+                        paramDefinition.DbType = dbType;
+                    }
+                }
+
+                // Add to parameters dictionary without the leading @ sign
+                storedProcedureDefinition.Parameters.TryAdd(((string)row["PARAMETER_NAME"])[1..], paramDefinition);
+            }
+
+            // Loop through parameters specified in config, throw error if not found in schema
+            // else set runtime config defined default values.
+            // Note: we defer type checking of parameters specified in config until request time
+            Dictionary<string, object>? configParameters = procedureEntity.Source.Parameters;
+            if (configParameters is not null)
+            {
+                foreach ((string configParamKey, object configParamValue) in configParameters)
+                {
+                    if (!storedProcedureDefinition.Parameters.TryGetValue(configParamKey, out ParameterDefinition? parameterDefinition))
+                    {
+                        throw new DataApiBuilderException(
+                            message: $"Could not find parameter \"{configParamKey}\" specified in config for procedure \"{schemaName}.{storedProcedureSourceName}\"",
+                            statusCode: HttpStatusCode.ServiceUnavailable,
+                            subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization);
+                    }
+                    else
+                    {
+                        parameterDefinition.HasConfigDefault = true;
+                        parameterDefinition.ConfigDefaultValue = configParamValue?.ToString();
+                    }
+                }
+            }
+
+            // Generating exposed stored-procedure query/mutation name and adding to the dictionary mapping it to its entity name.
+            GraphQLStoredProcedureExposedNameToEntityNameMap.TryAdd(GenerateStoredProcedureGraphQLFieldName(entityName, procedureEntity), entityName);
+        }
+
+        /// <summary>
+        /// Takes a string version of a sql date/time type and returns its corresponding DbType.
+        /// </summary>
+        /// <param name="sqlDbTypeName">Name of the sqlDbType.<</param>
+        /// <param name="dbType">DbType of the parameter corresponding to its sqlDbTypeName.</param>
+        /// <returns>Returns true when the given sqlDbTypeName datetime type is supported by DAB and resolve it to its corresponding DbType, else false.</returns>
+        private bool TryResolveDbType(string sqlDbTypeName, out DbType dbType)
+        {
+            if (Enum.TryParse(sqlDbTypeName, ignoreCase: true, out SqlDbType sqlDbType))
+            {
+                // For MsSql, all the date time types i.e. date, smalldatetime, datetime, datetime2 map to System.DateTime system type.
+                // Hence we cannot directly determine the DbType from the system type.
+                // However, to make sure that the database correctly interprets these datatypes, it is necessary to correctly
+                // populate the DbTypes.
+                return TypeHelper.TryGetDbTypeFromSqlDbDateTimeType(sqlDbType, out dbType);
+            }
+            else
+            {
+                // This code should never be hit because every sqlDbTypeName must have a corresponding sqlDbType.
+                // However, when a new data type is introduced in MsSql which maps to .NET type of DateTime, this code block
+                // will be hit. Returning false instead of throwing an exception in that case prevents the engine from crashing.
+                _logger.LogWarning("Could not determine DbType for SqlDb type of {sqlDbTypeName}", sqlDbTypeName);
+                dbType = 0;
+                return false;
+            }
+        }
     }
 }

src/Core/Services/MetadataProviders/SqlMetadataProvider.cs

@@ -50,7 +50,7 @@ public abstract class SqlMetadataProvider<ConnectionT, DataAdapterT, CommandT> :
 
         protected IQueryExecutor QueryExecutor { get; }
 
-        private const int NUMBER_OF_RESTRICTIONS = 4;
+        protected const int NUMBER_OF_RESTRICTIONS = 4;
 
         protected string ConnectionString { get; init; }
 
@@ -320,7 +320,7 @@ private void LogPrimaryKeys()
         /// <summary>
         /// Verify that the stored procedure exists in the database schema, then populate its database object parameters accordingly
         /// </summary>
-        private async Task FillSchemaForStoredProcedureAsync(
+        protected virtual async Task FillSchemaForStoredProcedureAsync(
             Entity procedureEntity,
             string entityName,
             string schemaName,
@@ -374,15 +374,16 @@ private async Task FillSchemaForStoredProcedureAsync(
             {
                 // row["DATA_TYPE"] has value type string so a direct cast to System.Type is not supported.
                 // See https://learn.microsoft.com/en-us/dotnet/framework/data/adonet/sql-server-data-type-mappings
-                Type systemType = SqlToCLRType((string)row["DATA_TYPE"]);
+                string sqlType = (string)row["DATA_TYPE"];
+                Type systemType = SqlToCLRType(sqlType);
+                ParameterDefinition paramDefinition = new()
+                {
+                    SystemType = systemType,
+                    DbType = TypeHelper.GetDbTypeFromSystemType(systemType)
+                };
+
                 // Add to parameters dictionary without the leading @ sign
-                storedProcedureDefinition.Parameters.TryAdd(((string)row["PARAMETER_NAME"])[1..],
-                    new()
-                    {
-                        SystemType = systemType,
-                        DbType = TypeHelper.GetDbTypeFromSystemType(systemType)
-                    }
-                );
+                storedProcedureDefinition.Parameters.TryAdd(((string)row["PARAMETER_NAME"])[1..], paramDefinition);
             }
 
             // Loop through parameters specified in config, throw error if not found in schema
@@ -1097,13 +1098,11 @@ private async Task PopulateSourceDefinitionAsync(
                        subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization);
                 }
 
-                Type systemTypeOfColumn = (Type)columnInfoFromAdapter["DataType"];
                 ColumnDefinition column = new()
                 {
                     IsNullable = (bool)columnInfoFromAdapter["AllowDBNull"],
                     IsAutoGenerated = (bool)columnInfoFromAdapter["IsAutoIncrement"],
-                    SystemType = systemTypeOfColumn,
-                    DbType = TypeHelper.GetDbTypeFromSystemType(systemTypeOfColumn),
+                    SystemType = (Type)columnInfoFromAdapter["DataType"],
                     // An auto-increment column is also considered as a read-only column. For other types of read-only columns,
                     // the flag is populated later via PopulateColumnDefinitionsWithReadOnlyFlag() method.
                     IsReadOnly = (bool)columnInfoFromAdapter["IsAutoIncrement"]
@@ -1117,7 +1116,8 @@ private async Task PopulateSourceDefinitionAsync(
             }
 
             DataTable columnsInTable = await GetColumnsAsync(schemaName, tableName);
-            PopulateColumnDefinitionWithHasDefault(
+
+            PopulateColumnDefinitionWithHasDefaultAndDbType(
                 sourceDefinition,
                 columnsInTable);
 
@@ -1400,9 +1400,9 @@ protected virtual async Task<DataTable> GetColumnsAsync(
         }
 
         /// <summary>
-        /// Populates the column definition with HasDefault property.
+        /// Helper method to populate the column definition with HasDefault and DbType properties.
         /// </summary>
-        private static void PopulateColumnDefinitionWithHasDefault(
+        protected virtual void PopulateColumnDefinitionWithHasDefaultAndDbType(
             SourceDefinition sourceDefinition,
             DataTable allColumnsInTable)
         {
@@ -1419,6 +1419,8 @@ private static void PopulateColumnDefinitionWithHasDefault(
                     {
                         columnDefinition.DefaultValue = columnInfo["COLUMN_DEFAULT"];
                     }
+
+                    columnDefinition.DbType = TypeHelper.GetDbTypeFromSystemType(columnDefinition.SystemType);
                 }
             }
         }

src/Core/Services/ResolverMiddleware.cs

@@ -18,7 +18,7 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Primitives;
 using NodaTime.Text;
-using static Azure.DataApiBuilder.Service.GraphQLBuilder.GraphQLTypes.SupportedTypes;
+using static Azure.DataApiBuilder.Service.GraphQLBuilder.GraphQLTypes.SupportedHotChocolateTypes;
 
 namespace Azure.DataApiBuilder.Core.Services
 {