Make the category/group name unique index case insensitive

lib/sanbase/metric/category/display_order.ex

@@ -16,56 +16,92 @@ defmodule Sanbase.Metric.Category.DisplayOrder do
   Returns the same shape as DisplayOrder.get_ordered_metrics for compatibility.
   The ordering follows: category order → ungrouped mappings → groups (by order) →
   mappings within groups → ui_metadata within mappings.
+  Pass `true` as the second argument to include metrics without UI metadata.
   """
-  @spec get_ordered_metrics([MetricCategory.t()]) :: %{metrics: [map()], categories: [map()]}
-  def get_ordered_metrics(categories) do
+  @spec get_ordered_metrics([MetricCategory.t()], boolean()) :: %{
+          metrics: [map()],
+          categories: [map()]
+        }
+  def get_ordered_metrics(categories \\ nil, include_without_ui_metadata \\ false) do
+    # The returned metrics are taken from the mappings preloads in the categories.
+    # Make sure to preload the necessary associations when calling this function.
+    categories = categories || MetricCategory.list_ordered()
+    categories = Enum.sort_by(categories, & &1.display_order, :asc)
+
+    raise_if_no_preloads(categories)
+
     ordered_category_data = Enum.map(categories, fn cat -> %{id: cat.id, name: cat.name} end)
 
+    # This map is used to enrich the output with registry metric data
+    # It it is not used to generate the list of metrics.
+    # The list of metrics
     registry_metrics = build_registry_map()
 
     metrics =
       categories
-      |> Enum.flat_map(&flatten_category/1)
+      |> Enum.flat_map(&flatten_category(&1, include_without_ui_metadata))
       |> Enum.map(&transform_to_output_format(&1, registry_metrics))
 
-    %{
-      metrics: metrics,
-      categories: ordered_category_data
-    }
+    %{metrics: metrics, categories: ordered_category_data}
   end
 
-  defp flatten_category(category) do
-    ungrouped_metrics = flatten_mappings(category.mappings, category, nil)
+  defp raise_if_no_preloads(categories) do
+    if Enum.any?(categories, fn cat -> not Ecto.assoc_loaded?(cat.mappings) end),
+      do: raise("Mappings must be preloaded for all categories")
+
+    if Enum.any?(categories, fn cat ->
+         Enum.any?(cat.mappings, fn mapping ->
+           not Ecto.assoc_loaded?(mapping.metric_registry) or
+             not Ecto.assoc_loaded?(mapping.ui_metadata_list)
+         end)
+       end),
+       do: raise("Groups must be preloaded for all categories")
+  end
+
+  defp flatten_category(category, include_without_ui_metadata) do
+    ungrouped_metrics =
+      flatten_mappings(category.mappings, category, _group = nil, include_without_ui_metadata)
 
     grouped_metrics =
       category.groups
       |> Enum.flat_map(fn group ->
-        flatten_mappings(group.mappings, category, group)
+        flatten_mappings(group.mappings, category, group, include_without_ui_metadata)
       end)
 
     ungrouped_metrics ++ grouped_metrics
   end
 
-  defp flatten_mappings(mappings, category, group) do
+  defp flatten_mappings(mappings, category, group, include_without_ui_metadata) do
     mappings
     |> Enum.filter(&mapping_belongs_to_group?(&1, group))
-    |> Enum.flat_map(&expand_mapping_with_ui_metadata(&1, category, group))
+    |> Enum.flat_map(&expand_mapping(&1, category, group, include_without_ui_metadata))
   end
 
-  defp mapping_belongs_to_group?(mapping, nil), do: is_nil(mapping.group_id)
+  defp mapping_belongs_to_group?(mapping, _group = nil), do: is_nil(mapping.group_id)
   defp mapping_belongs_to_group?(mapping, group), do: mapping.group_id == group.id
 
-  defp expand_mapping_with_ui_metadata(mapping, category, group) do
-    if mapping.ui_metadata_list == [] do
-      [{mapping, nil, category, group}]
-    else
-      Enum.map(mapping.ui_metadata_list, fn ui_metadata ->
-        {mapping, ui_metadata, category, group}
-      end)
+  defp expand_mapping(mapping, category, group, include_without_ui_metadata) do
+    cond do
+      mapping.ui_metadata_list == [] and false == include_without_ui_metadata ->
+        []
+
+      mapping.ui_metadata_list == [] and true == include_without_ui_metadata ->
+        [{mapping, nil, category, group}]
+
+      true ->
+        Enum.map(mapping.ui_metadata_list, fn ui_metadata ->
+          {mapping, ui_metadata, category, group}
+        end)
     end
   end
 
   defp transform_to_output_format({mapping, ui_metadata, category, group}, registry_metrics) do
+    # Determine the metric name (not the human readable name) of the metric.
+    # In case of mapping with ui_metadata, the metric name comes from it.
+    # The mapping can be linked to a registry metric, or to a code metric.
+    # In case of include_without_ui_metadata=true, there is no ui_metadata,
+    # so the metric name comes from the mapping.
+    # Such records without ui_metadata will be used only locally for tests
     metric_name = determine_metric_name(mapping, ui_metadata)
     registry_metric = Map.get(registry_metrics, metric_name)
 
@@ -110,10 +146,16 @@ defmodule Sanbase.Metric.Category.DisplayOrder do
     }
   end
 
-  defp get_ui_human_readable_name(nil, metric_name), do: metric_name
-
   defp get_ui_human_readable_name(ui_metadata, metric_name) do
-    ui_metadata.ui_human_readable_name || metric_name
+    if ui_metadata && ui_metadata.ui_human_readable_name do
+      ui_metadata.ui_human_readable_name
+    else
+      with {:ok, human_readable_name} <- Sanbase.Metric.human_readable_name(metric_name) do
+        human_readable_name
+      else
+        _ -> metric_name
+      end
+    end
   end
 
   defp ui_metadata_or_mapping_field(nil, mapping, field), do: Map.get(mapping, field)

lib/sanbase/metric/category/metric_category_mapping.ex

@@ -87,7 +87,7 @@ defmodule Sanbase.Metric.Category.MetricCategoryMapping do
   def create(attrs) do
     %__MODULE__{}
     |> changeset(attrs)
-    |> Repo.insert()
+    |> Repo.insert(on_conflict: :nothing, returning: true)
   end
 
   @doc """

lib/sanbase/metric/category/metric_group.ex

@@ -62,7 +62,7 @@ defmodule Sanbase.Metric.Category.MetricGroup do
   def create(attrs) do
     %__MODULE__{}
     |> changeset(attrs)
-    |> Repo.insert()
+    |> Repo.insert(on_conflict: :nothing)
   end
 
   @doc """

lib/sanbase/metric/category/metric_ui_metadata.ex

@@ -78,7 +78,7 @@ defmodule Sanbase.Metric.UIMetadata do
   def create(attrs) do
     %__MODULE__{}
     |> changeset(attrs)
-    |> Repo.insert()
+    |> Repo.insert(on_conflict: :nothing)
   end
 
   @doc """

lib/sanbase/metric/category/scripts/copy_categories.ex

@@ -34,6 +34,7 @@ defmodule Sanbase.Metric.Category.Scripts.CopyCategories do
       case map do
         %{metric_registry_id: metric_registry_id} when is_integer(metric_registry_id) ->
           Sanbase.Metric.Category.create_mapping(%{
+            # metric: map.metric,
             metric_registry_id: metric_registry_id,
             category_id: category_id,
             group_id: group_id,

livebooks/auto_categorize_social_metrics.livemd

@@ -0,0 +1,159 @@
+# Auto-categorize Social Registry metrics
+
+## Section
+
+```elixir
+drop_existing_categories_and_groups = true
+add_social_metrics_new_categories = true
+add_social_metrics_ui_metadata = false
+```
+
+```elixir
+all_metrics = Sanbase.Metric.available_metrics()
+length(all_metrics)
+```
+
+```elixir
+categorized_metrics =
+  Sanbase.Metric.Category.MetricCategoryMapping.list_all()
+  |> Enum.flat_map(fn
+    %{metric_registry: %{} = registry} ->
+      [registry] |> Sanbase.Metric.Registry.resolve() |> Enum.map(& &1.metric)
+
+    %{metric: metric} when is_binary(metric) ->
+      [metric]
+  end)
+
+not_categorized_metrics = all_metrics -- categorized_metrics
+
+IO.puts("""
+Categorized: #{length(categorized_metrics)}
+Not Categorized: #{length(not_categorized_metrics)}
+""")
+```
+
+```elixir
+category_name_to_id = Sanbase.Metric.Category.list_categories() |> Map.new(&{&1.name, &1.id})
+```
+
+```elixir
+# Social groups
+social_groups = Sanbase.Metric.Category.list_groups_by_category(category_name_to_id["Social"])
+
+max_display_order =
+  case social_groups do
+    [] ->
+      1
+
+    [_ | _] = list ->
+      Enum.max_by(list, & &1.display_order).display_order
+  end
+
+groups = ["Sentiment", "Social Volume", "Social Dominance"]
+
+Enum.with_index(groups, max_display_order)
+|> Enum.map(fn {group_name, display_order} ->
+  category_id = category_name_to_id["Social"]
+  group = Sanbase.Metric.Category.MetricGroup.get_by_name_and_category(group_name, category_id)
+  group && Sanbase.Metric.Category.delete_group(group)
+
+  Sanbase.Metric.Category.create_group(%{
+    name: group_name,
+    category_id: category_id,
+    display_order: display_order
+  })
+end)
+
+group_name_to_id =
+  Sanbase.Metric.Category.list_groups_with_category() |> Map.new(&{&1.name, &1.id})
+```
+
+```elixir
+group_name_to_id =
+  Sanbase.Metric.Category.list_groups_with_category() |> Map.new(&{&1.name, &1.id})
+```
+
+```elixir
+deduce_category = fn m ->
+  cond do
+    String.contains?(m, [
+      "_change_",
+      "_1h",
+      "_v2",
+      "_1d",
+      "_5m",
+      "moving_average",
+      "volume_consumed"
+    ]) ->
+      nil
+
+    # Sentiment
+    m =~ "sentiment" and
+        String.contains?(m, ["positive", "negative", "neutral", "bullish", "bearish"]) ->
+      {"Social", "Sentiment"}
+
+    # Social Volume
+    m =~ "social_volume" ->
+      {"Social", "Social Volume"}
+
+    # Social Dominance
+    m =~ "social_dominance" ->
+      {"Social", "Social Dominance"}
+
+    true ->
+      nil
+  end
+end
+```
+
+```elixir
+metric_registry_name_to_id =
+  Sanbase.Metric.Registry.all()
+  |> Sanbase.Metric.Registry.resolve()
+  |> Map.new(&{&1.metric, &1.id})
+
+all_metrics
+|> Enum.filter(&String.contains?(&1, "social_dominance"))
+|> Enum.filter(&Map.has_key?(metric_registry_name_to_id, &1))
+|> IO.inspect()
+|> Enum.map(fn m ->
+  registry_id = metric_registry_name_to_id[m]
+
+  case deduce_category.(m) do
+    nil ->
+      :ok
+
+    {category, group} ->
+      # Drop existing mapping
+      case Sanbase.Metric.Category.get_mapping_by_metric_registry_id(registry_id) do
+        {:ok, [existing_mapping]} ->
+          Sanbase.Metric.Category.delete_mapping(existing_mapping)
+
+        _ ->
+          :ok
+      end
+
+      # Create new mapping
+      {:ok, mapping} =
+        Sanbase.Metric.Category.create_mapping(%{
+          metric_registry_id: registry_id,
+          category_id: category_name_to_id[category],
+          group_id: group_name_to_id[group],
+          display_order: 1
+        })
+
+      # Create UI Metadata
+      {:ok, _} =
+        Sanbase.Metric.Category.create_ui_metadata(%{
+          metric: m,
+          display_order_in_mapping: 1,
+          metric_category_mapping_id: mapping.id,
+          chart_style: "line"
+        })
+  end
+end)
+```
+
+```elixir
+group_name_to_id["Social Dominance"]
+```

priv/repo/migrations/20251030132017_improve_categorization_unique_index.exs

@@ -0,0 +1,18 @@
+defmodule Sanbase.Repo.Migrations.ImproveCategorizationUniqueIndex do
+  use Ecto.Migration
+
+  def change do
+    drop(unique_index(:metric_categories, [:name]))
+    drop(unique_index(:metric_groups, [:name, :category_id]))
+
+    create(
+      unique_index(:metric_categories, ["(lower(name))"], name: :metric_categories_name_index)
+    )
+
+    create(
+      unique_index(:metric_groups, ["(lower(name))", :category_id],
+        name: :metric_groups_name_category_id_index
+      )
+    )
+  end
+end

priv/repo/structure.sql

@@ -8343,7 +8343,7 @@ CREATE INDEX menus_user_id_index ON public.menus USING btree (user_id);
 -- Name: metric_categories_name_index; Type: INDEX; Schema: public; Owner: -
 --
 
-CREATE UNIQUE INDEX metric_categories_name_index ON public.metric_categories USING btree (name);
+CREATE UNIQUE INDEX metric_categories_name_index ON public.metric_categories USING btree (lower((name)::text));
 
 
 --
@@ -8371,7 +8371,7 @@ CREATE UNIQUE INDEX metric_category_mappings_module_metric_category_id_group_id_
 -- Name: metric_groups_name_category_id_index; Type: INDEX; Schema: public; Owner: -
 --
 
-CREATE UNIQUE INDEX metric_groups_name_category_id_index ON public.metric_groups USING btree (name, category_id);
+CREATE UNIQUE INDEX metric_groups_name_category_id_index ON public.metric_groups USING btree (lower((name)::text), category_id);
 
 
 --
@@ -11147,3 +11147,4 @@ INSERT INTO public."schema_migrations" (version) VALUES (20251023083446);
 INSERT INTO public."schema_migrations" (version) VALUES (20251023114153);
 INSERT INTO public."schema_migrations" (version) VALUES (20251027142731);
 INSERT INTO public."schema_migrations" (version) VALUES (20251027154645);
+INSERT INTO public."schema_migrations" (version) VALUES (20251030132017);