From 021797c8cf96f3f242e0f7d644f3a2f77c6ed2c6 Mon Sep 17 00:00:00 2001 From: Richard Towers Date: Wed, 16 Apr 2025 15:12:46 +0100 Subject: [PATCH 1/2] WIP recursive link expansion --- app/models/edition.rb | 4 ++ .../recursive_link_expansion/base_edition.rb | 25 +++++++++++ .../forward_edition_links.rb | 24 +++++++++++ .../forward_link_set_links.rb | 24 +++++++++++ .../link_expansion_rules.rb | 22 ++++++++++ .../recursive_link_expansion/lookahead.rb | 41 +++++++++++++++++++ .../recursive_link_expansion.rb | 37 +++++++++++++++++ .../reverse_edition_links.rb | 24 +++++++++++ .../reverse_link_set_links.rb | 24 +++++++++++ 9 files changed, 225 insertions(+) create mode 100644 app/queries/recursive_link_expansion/base_edition.rb create mode 100644 app/queries/recursive_link_expansion/forward_edition_links.rb create mode 100644 app/queries/recursive_link_expansion/forward_link_set_links.rb create mode 100644 app/queries/recursive_link_expansion/link_expansion_rules.rb create mode 100644 app/queries/recursive_link_expansion/lookahead.rb create mode 100644 app/queries/recursive_link_expansion/recursive_link_expansion.rb create mode 100644 app/queries/recursive_link_expansion/reverse_edition_links.rb create mode 100644 app/queries/recursive_link_expansion/reverse_link_set_links.rb diff --git a/app/models/edition.rb b/app/models/edition.rb index 55db8900e4..c7b8593699 100644 --- a/app/models/edition.rb +++ b/app/models/edition.rb @@ -223,6 +223,10 @@ def web_url Plek.website_root + base_path end + def linked_editions + Queries::RecursiveLinkExpansion::RecursiveLinkExpansion.new(self).call + end + private def renderable_content? diff --git a/app/queries/recursive_link_expansion/base_edition.rb b/app/queries/recursive_link_expansion/base_edition.rb new file mode 100644 index 0000000000..a15a9b31e5 --- /dev/null +++ b/app/queries/recursive_link_expansion/base_edition.rb @@ -0,0 +1,25 @@ +module Queries + module RecursiveLinkExpansion + ## + # Selects the columns required for recursive link expansion from the base edition (the root edition in the tree) + class BaseEdition + def initialize(edition_with_document, links = []) + @edition = edition_with_document + @links = links + end + + def call + Edition.with_document.where(id: @edition.id).select( + "'0 base' as type", + "'{}'::text[] as path", + "documents.content_id", + "documents.locale", + "editions.id as edition_id", + "0 as position", + "editions.state", + ActiveRecord::Base.send(:sanitize_sql_array, ["?::jsonb as links", @links.to_json]), + ) + end + end + end +end diff --git a/app/queries/recursive_link_expansion/forward_edition_links.rb b/app/queries/recursive_link_expansion/forward_edition_links.rb new file mode 100644 index 0000000000..222c085d6a --- /dev/null +++ b/app/queries/recursive_link_expansion/forward_edition_links.rb @@ -0,0 +1,24 @@ +module Queries + module RecursiveLinkExpansion + class ForwardEditionLinks + def call + # TODO - parameterize locale / state + Edition + .from("lookahead") + .joins("INNER JOIN links ON (links.edition_id = lookahead.edition_id AND links.link_type = lookahead.type AND lookahead.reverse = false)") + .joins("INNER JOIN documents ON (documents.content_id = links.target_content_id AND documents.locale='en')") + .joins("INNER JOIN editions ON (editions.document_id = documents.id AND editions.state='published')") + .select( + "'1 forward edition link' as type", + "path || lookahead.content_id::text || lookahead.type AS path", + "documents.content_id", + "documents.locale", + "editions.id as edition_id", + "links.position", + "editions.state", + "lookahead.links", + ) + end + end + end +end \ No newline at end of file diff --git a/app/queries/recursive_link_expansion/forward_link_set_links.rb b/app/queries/recursive_link_expansion/forward_link_set_links.rb new file mode 100644 index 0000000000..37eef79000 --- /dev/null +++ b/app/queries/recursive_link_expansion/forward_link_set_links.rb @@ -0,0 +1,24 @@ +module Queries + module RecursiveLinkExpansion + class ForwardLinkSetLinks + def call + # TODO - parameterize locale / state + Edition + .from("lookahead") + .joins("INNER JOIN links ON (links.link_set_content_id = lookahead.content_id AND links.link_type = lookahead.type AND lookahead.reverse = false)") + .joins("INNER JOIN documents ON (documents.content_id = links.target_content_id AND documents.locale='en')") + .joins("INNER JOIN editions ON (editions.document_id = documents.id AND editions.state='published')") + .select( + "'2 forward link set link' as type", + "path || lookahead.content_id::text || lookahead.type AS path", + "documents.content_id", + "documents.locale", + "editions.id as edition_id", + "links.position", + "editions.state", + "lookahead.links", + ) + end + end + end +end \ No newline at end of file diff --git a/app/queries/recursive_link_expansion/link_expansion_rules.rb b/app/queries/recursive_link_expansion/link_expansion_rules.rb new file mode 100644 index 0000000000..ca07917346 --- /dev/null +++ b/app/queries/recursive_link_expansion/link_expansion_rules.rb @@ -0,0 +1,22 @@ +module Queries + module RecursiveLinkExpansion + class LinkExpansionRules + RULES_BY_SCHEMA_NAME = { + ministers_index: [ + { "type": "ordered_cabinet_ministers", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [] }] }] }, + { "type": "ordered_also_attends_cabinet", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [] }] }] }, + { "type": "ordered_ministerial_departments", "reverse": false, "links": [{ "type": "ordered_ministers", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [{ "type": "ordered_roles", "reverse": true, "links": [] }] }] }] }] }, + { "type": "ordered_assistant_whips", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [] }] }] }, + { "type": "ordered_baronesses_and_lords_in_waiting_whips", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [] }] }] }, + { "type": "ordered_house_lords_whips", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [] }] }] }, + { "type": "ordered_house_of_commons_whips", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [] }] }] }, + { "type": "ordered_junior_lords_of_the_treasury_whips", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [] }] }] } + ] + }.with_indifferent_access.freeze + + def self.for(schema_name) + RULES_BY_SCHEMA_NAME.fetch(schema_name) + end + end + end +end diff --git a/app/queries/recursive_link_expansion/lookahead.rb b/app/queries/recursive_link_expansion/lookahead.rb new file mode 100644 index 0000000000..f15e7e0e28 --- /dev/null +++ b/app/queries/recursive_link_expansion/lookahead.rb @@ -0,0 +1,41 @@ +module Queries + module RecursiveLinkExpansion + ## + # Expects to be called with a CTE called "linked_editions" already defined. + # Defines a "lookahead" from a content id or an edition id and a link type. + # + # This allows the various types of link query (forward / reverse link set / edition links) to + # follow the link type from the edition / content id to the next edition / content_id. + # + # links is required in the constructor because we need to provide the maximum number of links that + # this CTE could return, as otherwise the postgresql planner will assume jsonb_to_recordset() will return 100 rows + # which pushes it towards inefficient plans. + class Lookahead + def initialize(links) + @links = links.map(&:deep_symbolize_keys) + end + + def call + Arel.sql(<<~SQL + SELECT content_id, edition_id, path, lookahead.type, lookahead.reverse, lookahead.links + FROM linked_editions + CROSS JOIN LATERAL ( + SELECT * from jsonb_to_recordset(linked_editions.links) AS lookahead(type varchar, reverse boolean, links jsonb) + LIMIT #{max_links_count(@links)} + ) AS lookahead + SQL + ) + end + + private + + def max_links_count(links) + case links + in { links: Array => ls } then max_links_count(ls) + in Hash | [] then 0 + in Array then [links.length, links.map { max_links_count(_1) }.max].max + end + end + end + end +end diff --git a/app/queries/recursive_link_expansion/recursive_link_expansion.rb b/app/queries/recursive_link_expansion/recursive_link_expansion.rb new file mode 100644 index 0000000000..9f4b39219b --- /dev/null +++ b/app/queries/recursive_link_expansion/recursive_link_expansion.rb @@ -0,0 +1,37 @@ +module Queries + module RecursiveLinkExpansion + class RecursiveLinkExpansion + + def initialize(edition) + @edition = edition + @links = ::Queries::RecursiveLinkExpansion::LinkExpansionRules.for(edition.schema_name) + end + + def call + base_case = BaseEdition.new(@edition, @links).call + + recursive_case = Edition.with( + lookahead: Lookahead.new(@links).call, + forward_link_set_links: ForwardLinkSetLinks.new.call, + forward_edition_links: ForwardEditionLinks.new.call, + reverse_link_set_links: ReverseLinkSetLinks.new.call, + reverse_edition_links: ReverseEditionLinks.new.call, + all_links: [ + Arel.sql("SELECT * from forward_link_set_links"), + Arel.sql("SELECT * from forward_edition_links"), + Arel.sql("SELECT * from reverse_link_set_links"), + Arel.sql("SELECT * from reverse_edition_links"), + ] + ).from("all_links").select("all_links.*") + + Edition.with_recursive( + linked_editions: [ + base_case, + Arel.sql(recursive_case.to_sql) # Workaround to ensure the recursive case is wrapped in parens + ] + ).from("linked_editions").select("linked_editions.*") + + end + end + end +end diff --git a/app/queries/recursive_link_expansion/reverse_edition_links.rb b/app/queries/recursive_link_expansion/reverse_edition_links.rb new file mode 100644 index 0000000000..f35b7c48a8 --- /dev/null +++ b/app/queries/recursive_link_expansion/reverse_edition_links.rb @@ -0,0 +1,24 @@ +module Queries + module RecursiveLinkExpansion + class ReverseEditionLinks + def call + # TODO - parameterize locale / state + Edition + .from("lookahead") + .joins("INNER JOIN links ON (links.target_content_id = lookahead.content_id AND links.link_type = lookahead.type AND lookahead.reverse = true)") + .joins("INNER JOIN editions ON (editions.id = links.edition_id AND editions.state='published')") + .joins("INNER JOIN documents ON (documents.id = editions.document_id AND documents.locale='en')") + .select( + "'1 forward edition link' as type", + "path || lookahead.content_id::text || lookahead.type AS path", + "documents.content_id", + "documents.locale", + "editions.id as edition_id", + "links.position", + "editions.state", + "lookahead.links", + ) + end + end + end +end \ No newline at end of file diff --git a/app/queries/recursive_link_expansion/reverse_link_set_links.rb b/app/queries/recursive_link_expansion/reverse_link_set_links.rb new file mode 100644 index 0000000000..8c0a0111c1 --- /dev/null +++ b/app/queries/recursive_link_expansion/reverse_link_set_links.rb @@ -0,0 +1,24 @@ +module Queries + module RecursiveLinkExpansion + class ReverseLinkSetLinks + def call + # TODO - parameterize locale / state + Edition + .from("lookahead") + .joins("INNER JOIN links ON links.target_content_id = lookahead.content_id and links.link_type = lookahead.type and lookahead.reverse = true") + .joins("INNER JOIN documents ON documents.content_id = links.link_set_content_id and documents.locale IN ('en')") + .joins("INNER JOIN editions ON editions.document_id = documents.id AND editions.state IN ('published')") + .select( + "'4 reverse link set link' as type", + "path || lookahead.content_id::text || lookahead.type as path", + "documents.content_id", + "documents.locale", + "editions.id as edition_id", + "links.position as position", + "editions.state as state", + "lookahead.links", + ) + end + end + end +end \ No newline at end of file From e06f9ea9164a7043bcc391567a989613942513fb Mon Sep 17 00:00:00 2001 From: Richard Towers Date: Tue, 22 Apr 2025 13:29:59 +0100 Subject: [PATCH 2/2] Automatic rubocop fixes --- .../forward_edition_links.rb | 6 +++--- .../forward_link_set_links.rb | 6 +++--- .../link_expansion_rules.rb | 4 ++-- .../recursive_link_expansion/lookahead.rb | 18 +++++++++--------- .../recursive_link_expansion.rb | 8 +++----- .../reverse_edition_links.rb | 6 +++--- .../reverse_link_set_links.rb | 6 +++--- 7 files changed, 26 insertions(+), 28 deletions(-) diff --git a/app/queries/recursive_link_expansion/forward_edition_links.rb b/app/queries/recursive_link_expansion/forward_edition_links.rb index 222c085d6a..87d1a6303f 100644 --- a/app/queries/recursive_link_expansion/forward_edition_links.rb +++ b/app/queries/recursive_link_expansion/forward_edition_links.rb @@ -2,7 +2,7 @@ module Queries module RecursiveLinkExpansion class ForwardEditionLinks def call - # TODO - parameterize locale / state + # TODO: - parameterize locale / state Edition .from("lookahead") .joins("INNER JOIN links ON (links.edition_id = lookahead.edition_id AND links.link_type = lookahead.type AND lookahead.reverse = false)") @@ -17,8 +17,8 @@ def call "links.position", "editions.state", "lookahead.links", - ) + ) end end end -end \ No newline at end of file +end diff --git a/app/queries/recursive_link_expansion/forward_link_set_links.rb b/app/queries/recursive_link_expansion/forward_link_set_links.rb index 37eef79000..300c79d11a 100644 --- a/app/queries/recursive_link_expansion/forward_link_set_links.rb +++ b/app/queries/recursive_link_expansion/forward_link_set_links.rb @@ -2,7 +2,7 @@ module Queries module RecursiveLinkExpansion class ForwardLinkSetLinks def call - # TODO - parameterize locale / state + # TODO: - parameterize locale / state Edition .from("lookahead") .joins("INNER JOIN links ON (links.link_set_content_id = lookahead.content_id AND links.link_type = lookahead.type AND lookahead.reverse = false)") @@ -17,8 +17,8 @@ def call "links.position", "editions.state", "lookahead.links", - ) + ) end end end -end \ No newline at end of file +end diff --git a/app/queries/recursive_link_expansion/link_expansion_rules.rb b/app/queries/recursive_link_expansion/link_expansion_rules.rb index ca07917346..2c26dcd627 100644 --- a/app/queries/recursive_link_expansion/link_expansion_rules.rb +++ b/app/queries/recursive_link_expansion/link_expansion_rules.rb @@ -10,8 +10,8 @@ class LinkExpansionRules { "type": "ordered_baronesses_and_lords_in_waiting_whips", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [] }] }] }, { "type": "ordered_house_lords_whips", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [] }] }] }, { "type": "ordered_house_of_commons_whips", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [] }] }] }, - { "type": "ordered_junior_lords_of_the_treasury_whips", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [] }] }] } - ] + { "type": "ordered_junior_lords_of_the_treasury_whips", "reverse": false, "links": [{ "type": "person", "reverse": true, "links": [{ "type": "role", "reverse": false, "links": [] }] }] }, + ], }.with_indifferent_access.freeze def self.for(schema_name) diff --git a/app/queries/recursive_link_expansion/lookahead.rb b/app/queries/recursive_link_expansion/lookahead.rb index f15e7e0e28..3ab9e2412f 100644 --- a/app/queries/recursive_link_expansion/lookahead.rb +++ b/app/queries/recursive_link_expansion/lookahead.rb @@ -16,15 +16,15 @@ def initialize(links) end def call - Arel.sql(<<~SQL - SELECT content_id, edition_id, path, lookahead.type, lookahead.reverse, lookahead.links - FROM linked_editions - CROSS JOIN LATERAL ( - SELECT * from jsonb_to_recordset(linked_editions.links) AS lookahead(type varchar, reverse boolean, links jsonb) - LIMIT #{max_links_count(@links)} - ) AS lookahead - SQL - ) + Arel.sql(<<~SQL, + SELECT content_id, edition_id, path, lookahead.type, lookahead.reverse, lookahead.links + FROM linked_editions + CROSS JOIN LATERAL ( + SELECT * from jsonb_to_recordset(linked_editions.links) AS lookahead(type varchar, reverse boolean, links jsonb) + LIMIT #{max_links_count(@links)} + ) AS lookahead + SQL + ) end private diff --git a/app/queries/recursive_link_expansion/recursive_link_expansion.rb b/app/queries/recursive_link_expansion/recursive_link_expansion.rb index 9f4b39219b..48424f06ee 100644 --- a/app/queries/recursive_link_expansion/recursive_link_expansion.rb +++ b/app/queries/recursive_link_expansion/recursive_link_expansion.rb @@ -1,7 +1,6 @@ module Queries module RecursiveLinkExpansion class RecursiveLinkExpansion - def initialize(edition) @edition = edition @links = ::Queries::RecursiveLinkExpansion::LinkExpansionRules.for(edition.schema_name) @@ -21,16 +20,15 @@ def call Arel.sql("SELECT * from forward_edition_links"), Arel.sql("SELECT * from reverse_link_set_links"), Arel.sql("SELECT * from reverse_edition_links"), - ] + ], ).from("all_links").select("all_links.*") Edition.with_recursive( linked_editions: [ base_case, - Arel.sql(recursive_case.to_sql) # Workaround to ensure the recursive case is wrapped in parens - ] + Arel.sql(recursive_case.to_sql), # Workaround to ensure the recursive case is wrapped in parens + ], ).from("linked_editions").select("linked_editions.*") - end end end diff --git a/app/queries/recursive_link_expansion/reverse_edition_links.rb b/app/queries/recursive_link_expansion/reverse_edition_links.rb index f35b7c48a8..d0961ee65d 100644 --- a/app/queries/recursive_link_expansion/reverse_edition_links.rb +++ b/app/queries/recursive_link_expansion/reverse_edition_links.rb @@ -2,7 +2,7 @@ module Queries module RecursiveLinkExpansion class ReverseEditionLinks def call - # TODO - parameterize locale / state + # TODO: - parameterize locale / state Edition .from("lookahead") .joins("INNER JOIN links ON (links.target_content_id = lookahead.content_id AND links.link_type = lookahead.type AND lookahead.reverse = true)") @@ -17,8 +17,8 @@ def call "links.position", "editions.state", "lookahead.links", - ) + ) end end end -end \ No newline at end of file +end diff --git a/app/queries/recursive_link_expansion/reverse_link_set_links.rb b/app/queries/recursive_link_expansion/reverse_link_set_links.rb index 8c0a0111c1..d11c05010c 100644 --- a/app/queries/recursive_link_expansion/reverse_link_set_links.rb +++ b/app/queries/recursive_link_expansion/reverse_link_set_links.rb @@ -2,7 +2,7 @@ module Queries module RecursiveLinkExpansion class ReverseLinkSetLinks def call - # TODO - parameterize locale / state + # TODO: - parameterize locale / state Edition .from("lookahead") .joins("INNER JOIN links ON links.target_content_id = lookahead.content_id and links.link_type = lookahead.type and lookahead.reverse = true") @@ -17,8 +17,8 @@ def call "links.position as position", "editions.state as state", "lookahead.links", - ) + ) end end end -end \ No newline at end of file +end