diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a010a5..6d7df5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: - name: Run rubocop run: | bundle exec rubocop + - name: Run tests run: | bundle exec rspec \ No newline at end of file diff --git a/.gitignore b/.gitignore index 559f8e2..b5403a8 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ # rspec failure tracking .rspec_status + +/spec/internal/log/ diff --git a/Gemfile b/Gemfile index ffb8b22..36e1674 100644 --- a/Gemfile +++ b/Gemfile @@ -3,20 +3,23 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } - -# gem 'activemodel', '~> 7' -# gem 'activesupport', '~> 7' -# gem 'mongoid', '~> 8' - # Specify your gem's dependencies in consyncful.gemspec gemspec group :development do gem 'bundler', '~> 2' - gem 'database_cleaner-mongoid' gem 'rake', '~> 13.0' - gem 'rspec', '~> 3.0' gem 'rubocop' gem 'rubocop-rake' gem 'rubocop-rspec' end + +group :development, :test do + gem 'combustion', '~> 1.3' + gem 'rspec', '~> 3.13' + gem 'rspec-rails', '~> 6.1' +end + +group :test do + gem 'database_cleaner-mongoid', '~> 2.0' +end diff --git a/Gemfile.lock b/Gemfile.lock index 473c612..ce2118b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - consyncful (1.0.2) + consyncful (1.1.0) contentful (>= 2.11.1, < 3.0.0) hooks (>= 0.4.1) mongoid (>= 7.0.2) @@ -10,6 +10,23 @@ PATH GEM remote: https://rubygems.org/ specs: + actionpack (7.2.2.1) + actionview (= 7.2.2.1) + activesupport (= 7.2.2.1) + nokogiri (>= 1.8.5) + racc + rack (>= 2.2.4, < 3.2) + rack-session (>= 1.0.1) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + useragent (~> 0.16) + actionview (7.2.2.1) + activesupport (= 7.2.2.1) + builder (~> 3.1) + erubi (~> 1.11) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) activemodel (7.2.2.1) activesupport (= 7.2.2.1) activesupport (7.2.2.1) @@ -31,18 +48,27 @@ GEM benchmark (0.4.0) bigdecimal (3.1.9) bson (5.0.2) + builder (3.3.0) + combustion (1.5.0) + activesupport (>= 3.0.0) + railties (>= 3.0.0) + thor (>= 0.14.6) concurrent-ruby (1.3.5) connection_pool (2.5.3) contentful (2.17.1) http (> 0.8, < 6.0) multi_json (~> 1) + crass (1.0.6) database_cleaner-core (2.0.1) database_cleaner-mongoid (2.0.1) database_cleaner-core (~> 2.0.0) mongoid + date (3.4.1) diff-lcs (1.6.2) domain_name (0.6.20240107) drb (2.2.3) + erb (5.0.2) + erubi (1.13.1) ffi (1.17.2) ffi (1.17.2-aarch64-linux-gnu) ffi (1.17.2-aarch64-linux-musl) @@ -70,6 +96,11 @@ GEM http-form_data (2.3.0) i18n (1.14.7) concurrent-ruby (~> 1.0) + io-console (0.8.1) + irb (1.15.2) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) json (2.12.2) language_server-protocol (3.17.0.5) lint_roller (1.1.0) @@ -77,6 +108,10 @@ GEM ffi-compiler (~> 1.0) rake (~> 13.0) logger (1.7.0) + loofah (2.24.1) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + mini_portile2 (2.8.9) minitest (5.25.5) mongo (2.21.1) base64 @@ -87,16 +122,69 @@ GEM mongo (>= 2.18.0, < 3.0.0) ruby2_keywords (~> 0.0.5) multi_json (1.15.0) + nokogiri (1.18.9) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.18.9-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.9-aarch64-linux-musl) + racc (~> 1.4) + nokogiri (1.18.9-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.9-arm-linux-musl) + racc (~> 1.4) + nokogiri (1.18.9-arm64-darwin) + racc (~> 1.4) + nokogiri (1.18.9-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.18.9-x86_64-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.9-x86_64-linux-musl) + racc (~> 1.4) parallel (1.27.0) parser (3.3.8.0) ast (~> 2.4.1) racc + pp (0.6.2) + prettyprint + prettyprint (0.2.0) prism (1.4.0) + psych (5.2.6) + date + stringio public_suffix (6.0.2) racc (1.8.1) + rack (3.1.16) + rack-session (2.1.1) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.2.0) + rack (>= 1.3) + rackup (2.2.1) + rack (>= 3) + rails-dom-testing (2.3.0) + activesupport (>= 5.0.0) + minitest + nokogiri (>= 1.6) + rails-html-sanitizer (1.6.2) + loofah (~> 2.21) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (7.2.2.1) + actionpack (= 7.2.2.1) + activesupport (= 7.2.2.1) + irb (~> 1.13) + rackup (>= 1.0.0) + rake (>= 12.2) + thor (~> 1.0, >= 1.2.2) + zeitwerk (~> 2.6) rainbow (3.1.1) rake (13.2.1) + rdoc (6.14.2) + erb + psych (>= 4.0.0) regexp_parser (2.10.0) + reline (0.6.2) + io-console (~> 0.5) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -109,6 +197,14 @@ GEM rspec-mocks (3.13.4) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) + rspec-rails (6.1.5) + actionpack (>= 6.1) + activesupport (>= 6.1) + railties (>= 6.1) + rspec-core (~> 3.13) + rspec-expectations (~> 3.13) + rspec-mocks (~> 3.13) + rspec-support (~> 3.13) rspec-support (3.13.3) rubocop (1.75.7) json (~> 2.3) @@ -133,12 +229,16 @@ GEM ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) securerandom (0.4.1) + stringio (3.1.7) + thor (1.4.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) uber (0.0.15) unicode-display_width (3.1.4) unicode-emoji (~> 4.0, >= 4.0.4) unicode-emoji (4.0.4) + useragent (0.16.11) + zeitwerk (2.7.3) PLATFORMS aarch64-linux-gnu @@ -155,10 +255,12 @@ PLATFORMS DEPENDENCIES bundler (~> 2) + combustion (~> 1.3) consyncful! - database_cleaner-mongoid + database_cleaner-mongoid (~> 2.0) rake (~> 13.0) - rspec (~> 3.0) + rspec (~> 3.13) + rspec-rails (~> 6.1) rubocop rubocop-rake rubocop-rspec diff --git a/README.md b/README.md index 2b205b6..e3d4959 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Consyncful -Contentful -> local database synchronisation for Rails +[Contentful](https://www.contentful.com/) -> MongoDB synchronisation for Rails. Requesting complicated models from the Contentful Delivery API in Rails applications is often too slow, and makes testing applications painful. Consyncful uses Contentful's synchronisation API to keep a local, up-to-date copy of the entire content in a Mongo database. @@ -8,23 +8,45 @@ Once the content is available locally, finding and interact with contentful data This gem doesn't provide any integration with the management API, or any way to update Contentful models from the local store. It is strictly read only. -- [Installation](#installation) -- [Usage](#usage) - - [Creating contentful models in your Rails app](#creating-contentful-models-in-your-rails-app) - - [Synchronizing contentful data](#synchronizing-contentful-data) - - [Finding and interacting with models](#finding-and-interacting-with-models) - - [Querying](#querying) - - [References](#references) - - [Finding entries from different content types](#finding-entries-from-different-content-types) +```mermaid +flowchart TD + CF["Contentful (Delivery API / Sync API)"] + SY["rake consyncful:sync"] + DB["MongoDB (single collection)"] + APP["Rails models (Mongoid, subclass Consyncful::Base)"] + + CF -->|sync| SY + SY -->|writes| DB + APP -->|queries| DB +``` + +## Contents +- [Setup](#setup) + - [Installation](#installation) + - [Configuration options](#configuration-options) +- [Creating contentful models in your Rails app](#creating-contentful-models-in-your-rails-app) +- [Synchronizing contentful data](#synchronizing-contentful-data) + - [Continuous sync](#continuous-sync-either-mode) + - [Refresh from scratch](#refresh-from-scratch) + - [Enabling webhook mode](#enabling-webhook-mode) - [Sync callbacks](#sync-callbacks) + - [Sync specific contents using contentful tags](#sync-specific-contents-using-contentful-tag) +- [Finding and interacting with models](#finding-and-interacting-with-models) + - [Querying](#querying) + - [References](#references) + - [Finding entries from different content types](#finding-entries-from-different-content-types) - [Using Locales for specific fields](#using-locales-for-specific-fields) - - [Configuring what Mongo database Consyncful uses](#configuring-what-mongo-database-consyncful-uses) - - [Why do I have to use MongoDB?](#why-do-i-have-to-use-mongodb) + - [Preserving Contentful timestamps](#preserving-contentful-timestamps) +- [MongoDB Configuration](#mongodb-configuration) + - [Choosing the Mongo Database](#choosing-the-mongo-database) + - [Why MongoDB?](#why-mongodb) - [Development](#development) - [Contributing](#contributing) - [License](#license) -## Installation +## Setup + +### Installation Add this line to your application's Gemfile: @@ -40,16 +62,15 @@ If you don't already use Mongoid, generate a mongoid.yml by running: $ rake g mongoid:config -Add an initializer: +Create `config/initializers/consyncful.rb`. An example with common configuration is: -Consyncful uses [contentful.rb](https://github.com/contentful/contentful.rb); client options are as documented there. Sync options are documented in the [Content Delivery Sync API docs](https://www.contentful.com/developers/docs/references/content-delivery-api/#/reference/synchronization). ```rb Consyncful.configure do |config| config.locale = 'en-NZ' config.contentful_client_options = { api_url: 'cdn.contentful.com', space: 'space_id', - access_token: 'ACCESS TOKEN', + access_token: 'ACCESS_TOKEN', environment: 'master', # optional logger: Logger.new(STDOUT) # optional for debugging } @@ -61,9 +82,28 @@ Consyncful.configure do |config| end ``` -## Usage +> [!NOTE] +> Consyncful uses the official [contentful.rb](https://github.com/contentful/contentful.rb) client. Any `contentful_client_options` you set are passed through to this library unchanged. Similary, settings in `contentful_sync_options` map to the parameters in Contentful’s [Content Delivery Sync API](https://www.contentful.com/developers/docs/references/content-delivery-api/#/reference/synchronization). + + +### Configuration options + +| Option | Description | Default | +| --- | --- | --- | +| `sync_mode` | How syncing is triggered: `:poll` (periodic polling used by default) or `:webhook` (sync runs when a webhook is received from contentful). | `:poll` | +| `contentful_client_options` | Options passed through to the `contentful.rb` client. Defaults include `reuse_entries: true`, `api_url: 'cdn.contentful.com'`. | `{}` (merged with defaults) | +| `contentful_sync_options` | Contentful Sync API parameters (e.g., `limit`, `type`). Defaults include `limit: 100`, `type: 'all'`. | `{}` (merged with defaults) | +| `locale` | Default locale when mapping fields. | `'en-NZ'` | +| `content_tags` | Only store entries that have **any** of these tags. | `[]` | +| `ignore_content_tags` | Ignore entries with **any** of these tags. | `[]` | +| `preserve_contentful_timestamps` | Adds `contentful_created_at` and `contentful_updated_at` to models. | `false` | +| `mongo_client` | Mongoid client to use (from `mongoid.yml`). | `:default` | +| `mongo_collection` | MongoDB collection name for all entries. | `'contentful_models'` | +| `webhook_authentication_enabled` | Require Basic Auth for the webhook endpoint (only relevant when `sync_mode: :webhook`, enabled by default). | `true` | +| `webhook_user` | Username for webhook Basic Auth (when enabled). | `nil` | +| `webhook_password` | Password for webhook Basic Auth (when enabled). | `nil` | -### Creating contentful models in your Rails app +## Creating contentful models in your Rails application Create models by inheriting from `Consyncful::Base` @@ -95,25 +135,123 @@ class ModelWithReferences < Consyncful::Base end ``` -### Synchronizing contentful data +## Synchronizing contentful data -To run a synchronization process run: +`Consyncful` supports **two sync modes**: +- **Polling (default)** — checks Contentful on an interval and syncs changes. +- **Webhook** — Contentful calls your app; the worker syncs when a webhook arrives. + +### Continuous sync (either mode) +Run the same task in both modes — the behaviour depends on your configuration: $ rake consyncful:sync -The first time you run this it will download all the Contentful content. It will then check every 15 seconds for changes to the content and update/delete records in the database when changes are made in Contentful. +- **Polling mode**: after the initial full sync, the worker polls every **15s** (configurable) and applies changes it finds. +- **Webhook mode**: after the initial full sync, the worker **does not poll**. It waits for a webhook signal and then runs a sync. + + +> [!NOTE] +> The first time you run this it will download all the Contentful content. -If you want to synchronise from scratch, run: +### Refresh from scratch + +If you want to resynchronize everything (e.g., after model/content type renames), run: $ rake consyncful:refresh +This performs a full rebuild of data from contentful. + It is recommended to refresh your data if you change model names. Now you've synced your data, it is all available via your Rails models. -### Finding and interacting with models +### Enabling webhook mode + +> [!TIP] +> **Webhook mode is recommended on limited plans**. +> Polling makes API requests on every interval (default ~15s), which can quickly add up and exhaust quotas on lower-tier Contentful plans. +> **Webhook mode** only syncs when Contentful sends an event, dramatically reducing API calls. If you’re hitting rate limits—or want to avoid them—switch to `:webhook`. +> If you are hitting API rate limits and need to use polling, consider increasing the interval to reduce load. + + +#### 1. Set the sync mode to webhook + +``` +# e.g. config/initializers/consyncful.rb +Consyncful.configure do |c| + c.sync_mode = :webhook +end +``` + +#### 2. Mount the webhooks controller: +Expose the engine so Contentful can POST to it +``` +# config/routes.rb +mount Consyncful::Engine, at: "/consyncful" +``` +The webhook endpoint lives under this mount (e.g. `/consyncful/trigger_sync`). + +#### 3. Authentication (recommended) +Webhook authentication is **on by default**: +``` +Consyncful.configure do |c| + c.webhook_authentication_required = true # default + c.webhook_user = ENV["CONSYNCFUL_WEBHOOK_USER"] + c.webhook_password = ENV["CONSYNCFUL_WEBHOOK_PASSWORD"] +end +``` +To accept webhooks **without** auth (not recommended), explicitly disable it: +``` +c.webhook_authentication_required = false +``` + +#### 4. Create the webhook in Contentful +In your Contentful space/environment, add a webhook that points to your mounted route (e.g. `https://your-app.example.com/consyncful/trigger_sync`) and select which events should trigger a sync (publish/unpublish, entries, assets, etc.). See Contentful documents here for information on setting up a webhook: [Configuring a webhook](https://www.contentful.com/developers/docs/webhooks/configure-webhook/) + +> [!IMPORTANT] +> If your application is behind global authentication, VPN, or an allowlist, Contentful won’t be able to reach the webhook endpoint. Ensure that `POST` requests from Contentful can reach your mounted path (e.g. `/consyncful/...`). In many setups this means adding an ingress rule or route exemption for the webhook path. Keeping webhook authentication **enabled** (default) is recommended; configure matching credentials in the Contentful webhook. + +### Sync callbacks + +You may want to attach some application logic to happen before or after a sync run, for example to update caches. + +Callbacks can be registered using: + +```ruby +Consyncful::Sync.before_run do + # do something before the run +end +``` + +```ruby +Consyncful::Sync.after_run do |updated_ids| + # invalidate cache for updated_ids, or something +end +``` + +### Sync specific contents using [Contentful Tag](https://www.contentful.com/help/tags/) +You can configure Consyncful to sync or ignore specific contents using Contentful Tag. + +```rb +Consyncful.configure do |config| + # Any contents tagged with 'myTag' will be stored in the database. + # Other contents without 'myTag' would be ignored. + config.content_tags = ['myTag'] # defaults to [] +end +``` + +Also, you can ignore contents with specific Tags. + +```rb +Consyncful.configure do |config| + # Any contents tagged with 'ignoreTag' won't be stored in the database. + config.ignore_content_tags = ['ignoreTag'] # defaults to [] +end +``` -#### Querying +## Finding and interacting with models + +### Querying Models are available using standard Mongoid [queries](https://docs.mongodb.com/mongoid/current/tutorials/mongoid-queries/). ```ruby @@ -122,7 +260,7 @@ instance = ModelName.find_by(instance: 'foo') instance.is_awesome # true ``` -#### References +### References References work like you would expect: ```ruby @@ -140,7 +278,7 @@ instance.other_things # all the referenced things, polymorphic, so might be diff instance.other_things.in_order # ordered the same as in Contentful ``` -#### Finding entries from different content types +### Finding entries from different content types Because all Contentful models are stored as polymorphic subtypes of `Consyncful::Base`, you can query all entries without knowing what type you are looking for: @@ -148,24 +286,6 @@ Because all Contentful models are stored as polymorphic subtypes of `Consyncful: Consyncful::Base.where(title: 'a title') # [ #, # ] ``` -### Sync callbacks - -You may want to attach some application logic to happen before or after a sync run, for example to update caches. - -Callbacks can be registered using: - -```ruby -Consyncful::Sync.before_run do - # do something before the run -end -``` - -```ruby -Consyncful::Sync.after_run do |updated_ids| - # invalidate cache for updated_ids, or something -end -``` - ### Using Locales for specific fields If fields have multiple locales then the default locale will be mapped to the field name. Additional locales will have a suffix (lower snake case) on the field name. e.g title (default), title_mi_nz (New Zealand Maori mi-NZ) @@ -183,29 +303,11 @@ Consyncful.configure do |config| end ``` -### Sync specific contents using [Contentful Tag](https://www.contentful.com/help/tags/). -You can configure Consyncful to sync or ignore specific contents using Contentful Tag. - -```rb -Consyncful.configure do |config| - # Any contents tagged with 'myTag' will be stored in the database. - # Other contents without 'myTag' would be ignored. - config.content_tags = ['myTag'] # defaults to [] -end -``` - -Also, you can ignore contents with specific Tags. - -```rb -Consyncful.configure do |config| - # Any contents tagged with 'ignoreTag' won't be stored in the database. - config.ignore_content_tags = ['ignoreTag'] # defaults to [] -end -``` +## MongoDB Configuration -### Configuring what Mongo database Consyncful uses +### Choosing the Mongo Database -You can also configure what Mongoid client Consyncful uses and the name of the collection the entries are stored under. This is useful if you want to have your consyncful data hosted in a different mongo database than your application-specific mongo database. +You can configure which Mongoid client Consyncful uses, as well as the name of the collection where entries are stored. This is useful if you want Consyncful data to live in a separate MongoDB database from your application-specific database. ```rb Consyncful.configure do |config| @@ -214,9 +316,9 @@ Consyncful.configure do |config| end ``` -### Why do I have to use MongoDB? +### Why MongoDB? -Consyncful currently only supports Mongoid ODM because models have dynamic schemas. And that's all we've had a chance to work out so far. The same pattern might be able to be extended to work with ActiveRecord, but having to migrate the local database as well as your contentful content type's seems tedious. +Consyncful currently only supports the Mongoid ODM because models require dynamic schemas. Extending support to ActiveRecord could be possible in the future, but it would also require maintaining database migrations alongside Contentful content type changes—which adds complexity we wanted to avoid. ## Development diff --git a/app/controllers/consyncful/webhook_controller.rb b/app/controllers/consyncful/webhook_controller.rb new file mode 100644 index 0000000..a38ad26 --- /dev/null +++ b/app/controllers/consyncful/webhook_controller.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Consyncful + # The Consyncful::WebhookController is responsible for handling incoming + # webhook requests that can trigger synchronization jobs within Consyncful. + # + # Features: + # - Only responds to requests if `sync_mode` is configured as `:webhook`. + # - Optionally requires HTTP Basic authentication if + # `webhook_authentication_required` is enabled in configuration. + # - Exposes a single endpoint (`trigger_sync`) that signals a sync process + # through `Consyncful::Sync.signal_webhook!`. + # + # Security: + # - Uses `ActionController::HttpAuthentication::Basic` to enforce + # authentication when enabled. + # - Compares provided credentials with configured values using + # `ActiveSupport::SecurityUtils.secure_compare` to prevent timing attacks. + # + # Responses: + # - Returns `404 Not Found` if webhooks are not enabled. + # - Returns `202 Accepted` after signaling a sync. + class WebhookController < ActionController::API + include ActionController::HttpAuthentication::Basic::ControllerMethods + before_action :authenticate, if: -> { Consyncful.configuration.webhook_authentication_required && use_webhooks? } + + def trigger_sync + return head :not_found unless use_webhooks? + + Consyncful::Sync.signal_webhook! + head :accepted + end + + private + + def use_webhooks? + Consyncful.configuration.sync_mode == :webhook + end + + def authenticate + config = Consyncful.configuration + authenticate_or_request_with_http_basic('Consyncful: Authenticate to Trigger Sync') do |username, password| + secure_compare(username, config.webhook_user) && secure_compare(password, config.webhook_password) + end + end + + def secure_compare(value, expected) + ActiveSupport::SecurityUtils.secure_compare(value.to_s, expected.to_s) + end + end +end diff --git a/config/routes.rb b/config/routes.rb new file mode 100644 index 0000000..d54710b --- /dev/null +++ b/config/routes.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +Consyncful::Engine.routes.draw do + post 'webhook', to: 'webhook#trigger_sync' +end diff --git a/lib/consyncful.rb b/lib/consyncful.rb index 53c6793..81fd581 100644 --- a/lib/consyncful.rb +++ b/lib/consyncful.rb @@ -1,9 +1,17 @@ # frozen_string_literal: true -require 'consyncful/version' +# External dependencies require 'mongoid' require 'contentful' + +# Internal library files +require 'consyncful/version' require 'consyncful/configuration' require 'consyncful/base' require 'consyncful/sync' -require 'consyncful/railtie' if defined?(Rails) + +# Rails integration (only load if Rails is present) +if defined?(Rails) + require 'consyncful/railtie' + require 'consyncful/engine' +end diff --git a/lib/consyncful/configuration.rb b/lib/consyncful/configuration.rb index eabc655..bd365c6 100644 --- a/lib/consyncful/configuration.rb +++ b/lib/consyncful/configuration.rb @@ -2,7 +2,18 @@ # Handles Rails configurations for Consyncful module Consyncful - ## + # Provides configuration options for Consyncful, including: + # - Contentful API client and sync options + # - MongoDB client and collection settings + # - Locale and content tag filtering + # - Sync mode (poll or webhook) + # - Webhook authentication credentials + # + # This class is typically accessed and customized via + # Consyncful.configure do |config| + # config.locale = 'en-NZ' + # config.mongo_collection = 'my_models' + # end class Configuration attr_accessor :contentful_client_options, :contentful_sync_options, @@ -11,9 +22,15 @@ class Configuration :mongo_collection, :content_tags, :ignore_content_tags, - :preserve_contentful_timestamps + :preserve_contentful_timestamps, + :sync_mode, + :webhook_authentication_required, + :webhook_user, + :webhook_password + # rubocop:disable Metrics/MethodLength def initialize + @sync_mode = :poll @contentful_client_options = {} @contentful_sync_options = {} @locale = 'en-NZ' @@ -22,7 +39,12 @@ def initialize @content_tags = [] @ignore_content_tags = [] @preserve_contentful_timestamps = false + + @webhook_authentication_required = true + @webhook_user = nil + @webhook_password = nil end + # rubocop:enable Metrics/MethodLength def initial_sync_options options = { initial: true } diff --git a/lib/consyncful/engine.rb b/lib/consyncful/engine.rb new file mode 100644 index 0000000..e022dc6 --- /dev/null +++ b/lib/consyncful/engine.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Consyncful + # Rails engine for Consyncful. + # + # This isolates the Consyncful namespace and allows the gem + # to provide its own routes, controllers, and configuration + # within a Rails application without clashing with the host app. + class Engine < ::Rails::Engine + isolate_namespace Consyncful + end +end diff --git a/lib/consyncful/item_mapper.rb b/lib/consyncful/item_mapper.rb index b4f3491..c9d09f1 100644 --- a/lib/consyncful/item_mapper.rb +++ b/lib/consyncful/item_mapper.rb @@ -14,8 +14,12 @@ def deletion? end def excluded_by_tag? - return (Consyncful.configuration.content_tags & item_tag_ids).empty? if Consyncful.configuration.content_tags.any? - return (Consyncful.configuration.ignore_content_tags & item_tag_ids).any? if Consyncful.configuration.ignore_content_tags.any? + config = Consyncful.configuration + content_tags = config.content_tags + ignore_content_tags = config.ignore_content_tags + + return (content_tags & item_tag_ids).empty? if content_tags.any? + return (ignore_content_tags & item_tag_ids).any? if ignore_content_tags.any? false end @@ -65,7 +69,7 @@ def localized_fields(default_locale) @item.fields_with_locales.each do |field, value_with_locales| value_with_locales.each do |locale_code, value| - next if value.is_a? Contentful::File # assets are handeled below + next if value.is_a? Contentful::File # assets are handled below field_name = localized_field_name(field, locale_code, default_locale) field_name, value = mapped_field_entry_for(field_name, value) diff --git a/lib/consyncful/sync.rb b/lib/consyncful/sync.rb index fc4cfea..7934039 100644 --- a/lib/consyncful/sync.rb +++ b/lib/consyncful/sync.rb @@ -27,10 +27,25 @@ class Sync field :next_url field :last_run_at, type: DateTime + field :webhook_pending, type: Boolean, default: false + def self.latest last || new end + ## + # Signal that a webhook has been received and a sync should be triggered + def self.signal_webhook! + latest.set(webhook_pending: true) + true + end + + ## + # Consume the webhook signal and set webhook_pending to false + def self.consume_webhook_signal! + latest.set(webhook_pending: false) + end + ## # Delete the previous sync chains from database and create a fresh one. # Used to completely resync all items from Contentful. diff --git a/lib/consyncful/sync_runner.rb b/lib/consyncful/sync_runner.rb new file mode 100644 index 0000000..7c38453 --- /dev/null +++ b/lib/consyncful/sync_runner.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Consyncful + # The SyncRunner is responsible for continuously executing Contentful sync + # jobs at a configurable interval or in response to webhook signals. + # + # Modes: + # - :poll — runs the sync every N seconds (default 15) + # - :webhook — waits for webhook signals and triggers a sync when received + # + # Behavior: + # - Starts with an initial sync (`Consyncful::Sync.latest.run`). + # - In poll mode, sleeps for the configured interval and then re-runs sync. + # - In webhook mode, listens for webhook signals and runs sync immediately. + class SyncRunner + DEFAULT_INTERVAL = 15 + VALID_MODES = %i[poll webhook].freeze + + def initialize(seconds: nil, mode: nil) + @interval = seconds || DEFAULT_INTERVAL + @mode = validate_mode(mode) + end + + def run + current_sync = Consyncful::Sync.latest + current_sync.run # Run initial sync + + loop do + sleep(@interval) + current_sync.run if @mode == :poll || Consyncful::Sync.consume_webhook_signal! + end + end + + private + + def validate_mode(value) + sym = value.to_sym + return sym if VALID_MODES.include?(sym) + + raise ArgumentError, "Unknown sync mode: #{sym.inspect} (expected :poll or :webhook)" + end + end +end diff --git a/lib/consyncful/tasks/consyncful.rake b/lib/consyncful/tasks/consyncful.rake index 31fb6ff..72f714c 100644 --- a/lib/consyncful/tasks/consyncful.rake +++ b/lib/consyncful/tasks/consyncful.rake @@ -1,28 +1,32 @@ # frozen_string_literal: true namespace :consyncful do + desc 'Run a one-time sync of the latest Contentful data into the app' task update: [:environment] do Consyncful::Sync.latest.run end + desc 'Run a one-time full refresh of all Contentful data into the app (bypasses caching)' task refresh: [:environment] do Consyncful::Sync.fresh.run end + desc 'Continuously sync Contentful data. Default: poll every N seconds (default: 15)' task :sync, [:seconds] => %i[environment update_model_names] do |_task, args| + require 'consyncful/sync_runner' Signal.trap('TERM') do puts Rainbow("Graceful shutdown PID=#{Process.pid}").red exit 0 end - seconds = args[:seconds].to_i - seconds = 15 if seconds.zero? - loop do - Consyncful::Sync.latest.run - sleep(seconds) - end + seconds = args[:seconds] + mode = Consyncful.configuration&.sync_mode || :poll + puts "mode=#{mode.inspect} interval=#{seconds.inspect}s" + + Consyncful::SyncRunner.new(seconds: seconds, mode: mode).run end + desc 'Update stored model_type fields based on Contentful type mappings' task update_model_names: [:environment] do if Rails.autoloaders.zeitwerk_enabled? Zeitwerk::Loader.eager_load_all diff --git a/lib/consyncful/version.rb b/lib/consyncful/version.rb index f91421b..dd49bd2 100644 --- a/lib/consyncful/version.rb +++ b/lib/consyncful/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Consyncful - VERSION = '1.0.2' + VERSION = '1.1.0' end diff --git a/spec/consyncful/configuration_spec.rb b/spec/consyncful/configuration_spec.rb index b34d17a..878e460 100644 --- a/spec/consyncful/configuration_spec.rb +++ b/spec/consyncful/configuration_spec.rb @@ -5,6 +5,10 @@ RSpec.describe Consyncful::Configuration do let(:configuration) { Consyncful::Configuration.new } + it 'defaults to poll sync_mode' do + expect(configuration.sync_mode).to eq(:poll) + end + describe '.initial_sync_options' do it 'always contains initial:true' do result = configuration.initial_sync_options diff --git a/spec/controllers/consyncful/webhook_controller_spec.rb b/spec/controllers/consyncful/webhook_controller_spec.rb new file mode 100644 index 0000000..3755918 --- /dev/null +++ b/spec/controllers/consyncful/webhook_controller_spec.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe 'Consyncful::WebhookController', type: :request do + let(:config) do + # Provide a simple configuration double with the fields used by the controller + instance_double( + 'Consyncful::Configuration', + sync_mode: sync_mode, + webhook_authentication_required: auth_required, + webhook_user: 'user1', + webhook_password: 'secret' + ) + end + + before do + allow(Consyncful).to receive(:configuration).and_return(config) + end + + describe 'POST /consyncful/webhook' do + subject(:perform) { post '/consyncful/webhook', headers: headers } + + let(:headers) { {} } + + context 'when sync mode is not :webhook' do + let(:sync_mode) { :poll } + let(:auth_required) { false } + + it 'returns 404 Not Found' do + perform + expect(response).to have_http_status(:not_found) + end + end + + context 'when sync mode is :webhook' do + let(:sync_mode) { :webhook } + + context 'and authentication is not required' do + let(:auth_required) { false } + + it 'signals the sync and returns 202 Accepted' do + expect(Consyncful::Sync).to receive(:signal_webhook!) + perform + expect(response).to have_http_status(:accepted) + end + end + + context 'and authentication is required' do + let(:auth_required) { true } + + context 'with no credentials' do + it 'returns 401 Unauthorized' do + perform + expect(response).to have_http_status(:unauthorized) + end + end + + context 'with wrong credentials' do + let(:headers) do + { + 'HTTP_AUTHORIZATION' => + ActionController::HttpAuthentication::Basic.encode_credentials('user1', 'wrong') + } + end + + it 'returns 401 Unauthorized' do + perform + expect(response).to have_http_status(:unauthorized) + end + end + + context 'with correct credentials' do + let(:headers) do + { + 'HTTP_AUTHORIZATION' => + ActionController::HttpAuthentication::Basic.encode_credentials('user1', 'secret') + } + end + + it 'signals the sync and returns 202 Accepted' do + expect(Consyncful::Sync).to receive(:signal_webhook!) + perform + expect(response).to have_http_status(:accepted) + end + end + end + end + end +end diff --git a/spec/internal/config/routes.rb b/spec/internal/config/routes.rb new file mode 100644 index 0000000..33315b0 --- /dev/null +++ b/spec/internal/config/routes.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +Rails.application.routes.draw do + mount Consyncful::Engine => '/consyncful' +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb new file mode 100644 index 0000000..7c1e4e9 --- /dev/null +++ b/spec/rails_helper.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +ENV['RAILS_ENV'] ||= 'test' + +require 'bundler/setup' +require 'combustion' +require 'consyncful' +require 'consyncful/engine' + +Combustion.initialize! :action_controller, :active_support + +require 'rspec/rails' + +RSpec.configure do |config| + config.use_active_record = false + config.infer_spec_type_from_file_location! + config.filter_rails_from_backtrace! +end