(#917) Upload envelope graphs to S3 during publishing

Author: excelsior

app/api/v1/publish.rb

@@ -1,5 +1,6 @@
 require 'policies/envelope_policy'
 require 'services/publish_interactor'
+require 'services/sync_envelope_graph_with_s3'
 
 module API
   module V1
@@ -34,7 +35,7 @@ module Publish
               optional :published_by, type: String
               use :update_if_exists
             end
-            post do
+            post do # rubocop:todo Metrics/BlockLength
               authorize Envelope, :create?
 
               secondary_token_header = request.headers['Secondary-Token']
@@ -60,6 +61,7 @@ module Publish
               )
 
               if interactor.success?
+                SyncEnvelopeGraphWithS3.add(interactor.envelope)
                 present interactor.envelope, with: API::Entities::Envelope
                 update_if_exists? ? status(:ok) : status(:created)
               else
@@ -92,6 +94,7 @@ module Publish
             delete do
               authorize Envelope, :destroy?
               @envelope.destroy
+              SyncEnvelopeGraphWithS3.remove(@envelope)
               body ''
             end
 

app/jobs/publish_envelope_job.rb

@@ -8,6 +8,7 @@ def perform(publish_request_id)
       publish_result = PublishInteractor.call(**publish_args)
 
       if publish_result.success?
+        SyncEnvelopeGraphWithS3.add(publish_result.envelope)
         publish_request.complete(publish_result.envelope.id)
       else
         publish_request.fail(publish_result.error)

app/services/sync_envelope_graph_with_s3.rb

@@ -0,0 +1,53 @@
+# Uploads or deletes an envelope graph from the S3 bucket
+class SyncEnvelopeGraphWithS3
+  attr_reader :envelope
+
+  delegate :envelope_community, :envelope_ceterms_ctid, to: :envelope
+
+  def initialize(envelope)
+    @envelope = envelope
+  end
+
+  class << self
+    def add(envelope)
+      new(envelope).add
+    end
+
+    def remove(envelope)
+      new(envelope).remove
+    end
+  end
+
+  def add
+    return unless s3_bucket_name
+
+    s3_object.put(body: envelope.processed_resource.to_json, content_type: 'application/json')
+    s3_object.public_url
+  end
+
+  def remove
+    return unless s3_bucket_name
+
+    s3_object.delete
+  end
+
+  def s3_bucket
+    @s3_bucket ||= s3_resource.bucket(s3_bucket_name)
+  end
+
+  def s3_bucket_name
+    ENV['ENVELOPE_GRAPHS_BUCKET'].presence
+  end
+
+  def s3_key
+    "#{envelope_community.name}/#{envelope_ceterms_ctid}.json"
+  end
+
+  def s3_object
+    s3_bucket.object(s3_key)
+  end
+
+  def s3_resource
+    @s3_resource ||= Aws::S3::Resource.new(region: ENV['AWS_REGION'].presence)
+  end
+end

spec/api/v1/publish_spec.rb

@@ -98,6 +98,7 @@
     end
 
     context 'default community' do # rubocop:todo RSpec/ContextWording, RSpec/MultipleMemoizedHelpers
+      let(:community) { ce_registry }
       let(:user2) { create(:user) }
 
       # rubocop:todo RSpec/NestedGroups
@@ -131,8 +132,18 @@
         # rubocop:todo RSpec/NestedGroups
         context 'new envelope' do # rubocop:todo RSpec/ContextWording, RSpec/NestedGroups
           # rubocop:enable RSpec/NestedGroups
-          # rubocop:todo RSpec/ExampleLength
+          # rubocop:todo RSpec/ExampleLength, RSpec/MultipleExpectations
           it 'returns the newly created envelope with a 201 Created HTTP status code' do
+            expect(SyncEnvelopeGraphWithS3).to receive(:add) do |envelope| # rubocop:todo RSpec/MessageSpies
+              expect(envelope.envelope_community).to eq(community)
+              expect(envelope.process_resource).to eq(JSON(resource_json))
+            end.exactly(:twice)
+
+            expect(SyncEnvelopeGraphWithS3).to receive(:add) do |envelope| # rubocop:todo RSpec/MessageSpies
+              expect(envelope.envelope_community).to eq(community)
+              expect(envelope.process_resource).to eq(JSON(updated_resource_json))
+            end
+
             # New envelope
             travel_to now do
               post "/resources/organizations/#{organization._ctid}/documents",
@@ -199,7 +210,12 @@
             envelope
           end
 
-          it 'updates the envelope' do # rubocop:todo RSpec/ExampleLength
+          it 'updates the envelope' do # rubocop:todo RSpec/ExampleLength, RSpec/MultipleExpectations
+            allow(SyncEnvelopeGraphWithS3).to receive(:add) do |envelope|
+              expect(envelope.envelope_community).to eq(community)
+              expect(envelope.process_resource).to eq(JSON(updated_resource_json))
+            end
+
             travel_to now do
               expect do
                 post "/resources/organizations/#{organization._ctid}/documents",
@@ -233,6 +249,11 @@
         before do
           create(:organization_publisher, organization: organization, publisher: user.publisher)
 
+          allow(SyncEnvelopeGraphWithS3).to receive(:add) do |envelope|
+            expect(envelope.envelope_community).to eq(community) # rubocop:todo RSpec/ExpectInHook
+            expect(envelope.process_resource).to eq(JSON(resource_json)) # rubocop:todo RSpec/ExpectInHook
+          end
+
           travel_to now do
             post "/resources/organizations/#{organization._ctid}/documents",
                  resource_json,
@@ -282,6 +303,11 @@
           super_publisher_user = create(:user, publisher: super_publisher)
           token = "Token #{super_publisher_user.auth_tokens.first.value}"
 
+          allow(SyncEnvelopeGraphWithS3).to receive(:add) do |envelope|
+            expect(envelope.envelope_community).to eq(community) # rubocop:todo RSpec/ExpectInHook
+            expect(envelope.process_resource).to eq(JSON(resource_json)) # rubocop:todo RSpec/ExpectInHook
+          end
+
           travel_to now do
             post "/resources/organizations/#{organization._ctid}/documents",
                  resource_json, 'Authorization' => token
@@ -366,6 +392,11 @@
             publisher: user.publisher
           )
 
+          allow(SyncEnvelopeGraphWithS3).to receive(:add) do |envelope|
+            expect(envelope.envelope_community).to eq(community) # rubocop:todo RSpec/ExpectInHook
+            expect(envelope.process_resource).to eq(JSON(resource_json)) # rubocop:todo RSpec/ExpectInHook
+          end
+
           travel_to now do
             post "/resources/organizations/#{organization._ctid}/documents?" \
                  "published_by=#{publishing_organization._ctid}",
@@ -436,6 +467,11 @@
         end
 
         it 'creates an envelope with provisional publication status' do
+          allow(SyncEnvelopeGraphWithS3).to receive(:add) do |envelope|
+            expect(envelope.envelope_community).to eq(community)
+            expect(envelope.process_resource).to eq(JSON(resource_json))
+          end
+
           expect do
             post "/resources/organizations/#{organization._ctid}/documents",
                  resource_json, 'Authorization' => "Token #{user.auth_token.value}"
@@ -518,6 +554,8 @@
         # rubocop:enable RSpec/NestedGroups
         # rubocop:todo RSpec/MultipleExpectations
         it 'deletes the envelope' do # rubocop:todo RSpec/ExampleLength, RSpec/MultipleExpectations
+          allow(SyncEnvelopeGraphWithS3).to receive(:remove).with(envelope)
+
           # rubocop:enable RSpec/MultipleExpectations
           expect do
             delete_envelope
@@ -591,6 +629,8 @@
         # rubocop:enable RSpec/NestedGroups
         # rubocop:todo RSpec/MultipleExpectations
         it 'deletes the envelope' do # rubocop:todo RSpec/ExampleLength, RSpec/MultipleExpectations
+          allow(SyncEnvelopeGraphWithS3).to receive(:remove).with(envelope)
+
           # rubocop:enable RSpec/MultipleExpectations
           expect do
             delete_envelope

spec/jobs/publish_envelope_job_spec.rb

@@ -27,6 +27,13 @@
 
       # rubocop:todo RSpec/MultipleExpectations
       it 'creates an envelope and marks the request as completed successfully' do
+        expect(SyncEnvelopeGraphWithS3).to receive(:add) do |envelope| # rubocop:todo RSpec/MessageSpies
+          expect(envelope.envelope_community).to eq(envelope_community)
+          # rubocop:todo Layout/LineLength
+          expect(envelope.processed_resource).to eq(JSON(JSON(publish_request.request_params).fetch('raw_resource')))
+          # rubocop:enable Layout/LineLength
+        end
+
         # rubocop:enable RSpec/MultipleExpectations
         subject.new.perform(publish_request.id) # rubocop:todo RSpec/NamedSubject
         publish_request.reload

spec/services/sync_envelope_graph_with_s3_spec.rb

@@ -0,0 +1,65 @@
+RSpec.describe SyncEnvelopeGraphWithS3 do # rubocop:todo RSpec/MultipleMemoizedHelpers
+  let(:envelope) { create(:envelope, :from_cer) }
+  let(:s3_bucket) { double('s3_bucket') } # rubocop:todo RSpec/VerifiedDoubles
+  let(:s3_bucket_name) { Faker::Lorem.word }
+  let(:s3_object) { double('s3_object') } # rubocop:todo RSpec/VerifiedDoubles
+  let(:s3_region) { 'aws-s3_region-test' }
+  let(:s3_resource) { double('s3_resource') } # rubocop:todo RSpec/VerifiedDoubles
+  let(:s3_url) { Faker::Internet.url }
+
+  context 'without bucket' do # rubocop:todo RSpec/MultipleMemoizedHelpers
+    describe '.add' do # rubocop:todo RSpec/MultipleMemoizedHelpers
+      it 'does nothing' do
+        expect { described_class.add(envelope) }.not_to raise_error
+      end
+    end
+
+    describe '.remove' do # rubocop:todo RSpec/MultipleMemoizedHelpers
+      it 'does nothing' do
+        expect { described_class.remove(envelope) }.not_to raise_error
+      end
+    end
+  end
+
+  context 'with bucket' do # rubocop:todo RSpec/MultipleMemoizedHelpers
+    before do
+      ENV['AWS_REGION'] = s3_region
+      ENV['ENVELOPE_GRAPHS_BUCKET'] = s3_bucket_name
+
+      allow(Aws::S3::Resource).to receive(:new)
+        .with(region: s3_region)
+        .and_return(s3_resource)
+
+      allow(s3_resource).to receive(:bucket).with(s3_bucket_name).and_return(s3_bucket)
+
+      allow(s3_bucket).to receive(:object)
+        .with("ce_registry/#{envelope.envelope_ceterms_ctid}.json")
+        .and_return(s3_object)
+    end
+
+    describe '.add' do # rubocop:todo RSpec/MultipleMemoizedHelpers
+      before do
+        allow(s3_object).to receive(:put).with(
+          body: envelope.processed_resource.to_json,
+          content_type: 'application/json'
+        )
+
+        allow(s3_object).to receive(:public_url).and_return(s3_url)
+      end
+
+      it 'uploads the s3_resource to S3' do
+        expect(described_class.add(envelope)).to eq(s3_url)
+      end
+    end
+
+    describe '.remove' do # rubocop:todo RSpec/MultipleMemoizedHelpers
+      before do
+        allow(s3_object).to receive(:delete)
+      end
+
+      it 'uploads the s3_resource to S3' do
+        expect { described_class.remove(envelope) }.not_to raise_error
+      end
+    end
+  end
+end