From 402b811ddb30cb11ec6618bac56ba146bfc826cb Mon Sep 17 00:00:00 2001 From: Peter Kingswell Date: Tue, 9 Mar 2021 10:01:00 -0500 Subject: [PATCH 1/3] adds links to components.md --- .../ffwd/output/OutputPluginDecorator.java | 138 ++++++++++++++++++ .../ffwd/output/OutputPluginInterface.java | 2 + docs/content/_docs/components.md | 31 ++-- 3 files changed, 154 insertions(+), 17 deletions(-) create mode 100644 api/src/main/java/com/spotify/ffwd/output/OutputPluginDecorator.java create mode 100644 api/src/main/java/com/spotify/ffwd/output/OutputPluginInterface.java diff --git a/api/src/main/java/com/spotify/ffwd/output/OutputPluginDecorator.java b/api/src/main/java/com/spotify/ffwd/output/OutputPluginDecorator.java new file mode 100644 index 00000000..4f9a3f41 --- /dev/null +++ b/api/src/main/java/com/spotify/ffwd/output/OutputPluginDecorator.java @@ -0,0 +1,138 @@ +/*- + * -\-\- + * FastForward API + * -- + * Copyright (C) 2016 - 2018 Spotify AB + * -- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * -/-/- + */ + +package com.spotify.ffwd.output; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; +import com.google.inject.Key; +import com.google.inject.Module; +import com.google.inject.PrivateModule; +import com.google.inject.Provides; +import com.google.inject.Singleton; +import com.google.inject.name.Named; +import com.google.inject.name.Names; +import com.spotify.ffwd.filter.Filter; +import com.spotify.ffwd.module.Batching; +import com.spotify.ffwd.statistics.BatchingStatistics; +import com.spotify.ffwd.statistics.CoreStatistics; +import com.spotify.ffwd.statistics.NoopCoreStatistics; +import java.util.Optional; +import java.util.logging.Logger; + +@JsonTypeInfo(use = Id.NAME, property = "type") +public abstract class OutputPlugin { + + protected final Batching batching; + protected final Optional filter; + + public OutputPlugin( + final Optional filter, final Batching batching + ) { + this.filter = filter; + this.batching = batching; + } + + /** + * This method allows to wrap plugin sink implementation type depending on configuration. If no + * additional configuration is specified then subtype of com.spotify.ffwd.output + * .BatchedPluginSink will be bound per plugin. + *

+ * output := new SubtypeOfBatchedPluginSink() + *

+ * If 'flushInterval' key is specified, then corresponding subtype of + * com.spotify.ffwd.output.BatchedPluginSink will be wrapped into com.spotify.ffwd.output + * .FlushingPluginSink: + *

+ * output := new FlushingPluginSink(flushInterval, delegator:=output) + *

+ * The resulting plugin sink type may be further wrapped into com.spotify.ffwd.output + * .FilteringPluginSink type if 'filter' key is specified in plugin configuration: + *

+ * output := new FilteringPluginSink(filter, delegator:=output) + * + * @param input binding key with injection type of plugin sink + * @param wrapKey binding key, containing injection type of wrapping plugin sink + * + * @return module that exposes output binding key + */ + protected Module wrapPluginSink( + final Key input, final Key wrapKey + ) { + return new PrivateModule() { + @Override + protected void configure() { + @SuppressWarnings("unchecked") + Key sinkKey = (Key) input; + + if (batching.getFlushInterval() != null && + BatchablePluginSink.class.isAssignableFrom( + input.getTypeLiteral().getRawType())) { + final Key flushingKey = + Key.get(PluginSink.class, Names.named("flushing")); + // IDEA doesn't like this cast, but it's correct, tho admittedly not pretty + @SuppressWarnings({ "RedundantCast", "unchecked" }) + final Key batchedPluginSink = + (Key) (Key) sinkKey; + + // Use annotation so that we can avoid name space clash + bind(BatchablePluginSink.class) + .annotatedWith(BatchingDelegate.class) + .to(batchedPluginSink); + bind(flushingKey).toInstance( + new BatchingPluginSink(batching.getFlushInterval(), + batching.getBatchSizeLimit(), batching.getMaxPendingFlushes())); + + sinkKey = flushingKey; + } + + if (filter.isPresent()) { + final Key filteringKey = + Key.get(PluginSink.class, Names.named("filtered")); + + // Use annotation so that we can avoid name space clash + bind(PluginSink.class).annotatedWith(FilteringDelegate.class).to(sinkKey); + bind(filteringKey).toInstance(new FilteringPluginSink(filter.get())); + + sinkKey = filteringKey; + } + + bind(wrapKey).to(sinkKey); + expose(wrapKey); + } + + @Provides + @Singleton + public BatchingStatistics batchingStatisticsProvider( + CoreStatistics statistics, @Named("pluginId") String pluginId, Logger log + ) { + if (batching.isReportStatistics()) { + log.info("Enabling reporting of batching-specific metrics for output " + + input.getTypeLiteral().getRawType().getName()); + return statistics.newBatching(pluginId); + } else { + return NoopCoreStatistics.noopBatchingStatistics; + } + } + }; + } + + public abstract Module module(Key key, String id); +} diff --git a/api/src/main/java/com/spotify/ffwd/output/OutputPluginInterface.java b/api/src/main/java/com/spotify/ffwd/output/OutputPluginInterface.java new file mode 100644 index 00000000..dd854bc6 --- /dev/null +++ b/api/src/main/java/com/spotify/ffwd/output/OutputPluginInterface.java @@ -0,0 +1,2 @@ +package com.spotify.ffwd.output;public interface OutputPluginInterface { +} diff --git a/docs/content/_docs/components.md b/docs/content/_docs/components.md index db844cdc..96409979 100644 --- a/docs/content/_docs/components.md +++ b/docs/content/_docs/components.md @@ -6,42 +6,39 @@ title: Components This document covers some of the major components in FastForward. -#### Module +## Module -A module is a loadable component that extends the functionality of FastForward. -When a module is loaded it typically registers a set of plugins. +A [module](https://github.com/spotify/ffwd/blob/master/api/src/main/java/com/spotify/ffwd/module/FastForwardModule.java) is a dynamically loaded component that extends the functionality of FastForward. +When a module is [loaded](https://github.com/spotify/ffwd/blob/17de95af58f221ad655dce18515835b2818c7241/core/src/main/java/com/spotify/ffwd/AgentCore.java#L400) it typically [registers a plugin](https://github.com/spotify/ffwd/blob/17de95af58f221ad655dce18515835b2818c7241/modules/pubsub/src/main/java/com/spotify/ffwd/pubsub/PubsubOutputModule.java#L36). -#### Plugin +## Plugin -Either an input or an output plugin. +Either an [input](https://github.com/spotify/ffwd/blob/master/api/src/main/java/com/spotify/ffwd/input/PluginSource.java) or an [output](https://github.com/spotify/ffwd/blob/master/api/src/main/java/com/spotify/ffwd/output/PluginSink.java) plugin. -This provides the implementation to read or send data from the agent. -A plugin can have multiple _instances_ with different configurations (like -which port to listen to). +This provides an implementation which either ingests or emits data from the agent. -#### Early Injector +A plugin can have multiple _instances_ with different configurations (like which port to listen to). -The early injector is setup by AgentCore and is intended to provide the basic -facilities to perform module setup. +## Early Injector + +The [early injector](https://github.com/spotify/ffwd/blob/17de95af58f221ad655dce18515835b2818c7241/core/src/main/java/com/spotify/ffwd/AgentCore.java#L212) is setup by AgentCore and is intended to provide the basic facilities to perform module setup. The following is a list of components that are given access to and their purpose. -* _com.spotify.ffwd.module.PluginContext_ +* [_com.spotify.ffwd.module.PluginContext_](https://github.com/spotify/ffwd/blob/master/api/src/main/java/com/spotify/ffwd/module/PluginContext.java) Register input and output plugins. * _com.fasterxml.jackson.databind.ObjectMapper (config)_ ObjectMapper used to parse provided configuration file. -#### Primary Injector +## Primary Injector -The primary injector contains the dependencies which are available after -modules have been registered and the initial bootstrap is done. +The [primary injector](https://github.com/spotify/ffwd/blob/17de95af58f221ad655dce18515835b2818c7241/core/src/main/java/com/spotify/ffwd/AgentCore.java#L100) contains the dependencies which are available after modules have been registered and the initial bootstrap is done. It contains all the components of the early injector, with the following additions. -* _eu.toolchain.async.AsyncFramework_ - Framework implementation to use for - async operations. +* _eu.toolchain.async.AsyncFramework_ - Framework implementation to use for async operations. * _io.netty.channel.EventLoopGroup (boss)_ Event loop group used for boss threads in ServerBootstrap's. * _io.netty.channel.EventLoopGroup (worker)_ Event loop group used for From 959e1a08ef26f5139790364955b856f25df79d97 Mon Sep 17 00:00:00 2001 From: Peter Kingswell Date: Thu, 11 Mar 2021 11:21:38 -0500 Subject: [PATCH 2/3] added diagram, improved docco --- core/src/main/java/com/spotify/ffwd/AgentCore.java | 9 +++++---- docs/content/_docs/components.md | 2 ++ docs/content/_docs/images/ffwd-components.svg | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 docs/content/_docs/images/ffwd-components.svg diff --git a/core/src/main/java/com/spotify/ffwd/AgentCore.java b/core/src/main/java/com/spotify/ffwd/AgentCore.java index f2a4f944..4dccc754 100644 --- a/core/src/main/java/com/spotify/ffwd/AgentCore.java +++ b/core/src/main/java/com/spotify/ffwd/AgentCore.java @@ -378,10 +378,11 @@ protected void configure() { /** * Reads the configuration of the agent from multiple possible sources. In order of precedence, * the following sources are loaded: - * - System properties - * - Environment variables - * - YAML file in specified location - * - YAML file bundled with JAR + *

+ * - System properties

+ * - Environment variables

+ * - YAML file in specified location

+ * - YAML file bundled with JAR

* * Values are merged, those higher on this list override those that are lower. * diff --git a/docs/content/_docs/components.md b/docs/content/_docs/components.md index 96409979..31302d6f 100644 --- a/docs/content/_docs/components.md +++ b/docs/content/_docs/components.md @@ -6,6 +6,8 @@ title: Components This document covers some of the major components in FastForward. +![Overview of main FFWD components](./images/ffwd-components.svg "Overview of main FFWD components") + ## Module A [module](https://github.com/spotify/ffwd/blob/master/api/src/main/java/com/spotify/ffwd/module/FastForwardModule.java) is a dynamically loaded component that extends the functionality of FastForward. diff --git a/docs/content/_docs/images/ffwd-components.svg b/docs/content/_docs/images/ffwd-components.svg new file mode 100644 index 00000000..442383d8 --- /dev/null +++ b/docs/content/_docs/images/ffwd-components.svg @@ -0,0 +1 @@ + \ No newline at end of file From 5fefa473d35bddd879bb5bbef8c938089fa3f6b6 Mon Sep 17 00:00:00 2001 From: Peter Kingswell Date: Thu, 11 Mar 2021 11:24:35 -0500 Subject: [PATCH 3/3] removed accidentally-added files --- .../ffwd/output/OutputPluginDecorator.java | 138 ------------------ .../ffwd/output/OutputPluginInterface.java | 2 - 2 files changed, 140 deletions(-) delete mode 100644 api/src/main/java/com/spotify/ffwd/output/OutputPluginDecorator.java delete mode 100644 api/src/main/java/com/spotify/ffwd/output/OutputPluginInterface.java diff --git a/api/src/main/java/com/spotify/ffwd/output/OutputPluginDecorator.java b/api/src/main/java/com/spotify/ffwd/output/OutputPluginDecorator.java deleted file mode 100644 index 4f9a3f41..00000000 --- a/api/src/main/java/com/spotify/ffwd/output/OutputPluginDecorator.java +++ /dev/null @@ -1,138 +0,0 @@ -/*- - * -\-\- - * FastForward API - * -- - * Copyright (C) 2016 - 2018 Spotify AB - * -- - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -/-/- - */ - -package com.spotify.ffwd.output; - -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; -import com.google.inject.Key; -import com.google.inject.Module; -import com.google.inject.PrivateModule; -import com.google.inject.Provides; -import com.google.inject.Singleton; -import com.google.inject.name.Named; -import com.google.inject.name.Names; -import com.spotify.ffwd.filter.Filter; -import com.spotify.ffwd.module.Batching; -import com.spotify.ffwd.statistics.BatchingStatistics; -import com.spotify.ffwd.statistics.CoreStatistics; -import com.spotify.ffwd.statistics.NoopCoreStatistics; -import java.util.Optional; -import java.util.logging.Logger; - -@JsonTypeInfo(use = Id.NAME, property = "type") -public abstract class OutputPlugin { - - protected final Batching batching; - protected final Optional filter; - - public OutputPlugin( - final Optional filter, final Batching batching - ) { - this.filter = filter; - this.batching = batching; - } - - /** - * This method allows to wrap plugin sink implementation type depending on configuration. If no - * additional configuration is specified then subtype of com.spotify.ffwd.output - * .BatchedPluginSink will be bound per plugin. - *

- * output := new SubtypeOfBatchedPluginSink() - *

- * If 'flushInterval' key is specified, then corresponding subtype of - * com.spotify.ffwd.output.BatchedPluginSink will be wrapped into com.spotify.ffwd.output - * .FlushingPluginSink: - *

- * output := new FlushingPluginSink(flushInterval, delegator:=output) - *

- * The resulting plugin sink type may be further wrapped into com.spotify.ffwd.output - * .FilteringPluginSink type if 'filter' key is specified in plugin configuration: - *

- * output := new FilteringPluginSink(filter, delegator:=output) - * - * @param input binding key with injection type of plugin sink - * @param wrapKey binding key, containing injection type of wrapping plugin sink - * - * @return module that exposes output binding key - */ - protected Module wrapPluginSink( - final Key input, final Key wrapKey - ) { - return new PrivateModule() { - @Override - protected void configure() { - @SuppressWarnings("unchecked") - Key sinkKey = (Key) input; - - if (batching.getFlushInterval() != null && - BatchablePluginSink.class.isAssignableFrom( - input.getTypeLiteral().getRawType())) { - final Key flushingKey = - Key.get(PluginSink.class, Names.named("flushing")); - // IDEA doesn't like this cast, but it's correct, tho admittedly not pretty - @SuppressWarnings({ "RedundantCast", "unchecked" }) - final Key batchedPluginSink = - (Key) (Key) sinkKey; - - // Use annotation so that we can avoid name space clash - bind(BatchablePluginSink.class) - .annotatedWith(BatchingDelegate.class) - .to(batchedPluginSink); - bind(flushingKey).toInstance( - new BatchingPluginSink(batching.getFlushInterval(), - batching.getBatchSizeLimit(), batching.getMaxPendingFlushes())); - - sinkKey = flushingKey; - } - - if (filter.isPresent()) { - final Key filteringKey = - Key.get(PluginSink.class, Names.named("filtered")); - - // Use annotation so that we can avoid name space clash - bind(PluginSink.class).annotatedWith(FilteringDelegate.class).to(sinkKey); - bind(filteringKey).toInstance(new FilteringPluginSink(filter.get())); - - sinkKey = filteringKey; - } - - bind(wrapKey).to(sinkKey); - expose(wrapKey); - } - - @Provides - @Singleton - public BatchingStatistics batchingStatisticsProvider( - CoreStatistics statistics, @Named("pluginId") String pluginId, Logger log - ) { - if (batching.isReportStatistics()) { - log.info("Enabling reporting of batching-specific metrics for output " + - input.getTypeLiteral().getRawType().getName()); - return statistics.newBatching(pluginId); - } else { - return NoopCoreStatistics.noopBatchingStatistics; - } - } - }; - } - - public abstract Module module(Key key, String id); -} diff --git a/api/src/main/java/com/spotify/ffwd/output/OutputPluginInterface.java b/api/src/main/java/com/spotify/ffwd/output/OutputPluginInterface.java deleted file mode 100644 index dd854bc6..00000000 --- a/api/src/main/java/com/spotify/ffwd/output/OutputPluginInterface.java +++ /dev/null @@ -1,2 +0,0 @@ -package com.spotify.ffwd.output;public interface OutputPluginInterface { -}