From e96106d5e7b17f596b97d5c6c0deb79ff39faaa0 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Mon, 1 Jul 2013 13:37:06 +0200 Subject: [PATCH 01/83] Fix missing return in payload recoder --- src/kodo/payload_recoder.hpp | 4 ++-- test/src/test_rlnc_full_vector_codes.cpp | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/kodo/payload_recoder.hpp b/src/kodo/payload_recoder.hpp index 6c685e78..74367ce1 100644 --- a/src/kodo/payload_recoder.hpp +++ b/src/kodo/payload_recoder.hpp @@ -105,10 +105,10 @@ namespace kodo } /// @copydoc layer::recode(uint8_t*) - void recode(uint8_t *payload) + uint32_t recode(uint8_t *payload) { assert(m_recode_stack); - m_recode_stack->encode(payload); + return m_recode_stack->encode(payload); } /// Make sure we have enough space for both the payload diff --git a/test/src/test_rlnc_full_vector_codes.cpp b/test/src/test_rlnc_full_vector_codes.cpp index 318e80ef..e59191da 100644 --- a/test/src/test_rlnc_full_vector_codes.cpp +++ b/test/src/test_rlnc_full_vector_codes.cpp @@ -463,10 +463,16 @@ inline void invoke_recoding(recoding_parameters param) while( !decoder_two->is_complete() ) { - encoder->encode( &payload[0] ); + uint32_t encode_size = encoder->encode( &payload[0] ); + EXPECT_TRUE(encode_size <= payload.size()); + EXPECT_TRUE(encode_size > 0); + decoder_one->decode( &payload[0] ); - decoder_one->recode( &payload[0] ); + uint32_t recode_size = decoder_one->recode( &payload[0] ); + EXPECT_TRUE(recode_size <= payload.size()); + EXPECT_TRUE(recode_size > 0); + decoder_two->decode( &payload[0] ); } From 250c23bdff13ea01af8b1a1e7d1eb010f505abff Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Mon, 1 Jul 2013 13:38:05 +0200 Subject: [PATCH 02/83] Updating news --- NEWS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.rst b/NEWS.rst index bb713635..3cc10bc6 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -13,6 +13,7 @@ Latest * Minor: Added new example showing some one way to use some of the debug layers in kodo. The example is in the examples folder called use_debug_layers +* Bug: Fix missing return in the payload_recoder recode() function. 11.0.0 ------ From b1e8c8c4b74e7fc2eb0946ca38adc0f96c15c0e7 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Mon, 1 Jul 2013 20:31:59 +0200 Subject: [PATCH 03/83] Preparing to create tag 11.1.0 --- NEWS.rst | 4 ++++ wscript | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index 3cc10bc6..4ddcb4d9 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -6,6 +6,10 @@ of every change, see the Git log. Latest ------ +* tbd + +11.1.0 +------ * Minor: Added new cached_symbol_decoder layer, this layer does not perform any decoding on the incoming symbol, but provides access to the encoded symbol's coefficients and data. An example use_cached_symbol_decoder was diff --git a/wscript b/wscript index e4d13842..6828e4bb 100644 --- a/wscript +++ b/wscript @@ -4,7 +4,7 @@ import os APPNAME = 'kodo' -VERSION = '11.0.0' +VERSION = '11.1.0' def recurse_helper(ctx, name): if not ctx.has_dependency_path(name): From 972ae083727ce62de3e633fdb6d1d7c38938ef77 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 11:56:13 +0200 Subject: [PATCH 04/83] Updating documentation about dependencies --- docs/source/getting_started.rst | 4 +- .../source/using_kodo_in_your_application.rst | 101 ++++++++++++++++-- wscript | 25 ++--- 3 files changed, 107 insertions(+), 23 deletions(-) diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 4b9246de..82180fb2 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -211,7 +211,9 @@ Provided that you have the `Tools Needed`_ installed. .. note:: The ``waf configure`` step might take several minutes depending on the speed of your Internet connection. This would be a good time to grab a coffee or similar while the dependencies are - downloaded. + downloaded. If you do not have an Internet connection you can see + the :ref:`using-kodo-in-your-application` section which shows how + to manually download and specify the Kodo dependencies. .. note:: If you downloaded the library as a zip archive and you have not setup git to automatically authenticate against github.com you diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index 1b02ade4..011524b7 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -29,32 +29,90 @@ them. The libraries are: of C++ utilities. We use only a subset of these functionalities such as smart-pointers. - * http://boost.org + * http://github.com/steinwurf/external-boost-light If you have tried to use the Kodo build scripts you will notice that these download a number of additional libraries. These libraries are only needed when/if you want to compile the Kodo unit-tests, examples or benchmarks. +4. **waf-tools**: This repository contains additional tools used by + out build system. These tools adds functionality to waf which are + used e.g. by our continuous-integration build system. + + * http://github.com/steinwurf/external-waf-tools + +5. **gtest**: The Google C++ Unit Testing Framework is used by all the + Kodo unit tests to ensure the library functions correctly. + + * http://github.com/steinwurf/external-gtest + +6. **gauge**: Gauge is a C++ benchmarking tool which we use in Kodo to + profile the implemented algorithms. + + * http://github.com/steinwurf/cxx-gauge + Download Kodo Dependencies -------------------------- -All dependencies required by Kodo are hosted on github.com and may be found at -github.com/steinwurf. +All dependencies required by Kodo are hosted on github.com and may be found +at http://github.com/steinwurf. There are several ways in which you may get the Kodo library and its dependencies. Which approach you prefer might depend on your intended use-case for Kodo. -1. As shown in :ref:`quick-start` the Kodo build scripts supports downloading - the dependency repositories. The build script with do a git clone and - checkout the latest compatible tagged version of the dependency. +1. As shown in :ref:`getting_started` the Kodo build scripts supports + downloading the dependency repositories automatically. The build + script with do a git clone and checkout the latest compatible tagged + version of the dependency. 2. You may wish to manually download Kodo's dependencies as separate git repositories. In the following we describe this option. -Clone the git repository (recommended) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +3. You can also download the Kodo dependencies as zip or tar.gz archives + from the dependencies corresponding github.com page. + +Selecting the correct versions +------------------------------ +If you use the automatic approach by letting the build scripts download the +dependencies, they will select the latest compatible version. If you download +the dependencies manually you will have to select a compatible version. This +information is stored in the ``wscript`` file found in Kodo's root folder. + +Within that file you will find all Kodo's dependencies specified in the +following way: + +:: + + bundle.add_dependency(opt, + resolve.ResolveGitMajorVersion( + name = 'fifi', + git_repository = 'github.com/steinwurf/fifi.git', + major_version = 9)) + +The above command sets up a dependency for the Fifi library. The version +is specified in the ``major_version = 9`` line. This means that Kodo +requires version ``9.x.y`` of the Fifi library, where ``x.y`` should be +selected to pick the newest available version. Visiting the download page +at github.com for the Fifi library: + +* https://github.com/steinwurf/fifi/releases + +We get a list of available versions. At the time of writing this would be +version ``9.1.0``. These version numbers are available as ``git tags`` if you +choose to manually checkout the git repositories. + +Configuring Kodo with manually downloaded dependencies +------------------------------------------------------ +Before moving on it is important to stress that this step is only +necessary if you wish to build the Kodo unit tests and benchmarks using +the Kodo build system. If you simply want to use Kodo in your application +this is not necessary. If that is your goal you can skip to the +`Example application using makefile`_ section. + +Here we will show how to configure Kodo to use the manually downloaded +dependencies: 1. Create a suitable directory for the projects (optional) @@ -63,12 +121,15 @@ Clone the git repository (recommended) mkdir dev cd dev -2. Clone and download the Fifi libraries by running: +2. Clone or download the Fifi libraries by running: :: git clone git://github.com/steinwurf/fifi.git + or by downloading zip or tar.gz file from + https://github.com/steinwurf/fifi/releases + 3. Clone and download the Sak libraries by running: :: @@ -87,7 +148,27 @@ Clone the git repository (recommended) its own version control repositories, if you wish, you may also use download Boost using those repositories. -Example using makefile / command-line +4. Clone and download the extra Waf tools: + + :: + + git clone git://github.com/steinwurf/external-waf-tools.git + +4. Clone and download the Gtest library. + + :: + + git clone git://github.com/steinwurf/external-gtest.git + + +4. Clone and download the Gauge library. + + :: + + git clone git://github.com/steinwurf/cxx-gauge.git + + +Example application using makefile ------------------------------------- If you would like to see an example of building an application with diff --git a/wscript b/wscript index e4d13842..a09afadc 100644 --- a/wscript +++ b/wscript @@ -21,15 +21,15 @@ def options(opt): bundle.add_dependency(opt, resolve.ResolveGitMajorVersion( - name = 'waf-tools', - git_repository = 'github.com/steinwurf/external-waf-tools.git', - major_version = 2)) + name = 'fifi', + git_repository = 'github.com/steinwurf/fifi.git', + major_version = 9)) bundle.add_dependency(opt, resolve.ResolveGitMajorVersion( - name = 'gtest', - git_repository = 'github.com/steinwurf/external-gtest.git', - major_version = 2)) + name = 'sak', + git_repository = 'github.com/steinwurf/sak.git', + major_version = 10)) bundle.add_dependency(opt, resolve.ResolveGitMajorVersion( @@ -39,15 +39,16 @@ def options(opt): bundle.add_dependency(opt, resolve.ResolveGitMajorVersion( - name = 'sak', - git_repository = 'github.com/steinwurf/sak.git', - major_version = 10)) + name = 'waf-tools', + git_repository = 'github.com/steinwurf/external-waf-tools.git', + major_version = 2)) bundle.add_dependency(opt, resolve.ResolveGitMajorVersion( - name = 'fifi', - git_repository = 'github.com/steinwurf/fifi.git', - major_version = 9)) + name = 'gtest', + git_repository = 'github.com/steinwurf/external-gtest.git', + major_version = 2)) + bundle.add_dependency(opt, resolve.ResolveGitMajorVersion( From c6d8eccd442346803febf26022cf72cb9b91aae7 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 12:08:01 +0200 Subject: [PATCH 05/83] Updating documentation about dependencies --- .../source/using_kodo_in_your_application.rst | 46 +++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index 011524b7..ac380991 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -36,18 +36,18 @@ download a number of additional libraries. These libraries are only needed when/if you want to compile the Kodo unit-tests, examples or benchmarks. -4. **waf-tools**: This repository contains additional tools used by +4. **Waf-tools**: This repository contains additional tools used by out build system. These tools adds functionality to waf which are used e.g. by our continuous-integration build system. * http://github.com/steinwurf/external-waf-tools -5. **gtest**: The Google C++ Unit Testing Framework is used by all the +5. **Gtest**: The Google C++ Unit Testing Framework is used by all the Kodo unit tests to ensure the library functions correctly. * http://github.com/steinwurf/external-gtest -6. **gauge**: Gauge is a C++ benchmarking tool which we use in Kodo to +6. **Gauge**: Gauge is a C++ benchmarking tool which we use in Kodo to profile the implemented algorithms. * http://github.com/steinwurf/cxx-gauge @@ -114,6 +114,9 @@ this is not necessary. If that is your goal you can skip to the Here we will show how to configure Kodo to use the manually downloaded dependencies: +Download using Git +.................. + 1. Create a suitable directory for the projects (optional) :: @@ -127,9 +130,6 @@ dependencies: git clone git://github.com/steinwurf/fifi.git - or by downloading zip or tar.gz file from - https://github.com/steinwurf/fifi/releases - 3. Clone and download the Sak libraries by running: :: @@ -148,25 +148,53 @@ dependencies: its own version control repositories, if you wish, you may also use download Boost using those repositories. -4. Clone and download the extra Waf tools: +5. Clone and download the extra Waf-tools: :: git clone git://github.com/steinwurf/external-waf-tools.git -4. Clone and download the Gtest library. +6. Clone and download the Gtest library. :: git clone git://github.com/steinwurf/external-gtest.git -4. Clone and download the Gauge library. +7. Clone and download the Gauge library. :: git clone git://github.com/steinwurf/cxx-gauge.git +Now we have to visit the downloaded repositories and select the correct +versions e.g. for Fifi, first list the available tags: +:: + + cd fifi + git tag -l + +Using the information from the ``wscript`` (described in +`Selecting the correct versions`_) we can checkout a tagged version: + +:: + git checkout 9.1.0 + +We now do this for all the downloaded repositories. + +Download as zip/tar.gz archives +............................... + +Here we have to visit the download pages of the different dependencies +and download the correct versions (described in `Selecting the correct +versions`_): + +1. Fifi: https://github.com/steinwurf/fifi/releases +2. Sak: https://github.com/steinwurf/sak/releases +3. Boost: https://github.com/steinwurf/external-boost-light/releases +4. Waf-tools: https://github.com/steinwurf/external-waf-tools/releases +5. Gtest: https://github.com/steinwurf/external-gtest/releases +6. Gauge: https://github.com/steinwurf/cxx-gauge/releases Example application using makefile ------------------------------------- From 7edfe037623825fdd2604a0b256b267ffa43fcb9 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 12:09:29 +0200 Subject: [PATCH 06/83] Updating documentation about dependencies --- .../source/using_kodo_in_your_application.rst | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index ac380991..c80e26ff 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -146,7 +146,7 @@ Download using Git For example many Linux distributions support installing Boost via the package manager. Alternatively Boost also provides its own version control repositories, if you - wish, you may also use download Boost using those repositories. + wish, you may also download Boost using those repositories. 5. Clone and download the extra Waf-tools: @@ -189,12 +189,18 @@ Here we have to visit the download pages of the different dependencies and download the correct versions (described in `Selecting the correct versions`_): -1. Fifi: https://github.com/steinwurf/fifi/releases -2. Sak: https://github.com/steinwurf/sak/releases -3. Boost: https://github.com/steinwurf/external-boost-light/releases -4. Waf-tools: https://github.com/steinwurf/external-waf-tools/releases -5. Gtest: https://github.com/steinwurf/external-gtest/releases -6. Gauge: https://github.com/steinwurf/cxx-gauge/releases +1. Fifi: + https://github.com/steinwurf/fifi/releases +2. Sak: + https://github.com/steinwurf/sak/releases +3. Boost: + https://github.com/steinwurf/external-boost-light/releases +4. Waf-tools: + https://github.com/steinwurf/external-waf-tools/releases +5. Gtest: + https://github.com/steinwurf/external-gtest/releases +6. Gauge: + https://github.com/steinwurf/cxx-gauge/releases Example application using makefile ------------------------------------- From b081c6ac2996b5c3520727929ea6277c6628cc96 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 12:24:51 +0200 Subject: [PATCH 07/83] Updating documentation about dependencies --- .../source/using_kodo_in_your_application.rst | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index c80e26ff..ec141d0a 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -92,7 +92,7 @@ following way: major_version = 9)) The above command sets up a dependency for the Fifi library. The version -is specified in the ``major_version = 9`` line. This means that Kodo +required is specified in the ``major_version = 9`` line. This means that Kodo requires version ``9.x.y`` of the Fifi library, where ``x.y`` should be selected to pick the newest available version. Visiting the download page at github.com for the Fifi library: @@ -178,6 +178,7 @@ Using the information from the ``wscript`` (described in `Selecting the correct versions`_) we can checkout a tagged version: :: + git checkout 9.1.0 We now do this for all the downloaded repositories. @@ -202,6 +203,38 @@ versions`_): 6. Gauge: https://github.com/steinwurf/cxx-gauge/releases +Configure Kodo to use the manual dependencies +............................................. + +After downloading all the dependencies manually we have to inform the +Kodo build scripts to use those instead of trying to automatically +downloading them. This is done using the following command: + +:: + + python waf configure --bundle=NONE --fifi-path=insert-path-to/fifi --sak-path=insert-path-to/sak/ --boost-path=insert-path-to/external-boost-light/ --waf-tools-path=insert-path-to/external-waf-tools/ --gtest-path=insert-path-to/external-gtest/ --gauge-path=insert-path-to/cxx-gauge/ + +The bundle options supports a number of different use-cases: + +The following will bundle all dependencies but the Fifi library which we +have to manually specify a path for: +:: + + --bundle=ALL,-fifi --fifi-path=insert-path-to/fifi + +Or we may bundle only Fifi: +:: + + python waf configure --bundle=NONE,fifi --sak-path=insert-path-to/sak/ --boost-path=insert-path-to/external-boost-light/ --waf-tools-path=insert-path-to/external-waf-tools/ --gtest-path=insert-path-to/external-gtest/ --gauge-path=insert-path-to/cxx-gauge/ + +More libraries may be added to the ``--bundle=`` option using commas e.g. +bundle all but Fifi and Sak +:: + + --bundle=ALL,-fifi,-sak --fifi-path=insert-path-to/fifi --sak-path=insert-path-to/sak + + + Example application using makefile ------------------------------------- From edbd993c7fa5fde740a27efec0fe26f50cb2934a Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 12:26:36 +0200 Subject: [PATCH 08/83] Updating documentation about dependencies --- docs/source/using_kodo_in_your_application.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index ec141d0a..684fdf5e 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -214,13 +214,12 @@ downloading them. This is done using the following command: python waf configure --bundle=NONE --fifi-path=insert-path-to/fifi --sak-path=insert-path-to/sak/ --boost-path=insert-path-to/external-boost-light/ --waf-tools-path=insert-path-to/external-waf-tools/ --gtest-path=insert-path-to/external-gtest/ --gauge-path=insert-path-to/cxx-gauge/ -The bundle options supports a number of different use-cases: - -The following will bundle all dependencies but the Fifi library which we -have to manually specify a path for: +The bundle options supports a number of different use-cases. The following +will bundle all dependencies but the Fifi library which we have to +manually specify a path for: :: - --bundle=ALL,-fifi --fifi-path=insert-path-to/fifi + python waf configure --bundle=ALL,-fifi --fifi-path=insert-path-to/fifi Or we may bundle only Fifi: :: @@ -231,7 +230,7 @@ More libraries may be added to the ``--bundle=`` option using commas e.g. bundle all but Fifi and Sak :: - --bundle=ALL,-fifi,-sak --fifi-path=insert-path-to/fifi --sak-path=insert-path-to/sak + python waf configure --bundle=ALL,-fifi,-sak --fifi-path=insert-path-to/fifi --sak-path=insert-path-to/sak From 8637473641b20953de67b3b10cdb7cb4b152d6da Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 12:28:04 +0200 Subject: [PATCH 09/83] Updating documentation about dependencies --- docs/source/using_kodo_in_your_application.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index 684fdf5e..0b3fdbc9 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -232,6 +232,10 @@ bundle all but Fifi and Sak python waf configure --bundle=ALL,-fifi,-sak --fifi-path=insert-path-to/fifi --sak-path=insert-path-to/sak +The bundle options can be seen by running: +:: + + python waf --help Example application using makefile From 86631f8171857b655e84374c66721ba09d0e0a8b Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 12:28:55 +0200 Subject: [PATCH 10/83] Updating documentation about dependencies --- docs/source/using_kodo_in_your_application.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index 0b3fdbc9..1b35bb7e 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -248,6 +248,8 @@ in the ``examples/sample_makefile`` folder in the `Kodo repository`_. .. _`Kodo repository`: https://github.com/steinwurf/kodo +In this case it only requires that you have Fifi, Sak and Boost downloaded. + .. From aedec32d296d9186fb9145b147423d9295f2ceee Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 12:34:19 +0200 Subject: [PATCH 11/83] Updating documentation about dependencies --- .../source/using_kodo_in_your_application.rst | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index 1b35bb7e..40c82097 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -52,27 +52,6 @@ benchmarks. * http://github.com/steinwurf/cxx-gauge -Download Kodo Dependencies --------------------------- - -All dependencies required by Kodo are hosted on github.com and may be found -at http://github.com/steinwurf. - -There are several ways in which you may get the Kodo library and its -dependencies. Which approach you prefer might depend on your intended -use-case for Kodo. - -1. As shown in :ref:`getting_started` the Kodo build scripts supports - downloading the dependency repositories automatically. The build - script with do a git clone and checkout the latest compatible tagged - version of the dependency. - -2. You may wish to manually download Kodo's dependencies as separate git - repositories. In the following we describe this option. - -3. You can also download the Kodo dependencies as zip or tar.gz archives - from the dependencies corresponding github.com page. - Selecting the correct versions ------------------------------ If you use the automatic approach by letting the build scripts download the @@ -103,13 +82,38 @@ We get a list of available versions. At the time of writing this would be version ``9.1.0``. These version numbers are available as ``git tags`` if you choose to manually checkout the git repositories. +Download Kodo Dependencies +-------------------------- + +All dependencies required by Kodo are hosted on github.com and may be found +at http://github.com/steinwurf. + +There are several ways in which you may get the Kodo library and its +dependencies. Which approach you prefer might depend on your intended +use-case for Kodo. + +1. As shown in :ref:`getting_started` the Kodo build scripts supports + downloading the dependency repositories automatically. The build + script with do a ``git clone`` and checkout the latest compatible tagged + version of the dependency. + +2. You may wish to manually download Kodo's dependencies as separate git + repositories. In the following we describe this option. + +3. You can also download the Kodo dependencies as zip or tar.gz archives + from the dependencies corresponding github.com page. + + Configuring Kodo with manually downloaded dependencies ------------------------------------------------------ -Before moving on it is important to stress that this step is only -necessary if you wish to build the Kodo unit tests and benchmarks using -the Kodo build system. If you simply want to use Kodo in your application -this is not necessary. If that is your goal you can skip to the -`Example application using makefile`_ section. +Before moving on it is important to stress that downloading all +dependencies is only necessary if you wish to build the Kodo unit tests +and benchmarks using the Kodo build system. If you simply want to use Kodo +in your application you only need to download the Fifi, Sak and Boost +dependencies and you do not need to build the Kodo library (since it is +header-only). +If that is your goal you can skip to the `Example application using +makefile`_ section after downloading the three required libraries.. Here we will show how to configure Kodo to use the manually downloaded dependencies: From e9455c9958978cadc2bdd60b2be43a7c87126a2a Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 12:37:40 +0200 Subject: [PATCH 12/83] Updating documentation about dependencies --- docs/source/using_kodo_in_your_application.rst | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index 40c82097..dd647cb8 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -103,9 +103,6 @@ use-case for Kodo. 3. You can also download the Kodo dependencies as zip or tar.gz archives from the dependencies corresponding github.com page. - -Configuring Kodo with manually downloaded dependencies ------------------------------------------------------- Before moving on it is important to stress that downloading all dependencies is only necessary if you wish to build the Kodo unit tests and benchmarks using the Kodo build system. If you simply want to use Kodo @@ -115,8 +112,6 @@ header-only). If that is your goal you can skip to the `Example application using makefile`_ section after downloading the three required libraries.. -Here we will show how to configure Kodo to use the manually downloaded -dependencies: Download using Git .................. @@ -207,8 +202,9 @@ versions`_): 6. Gauge: https://github.com/steinwurf/cxx-gauge/releases -Configure Kodo to use the manual dependencies -............................................. + +Configuring Kodo with manually downloaded dependencies +------------------------------------------------------ After downloading all the dependencies manually we have to inform the Kodo build scripts to use those instead of trying to automatically From c9b602e046d26e19ed4c5ec50bac7555cf2c9cf0 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 12:39:38 +0200 Subject: [PATCH 13/83] Updating documentation about dependencies --- docs/source/getting_started.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 82180fb2..6020b919 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -1,8 +1,8 @@ +.. _getting_started + Getting Started =============== -.. _getting_started - Tools Needed ------------ In order to start developing using Kodo you need the following tools installed: From bfb7ec0fb2978d1192a85daf278461589f1776af Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 12:40:56 +0200 Subject: [PATCH 14/83] Updating documentation about dependencies --- docs/source/getting_started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 6020b919..aa9dffdc 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -1,4 +1,4 @@ -.. _getting_started +.. _getting_started: Getting Started =============== From 3b74775c8608e67f938a600c10d1556d97ced6d8 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 15:38:28 +0200 Subject: [PATCH 15/83] Updating documentation about dependencies --- docs/source/c_bindings.rst | 103 +++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 docs/source/c_bindings.rst diff --git a/docs/source/c_bindings.rst b/docs/source/c_bindings.rst new file mode 100644 index 00000000..c11bbd61 --- /dev/null +++ b/docs/source/c_bindings.rst @@ -0,0 +1,103 @@ +.. _c_bindings: + +Kodo C bindings +=============== + +This repository C high-level bindings for the Kodo Network Coding library. +The bindings provide access to basic functionality provided by Kodo, such +as encoding and decoding of data. The examples folder provides sample +applications showing usage of the C API. + +Getting started +----------------------- +The source code for the Kodo C bindings are available at our github.com +repository: + +* https://github.com/steinwurf/kodo-c-bindings + +To obtain the source code you can either clone the repository with +git or download a released version. + +To build the library you need the tools described on the Kodo +`getting_started`_ page. + +Dependencies +------------ +To use the Kodo C bindings you first have to fetch the necessary +dependencies. There are two ways of getting the dependencies: + +1. Allowing the build scripts to automatically download the + dependencies via git (this requires an Internet connection). +2. Downloading the dependencies manually. + +For step 1 you may proceed directly to the `Build`_ section which will +do the automatic download. + +For step 2 we have to first download the dependencies needed. + +Downloading the Dependencies +---------------------------- +The dependencies required to build the Kodo C bindings are Kodo itself + +the dependencies of Kodo. Kodo's dependencies are specified on the +`using-kodo-in-your-application`_ page. On that page you will also find the +description of how to build Kodo with manually downloaded dependencies. This +procedure is the same for the Kodo C-bindings. + +Be sure to read the `Selecting the correct versions`_ section to see +which versions you need to download (here you should use the ``wscript`` +in the kodo-c-bindings repository to obtain the version numbers). + +Build (manually downloaded dependencies) +======================================== +Following the instructions on the `using-kodo-in-your-application`_ page +you should have downloaded the following dependencies: Fifi, Sak, Boost, +Waf-tools, Gtest, Gauge and Kodo. + +To configure the project run: +:: + + python waf configure --bundle=NONE --kodo-path=insert-path-to/kodo --fifi-path=insert-path-to/fifi --sak-path=insert-path-to/sak/ --boost-path=insert-path-to/external-boost-light/ --waf-tools-path=insert-path-to/external-waf-tools/ --gtest-path=insert-path-to/external-gtest/ --gauge-path=insert-path-to/cxx-gauge/ + +After configure run the following command to build the static library: +:: + + python waf build + +Build (automatically downloading dependencies) +============================================== +If you wish to let the waf build scripts automatically download the +source code you can use the steps outline here: + +To configure the project run: +:: + + python waf configure --bundle=ALL + +This will configure the project and download all the dependencies needed +(the dependencies will be located in a folder called bundle_depencies) in +the folder where you run the command. + +After configure run the following command to build the static library: +:: + + python waf build + + +Linking with an application +=========================== +Running the ``waf build`` step will produce a static library in +the ``build`` folder called ``libckodo.a`` on Linux and Mac and +``ckodo.lib`` on Windows. The following section will show you how to +link with the library in your application. To do this you need to +include ``ckodo.h`` in your application and link with ``libckodo``. +Here is a typical gcc link command: + +:: + + gcc myapp.c -o myapp -Ipath_to_ckodo_h -Wl,-Bstatic -Lpath_to_libkodo_a -lckodo -Wl,-Bdynamic -lstdc++ + +Substitute the ``path_to_ckodo_h`` with the path of ``ckodo.h`` similarly +Substitute the ``path_to_libkodo_a`` with the path of the ``libkodo.a`` +library. + + From 3c0a8f2c031d6992e7403b546ee56ab364d2c921 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 15:38:55 +0200 Subject: [PATCH 16/83] Updating documentation about dependencies --- docs/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index 6a9e3dc1..147f0eb9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -43,6 +43,7 @@ Documentation hacking_kodo howto misc + c_bindings nc_glossary From 48e0616b4e5b18e4fc68bad4172a78c6a00e730e Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 15:45:29 +0200 Subject: [PATCH 17/83] Updating documentation about dependencies --- docs/source/c_bindings.rst | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/source/c_bindings.rst b/docs/source/c_bindings.rst index c11bbd61..ebc093ef 100644 --- a/docs/source/c_bindings.rst +++ b/docs/source/c_bindings.rst @@ -9,7 +9,7 @@ as encoding and decoding of data. The examples folder provides sample applications showing usage of the C API. Getting started ------------------------ +--------------- The source code for the Kodo C bindings are available at our github.com repository: @@ -21,8 +21,8 @@ git or download a released version. To build the library you need the tools described on the Kodo `getting_started`_ page. -Dependencies ------------- +Building +-------- To use the Kodo C bindings you first have to fetch the necessary dependencies. There are two ways of getting the dependencies: @@ -30,13 +30,15 @@ dependencies. There are two ways of getting the dependencies: dependencies via git (this requires an Internet connection). 2. Downloading the dependencies manually. -For step 1 you may proceed directly to the `Build`_ section which will -do the automatic download. +For step 1 you may proceed directly to the `Build (automatically +downloading dependencies)`_ section which will do the automatic download. -For step 2 we have to first download the dependencies needed. +For step 2 we have to first download the dependencies needed described in +the `Build (manually downloaded dependencies)`_ section. -Downloading the Dependencies ----------------------------- + +Build (manually downloaded dependencies) +---------------------------------------- The dependencies required to build the Kodo C bindings are Kodo itself + the dependencies of Kodo. Kodo's dependencies are specified on the `using-kodo-in-your-application`_ page. On that page you will also find the @@ -47,8 +49,6 @@ Be sure to read the `Selecting the correct versions`_ section to see which versions you need to download (here you should use the ``wscript`` in the kodo-c-bindings repository to obtain the version numbers). -Build (manually downloaded dependencies) -======================================== Following the instructions on the `using-kodo-in-your-application`_ page you should have downloaded the following dependencies: Fifi, Sak, Boost, Waf-tools, Gtest, Gauge and Kodo. @@ -64,7 +64,7 @@ After configure run the following command to build the static library: python waf build Build (automatically downloading dependencies) -============================================== +---------------------------------------------- If you wish to let the waf build scripts automatically download the source code you can use the steps outline here: @@ -84,7 +84,7 @@ After configure run the following command to build the static library: Linking with an application -=========================== +--------------------------- Running the ``waf build`` step will produce a static library in the ``build`` folder called ``libckodo.a`` on Linux and Mac and ``ckodo.lib`` on Windows. The following section will show you how to From c47957595ddc9c34d83796f7f0a231c4c6370e13 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 15:48:38 +0200 Subject: [PATCH 18/83] Updating documentation about dependencies --- docs/source/c_bindings.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/c_bindings.rst b/docs/source/c_bindings.rst index ebc093ef..5c45e66f 100644 --- a/docs/source/c_bindings.rst +++ b/docs/source/c_bindings.rst @@ -19,7 +19,7 @@ To obtain the source code you can either clone the repository with git or download a released version. To build the library you need the tools described on the Kodo -`getting_started`_ page. +:ref:`getting_started` page. Building -------- @@ -41,15 +41,15 @@ Build (manually downloaded dependencies) ---------------------------------------- The dependencies required to build the Kodo C bindings are Kodo itself + the dependencies of Kodo. Kodo's dependencies are specified on the -`using-kodo-in-your-application`_ page. On that page you will also find the -description of how to build Kodo with manually downloaded dependencies. This -procedure is the same for the Kodo C-bindings. +:ref:`using-kodo-in-your-application` page. On that page you will also +find the description of how to build Kodo with manually downloaded +dependencies. This procedure is the same for the Kodo C-bindings. Be sure to read the `Selecting the correct versions`_ section to see which versions you need to download (here you should use the ``wscript`` in the kodo-c-bindings repository to obtain the version numbers). -Following the instructions on the `using-kodo-in-your-application`_ page +Following the instructions on the :ref:`using-kodo-in-your-application` page you should have downloaded the following dependencies: Fifi, Sak, Boost, Waf-tools, Gtest, Gauge and Kodo. From b07678f9c963754b4c27b0b4d8bc18c44f703a2a Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 15:49:11 +0200 Subject: [PATCH 19/83] Updating documentation about dependencies --- docs/source/c_bindings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/c_bindings.rst b/docs/source/c_bindings.rst index 5c45e66f..7f82af63 100644 --- a/docs/source/c_bindings.rst +++ b/docs/source/c_bindings.rst @@ -45,7 +45,7 @@ the dependencies of Kodo. Kodo's dependencies are specified on the find the description of how to build Kodo with manually downloaded dependencies. This procedure is the same for the Kodo C-bindings. -Be sure to read the `Selecting the correct versions`_ section to see +Be sure to read the :ref:`Selecting the correct versions` section to see which versions you need to download (here you should use the ``wscript`` in the kodo-c-bindings repository to obtain the version numbers). From 37e5f4f2a74868a5dd9215bae3365de5533d894e Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 15:51:57 +0200 Subject: [PATCH 20/83] Updating documentation about dependencies --- docs/source/c_bindings.rst | 4 ++-- docs/source/using_kodo_in_your_application.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/c_bindings.rst b/docs/source/c_bindings.rst index 7f82af63..130a3c73 100644 --- a/docs/source/c_bindings.rst +++ b/docs/source/c_bindings.rst @@ -41,11 +41,11 @@ Build (manually downloaded dependencies) ---------------------------------------- The dependencies required to build the Kodo C bindings are Kodo itself + the dependencies of Kodo. Kodo's dependencies are specified on the -:ref:`using-kodo-in-your-application` page. On that page you will also +:ref:`kodo-dependencies` section. On that page you will also find the description of how to build Kodo with manually downloaded dependencies. This procedure is the same for the Kodo C-bindings. -Be sure to read the :ref:`Selecting the correct versions` section to see +Be sure to read the :ref:`selecting-the-correct-versions` section to see which versions you need to download (here you should use the ``wscript`` in the kodo-c-bindings repository to obtain the version numbers). diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index dd647cb8..43a4b09a 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -52,6 +52,7 @@ benchmarks. * http://github.com/steinwurf/cxx-gauge +.. _selecting-the-correct-versions: Selecting the correct versions ------------------------------ If you use the automatic approach by letting the build scripts download the From d3b30e583779a9da71fc6b92a2e44335f84d11b3 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 15:52:54 +0200 Subject: [PATCH 21/83] Updating documentation about dependencies --- docs/source/c_bindings.rst | 4 ++-- docs/source/using_kodo_in_your_application.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/c_bindings.rst b/docs/source/c_bindings.rst index 130a3c73..cec1a631 100644 --- a/docs/source/c_bindings.rst +++ b/docs/source/c_bindings.rst @@ -30,10 +30,10 @@ dependencies. There are two ways of getting the dependencies: dependencies via git (this requires an Internet connection). 2. Downloading the dependencies manually. -For step 1 you may proceed directly to the `Build (automatically +For option 1 you may proceed directly to the `Build (automatically downloading dependencies)`_ section which will do the automatic download. -For step 2 we have to first download the dependencies needed described in +For option 2 we have to first download the dependencies needed described in the `Build (manually downloaded dependencies)`_ section. diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index 43a4b09a..7cc8e206 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -6,6 +6,7 @@ In the following we will describe what you need to do to use Kodo in your application / project. .. _kodo-dependencies: + Kodo dependencies ----------------- In Kodo we rely on a number of external libraries, these must be available From 8c66b530c68b840ae7807f7a16ea431566141113 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 15:53:25 +0200 Subject: [PATCH 22/83] Updating documentation about dependencies --- docs/source/using_kodo_in_your_application.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index 7cc8e206..a2e7533c 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -54,6 +54,7 @@ benchmarks. * http://github.com/steinwurf/cxx-gauge .. _selecting-the-correct-versions: + Selecting the correct versions ------------------------------ If you use the automatic approach by letting the build scripts download the From d7ea80b2894c515555cbf6aec025402cbe43c97c Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 15:56:07 +0200 Subject: [PATCH 23/83] Updating documentation about dependencies --- docs/source/getting_started.rst | 2 ++ docs/source/getting_the_sourcecode.rst | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index aa9dffdc..a733a220 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -3,6 +3,8 @@ Getting Started =============== +.. _tools-needed: + Tools Needed ------------ In order to start developing using Kodo you need the following tools installed: diff --git a/docs/source/getting_the_sourcecode.rst b/docs/source/getting_the_sourcecode.rst index bc1dce76..42097650 100644 --- a/docs/source/getting_the_sourcecode.rst +++ b/docs/source/getting_the_sourcecode.rst @@ -16,7 +16,7 @@ use-case for Kodo. 2. If you wish to use Kodo in a separate project, possibly your own build tools. You may wish to download Kodo's dependencies as separate git repositories. For more information about this see the - section `Using Kodo in Your Application`_. + section :ref:`using-kodo-in-your-application`. In the following we will only describe option 1 (using the Kodo buildscripts). @@ -76,11 +76,12 @@ Quick Start (building Kodo examples and unit tests) If you are primarily interested in quickly trying some Kodo examples or building the unit-tests, we have tried to make that really easy. -Provided that you have the `Tools Needed`_ installed. +Provided that you have the :ref:`tools-needed` installed. .. note:: We recommend trying to build and run the unit-tests, before using Kodo in your own project. However, if you want to skip - this step you may jump directly to :ref:include-kodo-in-project + this step you may jump directly to + :ref:`using-kodo-in-your-application` 1. Navigate to the directory containing the Kodo sources: From afe8051e9d5bb6af6eec975d2a4b9975759efc74 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 15:57:13 +0200 Subject: [PATCH 24/83] Updating documentation about dependencies --- docs/source/c_bindings.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/c_bindings.rst b/docs/source/c_bindings.rst index cec1a631..b554dfc7 100644 --- a/docs/source/c_bindings.rst +++ b/docs/source/c_bindings.rst @@ -38,7 +38,7 @@ the `Build (manually downloaded dependencies)`_ section. Build (manually downloaded dependencies) ----------------------------------------- +........................................ The dependencies required to build the Kodo C bindings are Kodo itself + the dependencies of Kodo. Kodo's dependencies are specified on the :ref:`kodo-dependencies` section. On that page you will also @@ -64,7 +64,7 @@ After configure run the following command to build the static library: python waf build Build (automatically downloading dependencies) ----------------------------------------------- +.............................................. If you wish to let the waf build scripts automatically download the source code you can use the steps outline here: From 71520ec95ae4c029d3fce54f7ddeb0e7dc7a53d4 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 15:59:07 +0200 Subject: [PATCH 25/83] Updating documentation about dependencies --- docs/source/c_bindings.rst | 2 +- docs/source/using_kodo_in_your_application.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/c_bindings.rst b/docs/source/c_bindings.rst index b554dfc7..81c11e97 100644 --- a/docs/source/c_bindings.rst +++ b/docs/source/c_bindings.rst @@ -49,7 +49,7 @@ Be sure to read the :ref:`selecting-the-correct-versions` section to see which versions you need to download (here you should use the ``wscript`` in the kodo-c-bindings repository to obtain the version numbers). -Following the instructions on the :ref:`using-kodo-in-your-application` page +Following the instructions on the :ref:`download-kodo-dependencies` page you should have downloaded the following dependencies: Fifi, Sak, Boost, Waf-tools, Gtest, Gauge and Kodo. diff --git a/docs/source/using_kodo_in_your_application.rst b/docs/source/using_kodo_in_your_application.rst index a2e7533c..8860f31b 100644 --- a/docs/source/using_kodo_in_your_application.rst +++ b/docs/source/using_kodo_in_your_application.rst @@ -85,6 +85,8 @@ We get a list of available versions. At the time of writing this would be version ``9.1.0``. These version numbers are available as ``git tags`` if you choose to manually checkout the git repositories. +.. _download-kodo-dependencies: + Download Kodo Dependencies -------------------------- From a7449c63b536ebd6c99db4a79e72122ef202a778 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Thu, 4 Jul 2013 16:02:39 +0200 Subject: [PATCH 26/83] Updating documentation about dependencies --- docs/source/c_bindings.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/source/c_bindings.rst b/docs/source/c_bindings.rst index 81c11e97..90de7e85 100644 --- a/docs/source/c_bindings.rst +++ b/docs/source/c_bindings.rst @@ -40,7 +40,7 @@ the `Build (manually downloaded dependencies)`_ section. Build (manually downloaded dependencies) ........................................ The dependencies required to build the Kodo C bindings are Kodo itself + -the dependencies of Kodo. Kodo's dependencies are specified on the +the dependencies of Kodo. Kodo's dependencies are specified in the :ref:`kodo-dependencies` section. On that page you will also find the description of how to build Kodo with manually downloaded dependencies. This procedure is the same for the Kodo C-bindings. @@ -49,9 +49,10 @@ Be sure to read the :ref:`selecting-the-correct-versions` section to see which versions you need to download (here you should use the ``wscript`` in the kodo-c-bindings repository to obtain the version numbers). -Following the instructions on the :ref:`download-kodo-dependencies` page +Following the instructions in the :ref:`download-kodo-dependencies` section you should have downloaded the following dependencies: Fifi, Sak, Boost, -Waf-tools, Gtest, Gauge and Kodo. +Waf-tools, Gtest and Gauge. You should download Kodo in the same way. After +this we are ready to build. To configure the project run: :: From bf5c924293907fcf6a104db137a089bcf3d38b02 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Fri, 5 Jul 2013 02:05:11 +0200 Subject: [PATCH 27/83] Adding new policies for direction of linear block decoder --- src/kodo/forward_linear_block_decoder.hpp | 718 ++++++++++++++++++ .../src/test_forward_linear_block_decoder.cpp | 100 +++ 2 files changed, 818 insertions(+) create mode 100644 src/kodo/forward_linear_block_decoder.hpp create mode 100644 test/src/test_forward_linear_block_decoder.cpp diff --git a/src/kodo/forward_linear_block_decoder.hpp b/src/kodo/forward_linear_block_decoder.hpp new file mode 100644 index 00000000..920e9198 --- /dev/null +++ b/src/kodo/forward_linear_block_decoder.hpp @@ -0,0 +1,718 @@ +// Copyright Steinwurf ApS 2011-2012. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +#pragma once + +#include + +#include +#include +#include +#include + +#include + +#include +#include + +namespace kodo +{ + + template + struct forward_policy + { + + forward_policy(const Stack& stack) + : m_stack(stack), + m_index(0), + m_end(stack.symbols()) + { } + + forward_policy(const Stack& stack, uint32_t start) + : m_stack(stack), + m_index(start), + m_end(stack.symbols()) + { } + + forward_policy(const Stack& stack, uint32_t start, uint32_t end) + : m_stack(stack), + m_index(start), + m_end(end) + { } + + + bool at_end() const + { + return m_index >= m_end; + } + + void advance() + { + ++m_index; + } + + uint32_t index() const + { + return m_index; + } + + static uint32_t max(uint32_t a, uint32_t b) + { + return std::max(a,b); + } + + static uint32_t min(uint32_t a, uint32_t b) + { + return std::min(a,b); + } + + // static uint32_t next(uint32_t value) + // { + + // } + + const Stack& m_stack; + uint32_t m_index; + uint32_t m_end; + + }; + + + template + struct backward_policy + { + + backward_policy(const Stack& stack) + : m_stack(stack), + m_index(stack.symbols()), + m_end(1) + { } + + backward_policy(const Stack& stack, uint32_t start) + : m_stack(stack), + m_index(start), + m_end(1) + { } + + backward_policy(const Stack& stack, uint32_t start, uint32_t end) + : m_stack(stack), + m_index(start), + m_end(end) + { } + + bool at_end() const + { + return m_index < m_end; + } + + void advance() + { + --m_index; + } + + uint32_t index() const + { + return m_index - 1; + } + + static uint32_t max(uint32_t a, uint32_t b) + { + return std::min(a,b); + } + + static uint32_t min(uint32_t a, uint32_t b) + { + return std::max(a,b); + } + + + const Stack& m_stack; + uint32_t m_index; + uint32_t m_end; + + }; + + + + /// @ingroup codec_layers + /// @brief Implements basic linear block decoder. + /// + /// The linear block decoder + /// expects that an encoded symbol is described by a vector of + /// coefficients. Using these coefficients the block decoder subtracts + /// incoming symbols until the original data has been recreated. + template + class forward_linear_block_decoder : public SuperCoder + { + public: + + /// @copydoc layer::field_type + typedef typename SuperCoder::field_type field_type; + + /// @copydoc layer::value_type + typedef typename field_type::value_type value_type; + + /// @copydoc layer::factory + typedef typename SuperCoder::factory factory; + + typedef backward_policy + the_policy; + + public: + + /// Constructor + forward_linear_block_decoder() + : m_rank(0), + m_maximum_pivot(0) + { } + + /// @copydoc layer::construct(Factory&) + template + void construct(Factory &the_factory) + { + SuperCoder::construct(the_factory); + + m_uncoded.resize(the_factory.max_symbols(), false); + m_coded.resize(the_factory.max_symbols(), false); + } + + /// @copydoc layer::initialize(Factory&) + template + void initialize(Factory& the_factory) + { + SuperCoder::initialize(the_factory); + + std::fill_n(m_uncoded.begin(), the_factory.symbols(), false); + std::fill_n(m_coded.begin(), the_factory.symbols(), false); + + m_rank = 0; + + // Depending on the policy we either go from 0 to symbols or + // from symbols to 0. + m_maximum_pivot = the_policy::min(0, the_factory.symbols()); + } + + /// @copydoc layer::decode_symbol(uint8_t*,uint8_t*) + void decode_symbol(uint8_t *symbol_data, + uint8_t *symbol_coefficients) + { + assert(symbol_data != 0); + assert(symbol_coefficients != 0); + + value_type *symbol + = reinterpret_cast(symbol_data); + + value_type *coefficients + = reinterpret_cast(symbol_coefficients); + + decode_coefficients(symbol, coefficients); + } + + /// @copydoc layer::decode_symbol(uint8_t*, uint32_t) + void decode_symbol(uint8_t *symbol_data, + uint32_t symbol_index) + { + assert(symbol_index < SuperCoder::symbols()); + assert(symbol_data != 0); + + if(m_uncoded[symbol_index]) + { + return; + } + + const value_type *symbol + = reinterpret_cast( symbol_data ); + + if(m_coded[symbol_index]) + { + swap_decode(symbol, symbol_index); + } + else + { + // Stores the symbol and updates the corresponding + // encoding vector + store_uncoded_symbol(symbol, symbol_index); + + // Backwards substitution + value_type *coefficients = + SuperCoder::coefficients_value(symbol_index); + + backward_substitute(symbol, coefficients, symbol_index); + + // We have increased the rank if we have finished the + // backwards substitution + ++m_rank; + + m_uncoded[ symbol_index ] = true; + m_maximum_pivot = the_policy::max(symbol_index, m_maximum_pivot); + + } + } + + /// @copydoc layer::is_complete() const + bool is_complete() const + { + return m_rank == SuperCoder::symbols(); + } + + /// @copydoc layer::rank() const + uint32_t rank() const + { + return m_rank; + } + + /// @copydoc layer::symbol_pivot(uint32_t) const + bool symbol_pivot(uint32_t index) const + { + assert(index < SuperCoder::symbols()); + return m_coded[index] || m_uncoded[index]; + } + + /// @todo Add unit test + /// @copydoc layer::symbol_pivot(uint32_t) const + bool symbol_coded(uint32_t index) const + { + assert(symbol_pivot(index)); + return m_coded[index]; + } + + protected: + + /// Decodes a symbol based on the coefficients + /// @param symbol_data buffer containing the encoding symbol + /// @param symbol_id buffer containing the encoding vector + void decode_coefficients(value_type *symbol_data, + value_type *symbol_coefficients) + { + assert(symbol_data != 0); + assert(symbol_coefficients != 0); + + // See if we can find a pivot + auto pivot_index + = forward_substitute_to_pivot( + symbol_data, symbol_coefficients); + + if(!pivot_index) + return; + + if(!fifi::is_binary::value) + { + // Normalize symbol and vector + normalize( + symbol_data,symbol_coefficients,*pivot_index); + } + + // Reduce the symbol further + forward_substitute_from_pivot( + symbol_data, symbol_coefficients, *pivot_index); + + // Now with the found pivot reduce the existing symbols + backward_substitute( + symbol_data, symbol_coefficients, *pivot_index); + + // Now save the received symbol + store_coded_symbol( + symbol_data, symbol_coefficients, *pivot_index); + + // We have increased the rank + ++m_rank; + + m_coded[ *pivot_index ] = true; + + if(*pivot_index > m_maximum_pivot) + { + m_maximum_pivot = *pivot_index; + } + } + + /// When adding a raw symbol (i.e. uncoded) with a specific + /// pivot id and the decoder already contains a coded symbol + /// in that position this function performs the proper swap + /// between the two symbols. + /// @param symbol_data the data for the raw symbol + /// @param pivot_index the pivot position of the raw symbol + void swap_decode(const value_type *symbol_data, + uint32_t pivot_index) + { + assert(m_coded[pivot_index] == true); + assert(m_uncoded[pivot_index] == false); + + m_coded[pivot_index] = false; + + value_type *symbol_i = + SuperCoder::symbol_value(pivot_index); + + value_type *vector_i = + SuperCoder::coefficients_value(pivot_index); + + value_type value = + fifi::get_value(vector_i, pivot_index); + + assert(value == 1); + + // Subtract the new pivot symbol + fifi::set_value(vector_i, pivot_index, 0); + + SuperCoder::subtract(symbol_i, symbol_data, + SuperCoder::symbol_length()); + + // Now continue our new coded symbol we know that it must + // if found it will contain a pivot id > that the current. + decode_coefficients(symbol_i, vector_i); + + // The previous vector may still be in memory + std::fill_n(vector_i, SuperCoder::coefficients_length(), 0); + + // Stores the symbol and sets the pivot in the vector + store_uncoded_symbol(symbol_data, pivot_index); + + m_uncoded[pivot_index] = true; + + // No need to backwards substitute since we are + // replacing an existing symbol. I.e. backwards + // substitution must already have been done. + } + + /// Iterates the encoding vector from where a pivot has been + /// identified and subtracts existing symbols + /// @param symbol_data the data of the encoded symbol + /// @param symbol_id the data constituting the encoding vector + /// @param pivot_index the index of the found pivot element + void normalize(value_type *symbol_data, + value_type *symbol_id, + uint32_t pivot_index) + { + + assert(symbol_id != 0); + assert(symbol_data != 0); + + assert(pivot_index < SuperCoder::symbols()); + + assert(m_uncoded[pivot_index] == false); + assert(m_coded[pivot_index] == false); + + value_type coefficient = + fifi::get_value(symbol_id, pivot_index); + + assert(coefficient > 0); + + value_type inverted_coefficient = + SuperCoder::invert(coefficient); + + // Update symbol and corresponding vector + SuperCoder::multiply(symbol_id, inverted_coefficient, + SuperCoder::coefficients_length()); + + SuperCoder::multiply(symbol_data, inverted_coefficient, + SuperCoder::symbol_length()); + + } + + /// Iterates the encoding vector and subtracts existing symbols + /// until a pivot element is found. + /// @param symbol_data the data of the encoded symbol + /// @param symbol_id the data constituting the encoding vector + /// @return the pivot index if found. + boost::optional forward_substitute_to_pivot( + value_type *symbol_data, + value_type *symbol_id) + { + assert(symbol_id != 0); + assert(symbol_data != 0); + + // uint32_t from = forward_policy::from(*this); + // uint32_t to = forward_policy::to(*this); + + + for(the_policy p(*this); !p.at_end(); p.advance()) + { + uint32_t i = p.index(); + + value_type current_coefficient + = fifi::get_value(symbol_id, i); + + if( current_coefficient ) + { + // If symbol exists + if( symbol_pivot( i ) ) + { + value_type *vector_i = + SuperCoder::coefficients_value( i ); + + value_type *symbol_i = + SuperCoder::symbol_value( i ); + + if(fifi::is_binary::value) + { + SuperCoder::subtract( + symbol_id, vector_i, + SuperCoder::coefficients_length()); + + SuperCoder::subtract( + symbol_data, symbol_i, + SuperCoder::symbol_length()); + } + else + { + SuperCoder::multiply_subtract( + symbol_id, vector_i, + current_coefficient, + SuperCoder::coefficients_length()); + + SuperCoder::multiply_subtract( + symbol_data, symbol_i, + current_coefficient, + SuperCoder::symbol_length()); + } + } + else + { + return boost::optional( i ); + } + } + } + + return boost::none; + } + + /// Iterates the encoding vector from where a pivot has been + /// identified and subtracts existing symbols + /// @param symbol_data the data of the encoded symbol + /// @param symbol_id the data constituting the encoding vector + /// @param pivot_index the index of the found pivot element + void forward_substitute_from_pivot(value_type *symbol_data, + value_type *symbol_id, + uint32_t pivot_index) + { + // We have received an encoded symbol - described + // by the symbol group. We now normalize the + // the encoding vector according to the symbol id. + // I.e. we make sure the pivot position has a "1" + assert(symbol_id != 0); + assert(symbol_data != 0); + + assert(pivot_index < SuperCoder::symbols()); + + assert(m_uncoded[pivot_index] == false); + assert(m_coded[pivot_index] == false); + + // If this pivot index was smaller than the maximum pivot + // index we have, we might also need to backward + // substitute the higher pivot values into the new packet + the_policy p(*this, pivot_index); + p.advance(); + + for(; !p.at_end(); p.advance()) + { + uint32_t i = p.index(); + + // Do we have a non-zero value here? + value_type value = + fifi::get_value(symbol_id, i); + + + if( !value ) + { + continue; + } + + if( symbol_pivot(i) ) + { + value_type *vector_i = + SuperCoder::coefficients_value(i); + + value_type *symbol_i = + SuperCoder::symbol_value(i); + + if(fifi::is_binary::value) + { + SuperCoder::subtract( + symbol_id, vector_i, + SuperCoder::coefficients_length()); + + SuperCoder::subtract( + symbol_data, symbol_i, + SuperCoder::symbol_length()); + } + else + { + SuperCoder::multiply_subtract( + symbol_id, vector_i, value, + SuperCoder::coefficients_length()); + + SuperCoder::multiply_subtract( + symbol_data, symbol_i, value, + SuperCoder::symbol_length()); + } + } + } + } + + /// Backward substitute the found symbol into the + /// existing symbols. + /// @param symbol_data buffer containing the encoding symbol + /// @param symbol_id buffer containing the encoding vector + /// @param pivot_index the pivot index of the symbol in the + /// buffers symbol_id and symbol_data + void backward_substitute(const value_type *symbol_data, + const value_type *symbol_id, + uint32_t pivot_index) + { + assert(symbol_id != 0); + assert(symbol_data != 0); + + assert(pivot_index < SuperCoder::symbols()); + + uint32_t from = the_policy::min(0, SuperCoder::symbols()); + uint32_t to = m_maximum_pivot; + + the_policy p(*this, from, to); + + // We found a "1" that nobody else had as pivot, we now + // substract this packet from other coded packets + // - if they have a "1" on our pivot place + for(;!p.at_end(); p.advance()) + { + uint32_t i = p.index(); + + if( m_uncoded[i] ) + { + // We know that we have no non-zero elements + // outside the pivot position. + continue; + } + + if(i == pivot_index) + { + // We cannot backward substitute into ourself + continue; + } + + if( m_coded[i] ) + { + value_type *vector_i = + SuperCoder::coefficients_value(i); + + value_type value = + fifi::get_value( + vector_i, pivot_index); + + if( value ) + { + + value_type *symbol_i = + SuperCoder::symbol_value(i); + + if(fifi::is_binary::value) + { + SuperCoder::subtract( + vector_i, symbol_id, + SuperCoder::coefficients_length()); + + SuperCoder::subtract( + symbol_i, symbol_data, + SuperCoder::symbol_length()); + } + else + { + + // Update symbol and corresponding vector + SuperCoder::multiply_subtract( + vector_i, symbol_id, value, + SuperCoder::coefficients_length()); + + SuperCoder::multiply_subtract( + symbol_i, symbol_data, value, + SuperCoder::symbol_length()); + } + } + } + } + } + + /// Store an encoded symbol and encoding vector with the specified + /// pivot found. + /// @param symbol_data buffer containing the encoding symbol + /// @param symbol_coefficients buffer containing the symbol + /// coefficients + /// @param pivot_index the pivot index + void store_coded_symbol(const value_type *symbol_data, + const value_type *symbol_coefficients, + uint32_t pivot_index) + { + assert(m_uncoded[pivot_index] == false); + assert(m_coded[pivot_index] == false); + assert(symbol_coefficients != 0); + assert(symbol_data != 0); + assert(SuperCoder::is_symbol_available(pivot_index)); + + auto coefficient_storage = + sak::storage(symbol_coefficients, + SuperCoder::coefficients_size()); + + SuperCoder::set_coefficients( + pivot_index, coefficient_storage); + + // Copy it into the symbol storage + sak::mutable_storage dest = + sak::storage(SuperCoder::symbol(pivot_index), + SuperCoder::symbol_size()); + + sak::const_storage src = + sak::storage(symbol_data, SuperCoder::symbol_size()); + + sak::copy_storage(dest, src); + } + + /// Stores an uncoded or fully decoded symbol + /// @param symbol_data the data for the symbol + /// @param pivot_index the pivot index of the symbol + void store_uncoded_symbol(const value_type *symbol_data, + uint32_t pivot_index) + { + assert(symbol_data != 0); + assert(m_uncoded[pivot_index] == false); + assert(m_coded[pivot_index] == false); + assert(SuperCoder::is_symbol_available(pivot_index)); + + // Update the corresponding vector + value_type *vector_dest = + SuperCoder::coefficients_value( pivot_index ); + + // Zero out the memory first + std::fill_n(vector_dest, SuperCoder::coefficients_length(), 0); + + fifi::set_value(vector_dest, pivot_index, 1U); + + // Copy it into the symbol storage + sak::mutable_storage dest = + sak::storage(SuperCoder::symbol(pivot_index), + SuperCoder::symbol_size()); + + sak::const_storage src = + sak::storage(symbol_data, SuperCoder::symbol_size()); + + sak::copy_storage(dest, src); + + } + + protected: + + /// The current rank of the decoder + uint32_t m_rank; + + /// Stores the current maximum pivot index + uint32_t m_maximum_pivot; + + /// Tracks whether a symbol is contained which + /// is fully decoded + std::vector m_uncoded; + + /// Tracks whether a symbol is partially decoded + std::vector m_coded; + }; + +} + diff --git a/test/src/test_forward_linear_block_decoder.cpp b/test/src/test_forward_linear_block_decoder.cpp new file mode 100644 index 00000000..29316996 --- /dev/null +++ b/test/src/test_forward_linear_block_decoder.cpp @@ -0,0 +1,100 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +/// @file test_forward_linear_block_decoder.cpp Unit tests for the +/// kodo::forward_linear_block_decoder + +#include +#include + +#include +#include + +#include "basic_api_test_helper.hpp" + +namespace kodo +{ + template + class test_forward_stack + : public // Payload API + // Codec Header API + // Symbol ID API + // Codec API + forward_linear_block_decoder< + // Coefficient Storage API + coefficient_storage< + coefficient_info< + // Storage API + deep_symbol_storage< + storage_bytes_used< + storage_block_info< + // Finite Field API + finite_field_math::type, + finite_field_info + > > > > > > > > > + { }; + +} + + +/// Run the tests typical coefficients stack +TEST(TestForwardLinearBlockDecoder, test_decoder) +{ + + kodo::test_forward_stack::factory f(8, 1600); + + auto d = f.build(); + + EXPECT_EQ(d->coefficients_size(), 1U); + + uint8_t coefficients[1]; + + // Create an encoding vector looking like this: 01000010 + coefficients[0] = 0; + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 6, 1U); + + // Create a dummy symbol + std::vector symbol = random_vector(d->symbol_size()); + + d->decode_symbol(&symbol[0], coefficients); + + EXPECT_EQ(d->rank(), 1U); + EXPECT_TRUE(d->symbol_pivot(1)); + + // // Create an encoding vector looking like this: 10000010 + // coefficients[0] = 0; + // fifi::set_value(coefficients, 0, 1U); + // fifi::set_value(coefficients, 6, 1U); + + // d->decode_symbol(&symbol[0], coefficients); + + // EXPECT_EQ(d->rank(), 2U); + // EXPECT_TRUE(d->symbol_pivot(0)); + // EXPECT_TRUE(d->symbol_pivot(1)); + + // // Create an encoding vector looking like this: 11100010 + // coefficients[0] = 0; + // fifi::set_value(coefficients, 0, 1U); + // fifi::set_value(coefficients, 1, 1U); + // fifi::set_value(coefficients, 2, 1U); + // fifi::set_value(coefficients, 6, 1U); + + // d->decode_symbol(&symbol[0], coefficients); + + // EXPECT_EQ(d->rank(), 3U); + // EXPECT_TRUE(d->symbol_pivot(0)); + // EXPECT_TRUE(d->symbol_pivot(1)); + // EXPECT_TRUE(d->symbol_pivot(2)); + + +} + + + From 2b17123b592976b038d1435a3f90877c621df470 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Fri, 5 Jul 2013 16:34:39 +0200 Subject: [PATCH 28/83] Commiting the intial version of the forward policy --- benchmark/throughput/codes.hpp | 32 +++++++++ benchmark/throughput/main.cpp | 52 ++++++++++++++ benchmark/throughput/plot_throughput.py | 26 +++---- src/kodo/forward_linear_block_decoder.hpp | 87 ++++------------------- 4 files changed, 112 insertions(+), 85 deletions(-) diff --git a/benchmark/throughput/codes.hpp b/benchmark/throughput/codes.hpp index 85dddcf1..abb2fa4d 100644 --- a/benchmark/throughput/codes.hpp +++ b/benchmark/throughput/codes.hpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace kodo @@ -83,6 +84,37 @@ namespace kodo > > > > > > > > > > > > > > > > > { }; + /// RLNC decoder which uses the policy based linear block decoder + template + class forward_full_rlnc_decoder + : public // Payload API + payload_recoder::type, + finite_field_info + > > > > > > > > > > > > > > > + { }; + } diff --git a/benchmark/throughput/main.cpp b/benchmark/throughput/main.cpp index 57ffa405..941d729a 100644 --- a/benchmark/throughput/main.cpp +++ b/benchmark/throughput/main.cpp @@ -457,6 +457,9 @@ BENCHMARK_OPTION(throughput_density_options) gauge::runner::instance().register_options(options); } +//------------------------------------------------------------------ +// FullRLNC +//------------------------------------------------------------------ typedef throughput_benchmark< kodo::full_rlnc_encoder, @@ -494,6 +497,55 @@ BENCHMARK_F(setup_rlnc_throughput2325, FullRLNC, Prime2325, 5) run_benchmark(); } +//------------------------------------------------------------------ +// ForwardFullRLNC +//------------------------------------------------------------------ + +typedef throughput_benchmark< + kodo::full_rlnc_encoder, + kodo::forward_full_rlnc_decoder > + setup_forward_rlnc_throughput; + +BENCHMARK_F(setup_forward_rlnc_throughput, ForwardFullRLNC, Binary, 5) +{ + run_benchmark(); +} + +typedef throughput_benchmark< + kodo::full_rlnc_encoder, + kodo::forward_full_rlnc_decoder > + setup_forward_rlnc_throughput8; + +BENCHMARK_F(setup_forward_rlnc_throughput8, ForwardFullRLNC, Binary8, 5) +{ + run_benchmark(); +} + +typedef throughput_benchmark< + kodo::full_rlnc_encoder, + kodo::forward_full_rlnc_decoder > + setup_forward_rlnc_throughput16; + +BENCHMARK_F(setup_forward_rlnc_throughput16, ForwardFullRLNC, Binary16, 5) +{ + run_benchmark(); +} + +typedef throughput_benchmark< + kodo::full_rlnc_encoder, + kodo::forward_full_rlnc_decoder > + setup_forward_rlnc_throughput2325; + +BENCHMARK_F(setup_forward_rlnc_throughput2325, ForwardFullRLNC, Prime2325, 5) +{ + run_benchmark(); +} + + +//------------------------------------------------------------------ +// FullDelayedRLNC +//------------------------------------------------------------------ + typedef throughput_benchmark< kodo::full_rlnc_encoder, kodo::full_delayed_rlnc_decoder > diff --git a/benchmark/throughput/plot_throughput.py b/benchmark/throughput/plot_throughput.py index fa31304f..7bd41b69 100644 --- a/benchmark/throughput/plot_throughput.py +++ b/benchmark/throughput/plot_throughput.py @@ -15,15 +15,6 @@ def plot_throughput(csvfile): df = pd.read_csv(csvfile) - def density_to_string(density): - if not np.isnan(density): - return " density {}".format(density) - else: - return "" - - # df['density'] = df['density'].map(density_to_string) - # df['test'] = df['testcase'].map(str) + '.' + df['benchmark'] + df['density'] - plot_groups = list(df.groupby(by=['testcase', 'symbol_size', 'type'])) # Group the plots @@ -35,9 +26,18 @@ def density_to_string(density): else: return "" - df['benchmark'] = df['benchmark'] + ' ' + df['density'].map(density_to_string) + # Combine the testcase and benchmark columns into one (used for labels) + if not 'density' in df: + df['test'] = df['testcase'].map(str) + '.' + df['benchmark'] + + df = df.drop(['testcase','benchmark'], axis = 1) + else: + df['test'] = df['testcase'].map(str) + '.' + df['benchmark'] +\ + ' ' + df['density'].map(density_to_string) + df = df.drop(['testcase','benchmark', 'density'], axis = 1) + - group = df.groupby(by = ['benchmark', 'symbols']) + group = df.groupby(by = ['test', 'symbols']) def compute_throughput(group): s = group['throughput'] @@ -47,8 +47,8 @@ def compute_throughput(group): df = group.apply(compute_throughput) df = df.unstack(level=0) - df['mean'].plot(title="Throughput {} {} p={}B".format(test, type, symbol_size), - kind='bar') + df['mean'].plot(title="Throughput {} {} p={}B".format( + test, type, symbol_size), kind='bar') plt.show() diff --git a/src/kodo/forward_linear_block_decoder.hpp b/src/kodo/forward_linear_block_decoder.hpp index 920e9198..d64a277a 100644 --- a/src/kodo/forward_linear_block_decoder.hpp +++ b/src/kodo/forward_linear_block_decoder.hpp @@ -17,69 +17,11 @@ #include #include +#include + namespace kodo { - template - struct forward_policy - { - - forward_policy(const Stack& stack) - : m_stack(stack), - m_index(0), - m_end(stack.symbols()) - { } - - forward_policy(const Stack& stack, uint32_t start) - : m_stack(stack), - m_index(start), - m_end(stack.symbols()) - { } - - forward_policy(const Stack& stack, uint32_t start, uint32_t end) - : m_stack(stack), - m_index(start), - m_end(end) - { } - - - bool at_end() const - { - return m_index >= m_end; - } - - void advance() - { - ++m_index; - } - - uint32_t index() const - { - return m_index; - } - - static uint32_t max(uint32_t a, uint32_t b) - { - return std::max(a,b); - } - - static uint32_t min(uint32_t a, uint32_t b) - { - return std::min(a,b); - } - - // static uint32_t next(uint32_t value) - // { - - // } - - const Stack& m_stack; - uint32_t m_index; - uint32_t m_end; - - }; - - template struct backward_policy { @@ -157,8 +99,7 @@ namespace kodo /// @copydoc layer::factory typedef typename SuperCoder::factory factory; - typedef backward_policy - the_policy; + typedef forward_linear_block_decoder_policy the_policy; public: @@ -246,7 +187,9 @@ namespace kodo ++m_rank; m_uncoded[ symbol_index ] = true; - m_maximum_pivot = the_policy::max(symbol_index, m_maximum_pivot); + + m_maximum_pivot = + the_policy::max(symbol_index, m_maximum_pivot); } } @@ -422,11 +365,10 @@ namespace kodo assert(symbol_id != 0); assert(symbol_data != 0); - // uint32_t from = forward_policy::from(*this); - // uint32_t to = forward_policy::to(*this); + uint32_t start = the_policy::min(0, SuperCoder::symbols()-1); + uint32_t end = the_policy::max(0, SuperCoder::symbols()-1); - - for(the_policy p(*this); !p.at_end(); p.advance()) + for(the_policy p(start, end); !p.at_end(); p.advance()) { uint32_t i = p.index(); @@ -501,7 +443,10 @@ namespace kodo // If this pivot index was smaller than the maximum pivot // index we have, we might also need to backward // substitute the higher pivot values into the new packet - the_policy p(*this, pivot_index); + uint32_t end = the_policy::max(0, SuperCoder::symbols()-1); + the_policy p(pivot_index, end); + + // Jump past the pivot_index position p.advance(); for(; !p.at_end(); p.advance()) @@ -565,15 +510,13 @@ namespace kodo assert(pivot_index < SuperCoder::symbols()); - uint32_t from = the_policy::min(0, SuperCoder::symbols()); + uint32_t from = the_policy::min(0, SuperCoder::symbols()-1); uint32_t to = m_maximum_pivot; - the_policy p(*this, from, to); - // We found a "1" that nobody else had as pivot, we now // substract this packet from other coded packets // - if they have a "1" on our pivot place - for(;!p.at_end(); p.advance()) + for(the_policy p(from, to); !p.at_end(); p.advance()) { uint32_t i = p.index(); From 00078fc1d1c0c9974a2f475c8eea24876213faf6 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Fri, 5 Jul 2013 16:39:39 +0200 Subject: [PATCH 29/83] Working on the policies for the linear block decoder --- ...st_forward_linear_block_decoder_policy.cpp | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 test/src/test_forward_linear_block_decoder_policy.cpp diff --git a/test/src/test_forward_linear_block_decoder_policy.cpp b/test/src/test_forward_linear_block_decoder_policy.cpp new file mode 100644 index 00000000..f79c554d --- /dev/null +++ b/test/src/test_forward_linear_block_decoder_policy.cpp @@ -0,0 +1,53 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +/// @file test_forward_linear_block_decoder_policy.cpp Unit test +/// for the forward_linear_block_decoder_policy + +#include +#include + +#include + + +/// Run the tests typical coefficients stack +TEST(TestForwardLinearBlockDecoderPolicy, test_functions) +{ + + { + kodo::forward_linear_block_decoder_policy policy(0, 5); + EXPECT_EQ(policy.at_end(), false); + EXPECT_EQ(policy.index(), 0U); + policy.advance(); + + EXPECT_EQ(policy.at_end(), false); + EXPECT_EQ(policy.index(), 1U); + policy.advance(); + + EXPECT_EQ(policy.at_end(), false); + EXPECT_EQ(policy.index(), 2U); + policy.advance(); + + EXPECT_EQ(policy.at_end(), false); + EXPECT_EQ(policy.index(), 3U); + policy.advance(); + + EXPECT_EQ(policy.at_end(), false); + EXPECT_EQ(policy.index(), 4U); + policy.advance(); + + EXPECT_EQ(policy.at_end(), false); + EXPECT_EQ(policy.index(), 5U); + policy.advance(); + + EXPECT_EQ(policy.at_end(), true); + + EXPECT_EQ(kodo::forward_linear_block_decoder_policy::max(1,2), 2); + EXPECT_EQ(kodo::forward_linear_block_decoder_policy::min(1,2), 1); + } +} + + + From cdd7ab620bce23be710cd4ed686f5b370af5411c Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Fri, 5 Jul 2013 16:41:34 +0200 Subject: [PATCH 30/83] Working on the policies for the linear block decoder --- .../forward_linear_block_decoder_policy.hpp | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 src/kodo/forward_linear_block_decoder_policy.hpp diff --git a/src/kodo/forward_linear_block_decoder_policy.hpp b/src/kodo/forward_linear_block_decoder_policy.hpp new file mode 100644 index 00000000..bd6b6251 --- /dev/null +++ b/src/kodo/forward_linear_block_decoder_policy.hpp @@ -0,0 +1,89 @@ +// Copyright Steinwurf ApS 2011-2012. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +#pragma once + +#include +#include + +namespace kodo +{ + + /// Policy object for determining the "direction" of the + /// linear_block_decoder e.g. whether we will look for the + /// first pivot from the beginning of the coefficients vector + /// or in some other way. + /// + /// The forward_linear_block decoder will search for pivots from + /// the left of the encoding vector. + /// E.g. if you see an encoding vector like: + /// + /// +-----------> direction of search for pivots + /// + /// 0 1 0 1 1 0 0 + /// ^ ^ ^ + /// | | | + /// | | +----+ Last coefficient searched + /// | +--------------+ First non zero coefficient + /// +----------------+ First coefficient searched + /// + struct forward_linear_block_decoder_policy + { + /// Construct the policy with the desired start and stop interval + /// but ends included so [start:stop] + /// @param start The first element to look at + /// @param stop The last element to look at + forward_linear_block_decoder_policy(uint32_t start, uint32_t stop) + : m_start(start), + m_stop(stop) + { + assert(m_start <= m_stop); + } + + /// @return true if the policy is at the end + bool at_end() const + { + return m_start > m_stop; + } + + /// Advance the policy to the next index + void advance() + { + assert(!at_end()); + ++m_start; + } + + /// @return the current index + uint32_t index() const + { + return m_start; + } + + /// @param a The first value + /// @param b The second value + /// @return the maximum value of the two input values + static uint32_t max(uint32_t a, uint32_t b) + { + return std::max(a,b); + } + + /// @param a The first value + /// @param b The second value + /// @return the minimum value of the two input values + static uint32_t min(uint32_t a, uint32_t b) + { + return std::min(a,b); + } + + /// The start value + uint32_t m_start; + + /// The end value + uint32_t m_stop; + + }; + +} + From 317cadc3318ce2a3b00a2d4d14dd02df0b5d7725 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Sun, 7 Jul 2013 23:54:52 +0200 Subject: [PATCH 31/83] Building blocks for directional block decoder done, missing unit-tests --- src/kodo/backward_linear_block_decoder.hpp | 38 ++ .../backward_linear_block_decoder_policy.hpp | 89 +++ src/kodo/directional_linear_block_decoder.hpp | 610 ++++++++++++++++++ src/kodo/forward_linear_block_decoder.hpp | 69 +- src/kodo/linear_block_decoder.hpp | 559 +--------------- src/kodo/linear_block_decoder_delayed.hpp | 22 +- .../test_backward_linear_block_decoder.cpp | 167 +++++ ...t_backward_linear_block_decoder_policy.cpp | 56 ++ .../src/test_forward_linear_block_decoder.cpp | 115 +++- 9 files changed, 1080 insertions(+), 645 deletions(-) create mode 100644 src/kodo/backward_linear_block_decoder.hpp create mode 100644 src/kodo/backward_linear_block_decoder_policy.hpp create mode 100644 src/kodo/directional_linear_block_decoder.hpp create mode 100644 test/src/test_backward_linear_block_decoder.cpp create mode 100644 test/src/test_backward_linear_block_decoder_policy.cpp diff --git a/src/kodo/backward_linear_block_decoder.hpp b/src/kodo/backward_linear_block_decoder.hpp new file mode 100644 index 00000000..8657faa6 --- /dev/null +++ b/src/kodo/backward_linear_block_decoder.hpp @@ -0,0 +1,38 @@ +// Copyright Steinwurf ApS 2011-2012. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +#pragma once + +#include + +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include + +namespace kodo +{ + /// @ingroup codec_layers + /// @brief Implements basic linear block decoder. + /// + /// The linear block decoder + /// expects that an encoded symbol is described by a vector of + /// coefficients. Using these coefficients the block decoder subtracts + /// incoming symbols until the original data has been recreated. + template + class backward_linear_block_decoder : public directional_linear_block_decoder< + backward_linear_block_decoder_policy, SuperCoder> + { }; + +} + diff --git a/src/kodo/backward_linear_block_decoder_policy.hpp b/src/kodo/backward_linear_block_decoder_policy.hpp new file mode 100644 index 00000000..ca1830b6 --- /dev/null +++ b/src/kodo/backward_linear_block_decoder_policy.hpp @@ -0,0 +1,89 @@ +// Copyright Steinwurf ApS 2011-2012. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +#pragma once + +#include +#include + +namespace kodo +{ + + /// Policy object for determining the "direction" of the + /// linear_block_decoder e.g. whether we will look for the + /// first pivot from the beginning of the coefficients vector + /// or in some other way. + /// + /// The backward_linear_block decoder will search for pivots from + /// the right of the encoding vector. + /// E.g. if you see an encoding vector like: + /// + /// <-----------+ direction of search for pivots + /// + /// 0 1 0 1 1 0 0 + /// ^ ^ ^ + /// | | | + /// | | +----+ First coefficient searched + /// | +--------+ First non zero coefficient + /// +----------------+ Last coefficient searched + /// + struct backward_linear_block_decoder_policy + { + /// Construct the policy with the desired start and stop interval + /// but ends included so [start:stop] + /// @param start The first element to look at + /// @param stop The last element to look at + backward_linear_block_decoder_policy(uint32_t start, uint32_t stop) + : m_start(start+1), + m_stop(stop) + { + assert(m_start > m_stop); + } + + /// @return true if the policy is at the end + bool at_end() const + { + return m_start <= m_stop; + } + + /// Advance the policy to the next index + void advance() + { + assert(!at_end()); + --m_start; + } + + /// @return the current index + uint32_t index() const + { + return m_start - 1; + } + + /// @param a The first value + /// @param b The second value + /// @return the maximum value of the two input values + static uint32_t max(uint32_t a, uint32_t b) + { + return std::min(a,b); + } + + /// @param a The first value + /// @param b The second value + /// @return the minimum value of the two input values + static uint32_t min(uint32_t a, uint32_t b) + { + return std::max(a,b); + } + + /// The start value + uint32_t m_start; + + /// The end value + uint32_t m_stop; + + }; + +} + diff --git a/src/kodo/directional_linear_block_decoder.hpp b/src/kodo/directional_linear_block_decoder.hpp new file mode 100644 index 00000000..2713aeda --- /dev/null +++ b/src/kodo/directional_linear_block_decoder.hpp @@ -0,0 +1,610 @@ +// Copyright Steinwurf ApS 2011-2012. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +#pragma once + +#include + +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include + +namespace kodo +{ + /// @ingroup codec_layers + /// @brief Implements basic linear block decoder. + /// + /// The linear block decoder + /// expects that an encoded symbol is described by a vector of + /// coefficients. Using these coefficients the block decoder subtracts + /// incoming symbols until the original data has been recreated. + template + class directional_linear_block_decoder : public SuperCoder + { + public: + + /// @copydoc layer::field_type + typedef typename SuperCoder::field_type field_type; + + /// @copydoc layer::value_type + typedef typename field_type::value_type value_type; + + /// @copydoc layer::factory + typedef typename SuperCoder::factory factory; + + typedef DirectionPolicy direction_policy; + + public: + + /// Constructor + directional_linear_block_decoder() + : m_rank(0), + m_maximum_pivot(0) + { } + + /// @copydoc layer::construct(Factory&) + template + void construct(Factory &the_factory) + { + SuperCoder::construct(the_factory); + + m_uncoded.resize(the_factory.max_symbols(), false); + m_coded.resize(the_factory.max_symbols(), false); + } + + /// @copydoc layer::initialize(Factory&) + template + void initialize(Factory& the_factory) + { + SuperCoder::initialize(the_factory); + + std::fill_n(m_uncoded.begin(), the_factory.symbols(), false); + std::fill_n(m_coded.begin(), the_factory.symbols(), false); + + m_rank = 0; + + // Depending on the policy we either go from 0 to symbols or + // from symbols to 0. + m_maximum_pivot = direction_policy::min(0, the_factory.symbols()-1); + } + + /// @copydoc layer::decode_symbol(uint8_t*,uint8_t*) + void decode_symbol(uint8_t *symbol_data, + uint8_t *symbol_coefficients) + { + assert(symbol_data != 0); + assert(symbol_coefficients != 0); + + value_type *symbol + = reinterpret_cast(symbol_data); + + value_type *coefficients + = reinterpret_cast(symbol_coefficients); + + decode_coefficients(symbol, coefficients); + } + + /// @copydoc layer::decode_symbol(uint8_t*, uint32_t) + void decode_symbol(uint8_t *symbol_data, + uint32_t symbol_index) + { + assert(symbol_index < SuperCoder::symbols()); + assert(symbol_data != 0); + + if(m_uncoded[symbol_index]) + { + return; + } + + const value_type *symbol + = reinterpret_cast( symbol_data ); + + if(m_coded[symbol_index]) + { + swap_decode(symbol, symbol_index); + } + else + { + // Stores the symbol and updates the corresponding + // encoding vector + store_uncoded_symbol(symbol, symbol_index); + + // Backwards substitution + value_type *coefficients = + SuperCoder::coefficients_value(symbol_index); + + backward_substitute(symbol, coefficients, symbol_index); + + // We have increased the rank if we have finished the + // backwards substitution + ++m_rank; + + m_uncoded[ symbol_index ] = true; + + m_maximum_pivot = + direction_policy::max(symbol_index, m_maximum_pivot); + + } + } + + /// @copydoc layer::is_complete() const + bool is_complete() const + { + return m_rank == SuperCoder::symbols(); + } + + /// @copydoc layer::rank() const + uint32_t rank() const + { + return m_rank; + } + + /// @copydoc layer::symbol_pivot(uint32_t) const + bool symbol_pivot(uint32_t index) const + { + assert(index < SuperCoder::symbols()); + return m_coded[index] || m_uncoded[index]; + } + + /// @todo Add unit test + /// @copydoc layer::symbol_pivot(uint32_t) const + bool symbol_coded(uint32_t index) const + { + assert(symbol_pivot(index)); + return m_coded[index]; + } + + protected: + + /// Decodes a symbol based on the coefficients + /// @param symbol_data buffer containing the encoding symbol + /// @param symbol_id buffer containing the encoding vector + void decode_coefficients(value_type *symbol_data, + value_type *symbol_coefficients) + { + assert(symbol_data != 0); + assert(symbol_coefficients != 0); + + // See if we can find a pivot + auto pivot_index + = forward_substitute_to_pivot( + symbol_data, symbol_coefficients); + + if(!pivot_index) + return; + + if(!fifi::is_binary::value) + { + // Normalize symbol and vector + normalize( + symbol_data,symbol_coefficients,*pivot_index); + } + + // Reduce the symbol further + forward_substitute_from_pivot( + symbol_data, symbol_coefficients, *pivot_index); + + // Now with the found pivot reduce the existing symbols + backward_substitute( + symbol_data, symbol_coefficients, *pivot_index); + + // Now save the received symbol + store_coded_symbol( + symbol_data, symbol_coefficients, *pivot_index); + + // We have increased the rank + ++m_rank; + + m_coded[ *pivot_index ] = true; + + m_maximum_pivot = + direction_policy::max(*pivot_index, m_maximum_pivot); + } + + /// When adding a raw symbol (i.e. uncoded) with a specific + /// pivot id and the decoder already contains a coded symbol + /// in that position this function performs the proper swap + /// between the two symbols. + /// @param symbol_data the data for the raw symbol + /// @param pivot_index the pivot position of the raw symbol + void swap_decode(const value_type *symbol_data, + uint32_t pivot_index) + { + assert(m_coded[pivot_index] == true); + assert(m_uncoded[pivot_index] == false); + + m_coded[pivot_index] = false; + + value_type *symbol_i = + SuperCoder::symbol_value(pivot_index); + + value_type *vector_i = + SuperCoder::coefficients_value(pivot_index); + + value_type value = + fifi::get_value(vector_i, pivot_index); + + assert(value == 1); + + // Subtract the new pivot symbol + fifi::set_value(vector_i, pivot_index, 0); + + SuperCoder::subtract(symbol_i, symbol_data, + SuperCoder::symbol_length()); + + // Now continue our new coded symbol we know that it must + // if found it will contain a pivot id > that the current. + decode_coefficients(symbol_i, vector_i); + + // The previous vector may still be in memory + std::fill_n(vector_i, SuperCoder::coefficients_length(), 0); + + // Stores the symbol and sets the pivot in the vector + store_uncoded_symbol(symbol_data, pivot_index); + + m_uncoded[pivot_index] = true; + + // No need to backwards substitute since we are + // replacing an existing symbol. I.e. backwards + // substitution must already have been done. + } + + /// Iterates the encoding vector from where a pivot has been + /// identified and subtracts existing symbols + /// @param symbol_data the data of the encoded symbol + /// @param symbol_id the data constituting the encoding vector + /// @param pivot_index the index of the found pivot element + void normalize(value_type *symbol_data, + value_type *symbol_id, + uint32_t pivot_index) + { + + assert(symbol_id != 0); + assert(symbol_data != 0); + + assert(pivot_index < SuperCoder::symbols()); + + assert(m_uncoded[pivot_index] == false); + assert(m_coded[pivot_index] == false); + + value_type coefficient = + fifi::get_value(symbol_id, pivot_index); + + assert(coefficient > 0); + + value_type inverted_coefficient = + SuperCoder::invert(coefficient); + + // Update symbol and corresponding vector + SuperCoder::multiply(symbol_id, inverted_coefficient, + SuperCoder::coefficients_length()); + + SuperCoder::multiply(symbol_data, inverted_coefficient, + SuperCoder::symbol_length()); + + } + + /// Iterates the encoding vector and subtracts existing symbols + /// until a pivot element is found. + /// @param symbol_data the data of the encoded symbol + /// @param symbol_id the data constituting the encoding vector + /// @return the pivot index if found. + boost::optional forward_substitute_to_pivot( + value_type *symbol_data, + value_type *symbol_id) + { + assert(symbol_id != 0); + assert(symbol_data != 0); + + uint32_t start = direction_policy::min(0, SuperCoder::symbols()-1); + uint32_t end = direction_policy::max(0, SuperCoder::symbols()-1); + + // std::cout << start << " " << end << std::endl; + + for(direction_policy p(start, end); !p.at_end(); p.advance()) + { + uint32_t i = p.index(); + + value_type current_coefficient + = fifi::get_value(symbol_id, i); + + if( current_coefficient ) + { + // If symbol exists + if( symbol_pivot( i ) ) + { + value_type *vector_i = + SuperCoder::coefficients_value( i ); + + value_type *symbol_i = + SuperCoder::symbol_value( i ); + + if(fifi::is_binary::value) + { + SuperCoder::subtract( + symbol_id, vector_i, + SuperCoder::coefficients_length()); + + SuperCoder::subtract( + symbol_data, symbol_i, + SuperCoder::symbol_length()); + } + else + { + SuperCoder::multiply_subtract( + symbol_id, vector_i, + current_coefficient, + SuperCoder::coefficients_length()); + + SuperCoder::multiply_subtract( + symbol_data, symbol_i, + current_coefficient, + SuperCoder::symbol_length()); + } + } + else + { + return boost::optional( i ); + } + } + } + + return boost::none; + } + + /// Iterates the encoding vector from where a pivot has been + /// identified and subtracts existing symbols + /// @param symbol_data the data of the encoded symbol + /// @param symbol_id the data constituting the encoding vector + /// @param pivot_index the index of the found pivot element + void forward_substitute_from_pivot(value_type *symbol_data, + value_type *symbol_id, + uint32_t pivot_index) + { + // We have received an encoded symbol - described + // by the symbol group. We now normalize the + // the encoding vector according to the symbol id. + // I.e. we make sure the pivot position has a "1" + assert(symbol_id != 0); + assert(symbol_data != 0); + + assert(pivot_index < SuperCoder::symbols()); + + assert(m_uncoded[pivot_index] == false); + assert(m_coded[pivot_index] == false); + + // If this pivot index was smaller than the maximum pivot + // index we have, we might also need to backward + // substitute the higher pivot values into the new packet + uint32_t end = direction_policy::max(0, SuperCoder::symbols()-1); + + // std::cout << "Pivot " << pivot_index << " to " << end << std::endl; + + direction_policy p(pivot_index, end); + + // Jump past the pivot_index position + p.advance(); + + for(; !p.at_end(); p.advance()) + { + uint32_t i = p.index(); + + // Do we have a non-zero value here? + value_type value = + fifi::get_value(symbol_id, i); + + + if( !value ) + { + continue; + } + + if( symbol_pivot(i) ) + { + value_type *vector_i = + SuperCoder::coefficients_value(i); + + value_type *symbol_i = + SuperCoder::symbol_value(i); + + if(fifi::is_binary::value) + { + SuperCoder::subtract( + symbol_id, vector_i, + SuperCoder::coefficients_length()); + + SuperCoder::subtract( + symbol_data, symbol_i, + SuperCoder::symbol_length()); + } + else + { + SuperCoder::multiply_subtract( + symbol_id, vector_i, value, + SuperCoder::coefficients_length()); + + SuperCoder::multiply_subtract( + symbol_data, symbol_i, value, + SuperCoder::symbol_length()); + } + } + } + } + + /// Backward substitute the found symbol into the + /// existing symbols. + /// @param symbol_data buffer containing the encoding symbol + /// @param symbol_id buffer containing the encoding vector + /// @param pivot_index the pivot index of the symbol in the + /// buffers symbol_id and symbol_data + void backward_substitute(const value_type *symbol_data, + const value_type *symbol_id, + uint32_t pivot_index) + { + assert(symbol_id != 0); + assert(symbol_data != 0); + + assert(pivot_index < SuperCoder::symbols()); + + uint32_t from = direction_policy::min(0, SuperCoder::symbols()-1); + uint32_t to = m_maximum_pivot; + + // std::cout << from << " " << to << std::endl; + + // We found a "1" that nobody else had as pivot, we now + // substract this packet from other coded packets + // - if they have a "1" on our pivot place + for(direction_policy p(from, to); !p.at_end(); p.advance()) + { + uint32_t i = p.index(); + + if( m_uncoded[i] ) + { + // We know that we have no non-zero elements + // outside the pivot position. + continue; + } + + if(i == pivot_index) + { + // We cannot backward substitute into ourself + continue; + } + + if( m_coded[i] ) + { + value_type *vector_i = + SuperCoder::coefficients_value(i); + + value_type value = + fifi::get_value( + vector_i, pivot_index); + + if( value ) + { + + value_type *symbol_i = + SuperCoder::symbol_value(i); + + if(fifi::is_binary::value) + { + SuperCoder::subtract( + vector_i, symbol_id, + SuperCoder::coefficients_length()); + + SuperCoder::subtract( + symbol_i, symbol_data, + SuperCoder::symbol_length()); + } + else + { + + // Update symbol and corresponding vector + SuperCoder::multiply_subtract( + vector_i, symbol_id, value, + SuperCoder::coefficients_length()); + + SuperCoder::multiply_subtract( + symbol_i, symbol_data, value, + SuperCoder::symbol_length()); + } + } + } + } + } + + /// Store an encoded symbol and encoding vector with the specified + /// pivot found. + /// @param symbol_data buffer containing the encoding symbol + /// @param symbol_coefficients buffer containing the symbol + /// coefficients + /// @param pivot_index the pivot index + void store_coded_symbol(const value_type *symbol_data, + const value_type *symbol_coefficients, + uint32_t pivot_index) + { + assert(m_uncoded[pivot_index] == false); + assert(m_coded[pivot_index] == false); + assert(symbol_coefficients != 0); + assert(symbol_data != 0); + assert(SuperCoder::is_symbol_available(pivot_index)); + + auto coefficient_storage = + sak::storage(symbol_coefficients, + SuperCoder::coefficients_size()); + + SuperCoder::set_coefficients( + pivot_index, coefficient_storage); + + // Copy it into the symbol storage + sak::mutable_storage dest = + sak::storage(SuperCoder::symbol(pivot_index), + SuperCoder::symbol_size()); + + sak::const_storage src = + sak::storage(symbol_data, SuperCoder::symbol_size()); + + sak::copy_storage(dest, src); + } + + /// Stores an uncoded or fully decoded symbol + /// @param symbol_data the data for the symbol + /// @param pivot_index the pivot index of the symbol + void store_uncoded_symbol(const value_type *symbol_data, + uint32_t pivot_index) + { + assert(symbol_data != 0); + assert(m_uncoded[pivot_index] == false); + assert(m_coded[pivot_index] == false); + assert(SuperCoder::is_symbol_available(pivot_index)); + + // Update the corresponding vector + value_type *vector_dest = + SuperCoder::coefficients_value( pivot_index ); + + // Zero out the memory first + std::fill_n(vector_dest, SuperCoder::coefficients_length(), 0); + + fifi::set_value(vector_dest, pivot_index, 1U); + + // Copy it into the symbol storage + sak::mutable_storage dest = + sak::storage(SuperCoder::symbol(pivot_index), + SuperCoder::symbol_size()); + + sak::const_storage src = + sak::storage(symbol_data, SuperCoder::symbol_size()); + + sak::copy_storage(dest, src); + + } + + protected: + + /// The current rank of the decoder + uint32_t m_rank; + + /// Stores the current maximum pivot index + uint32_t m_maximum_pivot; + + /// Tracks whether a symbol is contained which + /// is fully decoded + std::vector m_uncoded; + + /// Tracks whether a symbol is partially decoded + std::vector m_coded; + }; + +} + diff --git a/src/kodo/forward_linear_block_decoder.hpp b/src/kodo/forward_linear_block_decoder.hpp index d64a277a..032b3860 100644 --- a/src/kodo/forward_linear_block_decoder.hpp +++ b/src/kodo/forward_linear_block_decoder.hpp @@ -18,66 +18,10 @@ #include #include +#include namespace kodo { - - template - struct backward_policy - { - - backward_policy(const Stack& stack) - : m_stack(stack), - m_index(stack.symbols()), - m_end(1) - { } - - backward_policy(const Stack& stack, uint32_t start) - : m_stack(stack), - m_index(start), - m_end(1) - { } - - backward_policy(const Stack& stack, uint32_t start, uint32_t end) - : m_stack(stack), - m_index(start), - m_end(end) - { } - - bool at_end() const - { - return m_index < m_end; - } - - void advance() - { - --m_index; - } - - uint32_t index() const - { - return m_index - 1; - } - - static uint32_t max(uint32_t a, uint32_t b) - { - return std::min(a,b); - } - - static uint32_t min(uint32_t a, uint32_t b) - { - return std::max(a,b); - } - - - const Stack& m_stack; - uint32_t m_index; - uint32_t m_end; - - }; - - - /// @ingroup codec_layers /// @brief Implements basic linear block decoder. /// @@ -99,7 +43,7 @@ namespace kodo /// @copydoc layer::factory typedef typename SuperCoder::factory factory; - typedef forward_linear_block_decoder_policy the_policy; + typedef backward_linear_block_decoder_policy the_policy; public: @@ -132,7 +76,7 @@ namespace kodo // Depending on the policy we either go from 0 to symbols or // from symbols to 0. - m_maximum_pivot = the_policy::min(0, the_factory.symbols()); + m_maximum_pivot = the_policy::min(0, the_factory.symbols()-1); } /// @copydoc layer::decode_symbol(uint8_t*,uint8_t*) @@ -368,6 +312,8 @@ namespace kodo uint32_t start = the_policy::min(0, SuperCoder::symbols()-1); uint32_t end = the_policy::max(0, SuperCoder::symbols()-1); + // std::cout << start << " " << end << std::endl; + for(the_policy p(start, end); !p.at_end(); p.advance()) { uint32_t i = p.index(); @@ -444,6 +390,9 @@ namespace kodo // index we have, we might also need to backward // substitute the higher pivot values into the new packet uint32_t end = the_policy::max(0, SuperCoder::symbols()-1); + + // std::cout << pivot_index << " " << end << std::endl; + the_policy p(pivot_index, end); // Jump past the pivot_index position @@ -513,6 +462,8 @@ namespace kodo uint32_t from = the_policy::min(0, SuperCoder::symbols()-1); uint32_t to = m_maximum_pivot; + // std::cout << from << " " << to << std::endl; + // We found a "1" that nobody else had as pivot, we now // substract this packet from other coded packets // - if they have a "1" on our pivot place diff --git a/src/kodo/linear_block_decoder.hpp b/src/kodo/linear_block_decoder.hpp index d34c524c..e4f41bd5 100644 --- a/src/kodo/linear_block_decoder.hpp +++ b/src/kodo/linear_block_decoder.hpp @@ -17,9 +17,12 @@ #include #include +#include +#include +#include + namespace kodo { - /// @ingroup codec_layers /// @brief Implements basic linear block decoder. /// @@ -28,557 +31,9 @@ namespace kodo /// coefficients. Using these coefficients the block decoder subtracts /// incoming symbols until the original data has been recreated. template - class linear_block_decoder : public SuperCoder - { - public: - - /// @copydoc layer::field_type - typedef typename SuperCoder::field_type field_type; - - /// @copydoc layer::value_type - typedef typename field_type::value_type value_type; - - /// @copydoc layer::factory - typedef typename SuperCoder::factory factory; - - public: - - /// Constructor - linear_block_decoder() - : m_rank(0), - m_maximum_pivot(0) - { } - - /// @copydoc layer::construct(Factory&) - template - void construct(Factory &the_factory) - { - SuperCoder::construct(the_factory); - - m_uncoded.resize(the_factory.max_symbols(), false); - m_coded.resize(the_factory.max_symbols(), false); - } - - /// @copydoc layer::initialize(Factory&) - template - void initialize(Factory& the_factory) - { - SuperCoder::initialize(the_factory); - - std::fill_n(m_uncoded.begin(), the_factory.symbols(), false); - std::fill_n(m_coded.begin(), the_factory.symbols(), false); - - m_rank = 0; - m_maximum_pivot = 0; - } - - /// @copydoc layer::decode_symbol(uint8_t*,uint8_t*) - void decode_symbol(uint8_t *symbol_data, - uint8_t *symbol_coefficients) - { - assert(symbol_data != 0); - assert(symbol_coefficients != 0); - - value_type *symbol - = reinterpret_cast(symbol_data); - - value_type *coefficients - = reinterpret_cast(symbol_coefficients); - - decode_coefficients(symbol, coefficients); - } - - /// @copydoc layer::decode_symbol(uint8_t*, uint32_t) - void decode_symbol(uint8_t *symbol_data, - uint32_t symbol_index) - { - assert(symbol_index < SuperCoder::symbols()); - assert(symbol_data != 0); - - if(m_uncoded[symbol_index]) - { - return; - } - - const value_type *symbol - = reinterpret_cast( symbol_data ); - - if(m_coded[symbol_index]) - { - swap_decode(symbol, symbol_index); - } - else - { - // Stores the symbol and updates the corresponding - // encoding vector - store_uncoded_symbol(symbol, symbol_index); - - // Backwards substitution - value_type *coefficients = - SuperCoder::coefficients_value(symbol_index); - - backward_substitute(symbol, coefficients, symbol_index); - - // We have increased the rank if we have finished the - // backwards substitution - ++m_rank; - - m_uncoded[ symbol_index ] = true; - - if(symbol_index > m_maximum_pivot) - { - m_maximum_pivot = symbol_index; - } - - } - } - - /// @copydoc layer::is_complete() const - bool is_complete() const - { - return m_rank == SuperCoder::symbols(); - } - - /// @copydoc layer::rank() const - uint32_t rank() const - { - return m_rank; - } - - /// @copydoc layer::symbol_pivot(uint32_t) const - bool symbol_pivot(uint32_t index) const - { - assert(index < SuperCoder::symbols()); - return m_coded[index] || m_uncoded[index]; - } - - /// @todo Add unit test - /// @copydoc layer::symbol_pivot(uint32_t) const - bool symbol_coded(uint32_t index) const - { - assert(symbol_pivot(index)); - return m_coded[index]; - } - - protected: - - /// Decodes a symbol based on the coefficients - /// @param symbol_data buffer containing the encoding symbol - /// @param symbol_id buffer containing the encoding vector - void decode_coefficients(value_type *symbol_data, - value_type *symbol_coefficients) - { - assert(symbol_data != 0); - assert(symbol_coefficients != 0); - - // See if we can find a pivot - auto pivot_index - = forward_substitute_to_pivot( - symbol_data, symbol_coefficients); - - if(!pivot_index) - return; - - if(!fifi::is_binary::value) - { - // Normalize symbol and vector - normalize( - symbol_data,symbol_coefficients,*pivot_index); - } - - // Reduce the symbol further - forward_substitute_from_pivot( - symbol_data, symbol_coefficients, *pivot_index); - - // Now with the found pivot reduce the existing symbols - backward_substitute( - symbol_data, symbol_coefficients, *pivot_index); - - // Now save the received symbol - store_coded_symbol( - symbol_data, symbol_coefficients, *pivot_index); - - // We have increased the rank - ++m_rank; - - m_coded[ *pivot_index ] = true; - - if(*pivot_index > m_maximum_pivot) - { - m_maximum_pivot = *pivot_index; - } - } - - /// When adding a raw symbol (i.e. uncoded) with a specific - /// pivot id and the decoder already contains a coded symbol - /// in that position this function performs the proper swap - /// between the two symbols. - /// @param symbol_data the data for the raw symbol - /// @param pivot_index the pivot position of the raw symbol - void swap_decode(const value_type *symbol_data, - uint32_t pivot_index) - { - assert(m_coded[pivot_index] == true); - assert(m_uncoded[pivot_index] == false); - - m_coded[pivot_index] = false; - - value_type *symbol_i = - SuperCoder::symbol_value(pivot_index); - - value_type *vector_i = - SuperCoder::coefficients_value(pivot_index); - - value_type value = - fifi::get_value(vector_i, pivot_index); - - assert(value == 1); - - // Subtract the new pivot symbol - fifi::set_value(vector_i, pivot_index, 0); - - SuperCoder::subtract(symbol_i, symbol_data, - SuperCoder::symbol_length()); - - // Now continue our new coded symbol we know that it must - // if found it will contain a pivot id > that the current. - decode_coefficients(symbol_i, vector_i); - - // The previous vector may still be in memory - std::fill_n(vector_i, SuperCoder::coefficients_length(), 0); - - // Stores the symbol and sets the pivot in the vector - store_uncoded_symbol(symbol_data, pivot_index); - - m_uncoded[pivot_index] = true; - - // No need to backwards substitute since we are - // replacing an existing symbol. I.e. backwards - // substitution must already have been done. - } - - /// Iterates the encoding vector from where a pivot has been - /// identified and subtracts existing symbols - /// @param symbol_data the data of the encoded symbol - /// @param symbol_id the data constituting the encoding vector - /// @param pivot_index the index of the found pivot element - void normalize(value_type *symbol_data, - value_type *symbol_id, - uint32_t pivot_index) - { - - assert(symbol_id != 0); - assert(symbol_data != 0); - - assert(pivot_index < SuperCoder::symbols()); - - assert(m_uncoded[pivot_index] == false); - assert(m_coded[pivot_index] == false); - - value_type coefficient = - fifi::get_value(symbol_id, pivot_index); - - assert(coefficient > 0); - - value_type inverted_coefficient = - SuperCoder::invert(coefficient); - - // Update symbol and corresponding vector - SuperCoder::multiply(symbol_id, inverted_coefficient, - SuperCoder::coefficients_length()); - - SuperCoder::multiply(symbol_data, inverted_coefficient, - SuperCoder::symbol_length()); - - } - - /// Iterates the encoding vector and subtracts existing symbols - /// until a pivot element is found. - /// @param symbol_data the data of the encoded symbol - /// @param symbol_id the data constituting the encoding vector - /// @return the pivot index if found. - boost::optional forward_substitute_to_pivot( - value_type *symbol_data, - value_type *symbol_id) - { - assert(symbol_id != 0); - assert(symbol_data != 0); - - for(uint32_t i = 0; i < SuperCoder::symbols(); ++i) - { - - value_type current_coefficient - = fifi::get_value(symbol_id, i); - - if( current_coefficient ) - { - // If symbol exists - if( symbol_pivot( i ) ) - { - value_type *vector_i = - SuperCoder::coefficients_value( i ); - - value_type *symbol_i = - SuperCoder::symbol_value( i ); - - if(fifi::is_binary::value) - { - SuperCoder::subtract( - symbol_id, vector_i, - SuperCoder::coefficients_length()); - - SuperCoder::subtract( - symbol_data, symbol_i, - SuperCoder::symbol_length()); - } - else - { - SuperCoder::multiply_subtract( - symbol_id, vector_i, - current_coefficient, - SuperCoder::coefficients_length()); - - SuperCoder::multiply_subtract( - symbol_data, symbol_i, - current_coefficient, - SuperCoder::symbol_length()); - } - } - else - { - return boost::optional( i ); - } - } - } - - return boost::none; - } - - /// Iterates the encoding vector from where a pivot has been - /// identified and subtracts existing symbols - /// @param symbol_data the data of the encoded symbol - /// @param symbol_id the data constituting the encoding vector - /// @param pivot_index the index of the found pivot element - void forward_substitute_from_pivot(value_type *symbol_data, - value_type *symbol_id, - uint32_t pivot_index) - { - // We have received an encoded symbol - described - // by the symbol group. We now normalize the - // the encoding vector according to the symbol id. - // I.e. we make sure the pivot position has a "1" - assert(symbol_id != 0); - assert(symbol_data != 0); - - assert(pivot_index < SuperCoder::symbols()); - - assert(m_uncoded[pivot_index] == false); - assert(m_coded[pivot_index] == false); - - // If this pivot index was smaller than the maximum pivot - // index we have, we might also need to backward - // substitute the higher pivot values into the new packet - for(uint32_t i = pivot_index + 1; i <= m_maximum_pivot; ++i) - { - // Do we have a non-zero value here? - - value_type value = - fifi::get_value(symbol_id, i); - - - if( !value ) - { - continue; - } - - if( symbol_pivot(i) ) - { - value_type *vector_i = - SuperCoder::coefficients_value(i); - - value_type *symbol_i = - SuperCoder::symbol_value(i); - - if(fifi::is_binary::value) - { - SuperCoder::subtract( - symbol_id, vector_i, - SuperCoder::coefficients_length()); - - SuperCoder::subtract( - symbol_data, symbol_i, - SuperCoder::symbol_length()); - } - else - { - SuperCoder::multiply_subtract( - symbol_id, vector_i, value, - SuperCoder::coefficients_length()); - - SuperCoder::multiply_subtract( - symbol_data, symbol_i, value, - SuperCoder::symbol_length()); - } - } - } - } - - /// Backward substitute the found symbol into the - /// existing symbols. - /// @param symbol_data buffer containing the encoding symbol - /// @param symbol_id buffer containing the encoding vector - /// @param pivot_index the pivot index of the symbol in the - /// buffers symbol_id and symbol_data - void backward_substitute(const value_type *symbol_data, - const value_type *symbol_id, - uint32_t pivot_index) - { - assert(symbol_id != 0); - assert(symbol_data != 0); - - assert(pivot_index < SuperCoder::symbols()); - - // We found a "1" that nobody else had as pivot, we now - // substract this packet from other coded packets - // - if they have a "1" on our pivot place - for(uint32_t i = 0; i <= m_maximum_pivot; ++i) - { - if( m_uncoded[i] ) - { - // We know that we have no non-zero elements - // outside the pivot position. - continue; - } - - if(i == pivot_index) - { - // We cannot backward substitute into ourself - continue; - } - - if( m_coded[i] ) - { - value_type *vector_i = - SuperCoder::coefficients_value(i); - - value_type value = - fifi::get_value( - vector_i, pivot_index); - - if( value ) - { - - value_type *symbol_i = - SuperCoder::symbol_value(i); - - if(fifi::is_binary::value) - { - SuperCoder::subtract( - vector_i, symbol_id, - SuperCoder::coefficients_length()); - - SuperCoder::subtract( - symbol_i, symbol_data, - SuperCoder::symbol_length()); - } - else - { - - // Update symbol and corresponding vector - SuperCoder::multiply_subtract( - vector_i, symbol_id, value, - SuperCoder::coefficients_length()); - - SuperCoder::multiply_subtract( - symbol_i, symbol_data, value, - SuperCoder::symbol_length()); - } - } - } - } - } - - /// Store an encoded symbol and encoding vector with the specified - /// pivot found. - /// @param symbol_data buffer containing the encoding symbol - /// @param symbol_coefficients buffer containing the symbol - /// coefficients - /// @param pivot_index the pivot index - void store_coded_symbol(const value_type *symbol_data, - const value_type *symbol_coefficients, - uint32_t pivot_index) - { - assert(m_uncoded[pivot_index] == false); - assert(m_coded[pivot_index] == false); - assert(symbol_coefficients != 0); - assert(symbol_data != 0); - assert(SuperCoder::is_symbol_available(pivot_index)); - - auto coefficient_storage = - sak::storage(symbol_coefficients, - SuperCoder::coefficients_size()); - - SuperCoder::set_coefficients( - pivot_index, coefficient_storage); - - // Copy it into the symbol storage - sak::mutable_storage dest = - sak::storage(SuperCoder::symbol(pivot_index), - SuperCoder::symbol_size()); - - sak::const_storage src = - sak::storage(symbol_data, SuperCoder::symbol_size()); - - sak::copy_storage(dest, src); - } - - /// Stores an uncoded or fully decoded symbol - /// @param symbol_data the data for the symbol - /// @param pivot_index the pivot index of the symbol - void store_uncoded_symbol(const value_type *symbol_data, - uint32_t pivot_index) - { - assert(symbol_data != 0); - assert(m_uncoded[pivot_index] == false); - assert(m_coded[pivot_index] == false); - assert(SuperCoder::is_symbol_available(pivot_index)); - - // Update the corresponding vector - value_type *vector_dest = - SuperCoder::coefficients_value( pivot_index ); - - // Zero out the memory first - std::fill_n(vector_dest, SuperCoder::coefficients_length(), 0); - - fifi::set_value(vector_dest, pivot_index, 1U); - - // Copy it into the symbol storage - sak::mutable_storage dest = - sak::storage(SuperCoder::symbol(pivot_index), - SuperCoder::symbol_size()); - - sak::const_storage src = - sak::storage(symbol_data, SuperCoder::symbol_size()); - - sak::copy_storage(dest, src); - - } - - protected: - - /// The current rank of the decoder - uint32_t m_rank; - - /// Stores the current maximum pivot index - uint32_t m_maximum_pivot; - - /// Tracks whether a symbol is contained which - /// is fully decoded - std::vector m_uncoded; - - /// Tracks whether a symbol is partially decoded - std::vector m_coded; - }; + class linear_block_decoder : public directional_linear_block_decoder< + forward_linear_block_decoder_policy, SuperCoder> + { }; } diff --git a/src/kodo/linear_block_decoder_delayed.hpp b/src/kodo/linear_block_decoder_delayed.hpp index da5cf7ed..cf99ec62 100644 --- a/src/kodo/linear_block_decoder_delayed.hpp +++ b/src/kodo/linear_block_decoder_delayed.hpp @@ -33,6 +33,9 @@ namespace kodo /// The value_type used to store the field elements typedef typename field_type::value_type value_type; + /// + typedef typename SuperCoder::direction_policy direction_policy; + public: /// @copydoc layer::decode_symbol(uint8_t*,uint8_t*) @@ -77,10 +80,8 @@ namespace kodo m_uncoded[ symbol_index ] = true; - if(symbol_index > m_maximum_pivot) - { - m_maximum_pivot = symbol_index; - } + m_maximum_pivot = + direction_policy::max(symbol_index, m_maximum_pivot); } @@ -136,10 +137,8 @@ namespace kodo m_coded[ *pivot_index ] = true; - if(*pivot_index > m_maximum_pivot) - { - m_maximum_pivot = *pivot_index; - } + m_maximum_pivot = + direction_policy::max(*pivot_index, m_maximum_pivot); if(SuperCoder::is_complete()) { @@ -156,10 +155,13 @@ namespace kodo { assert(SuperCoder::is_complete()); - uint32_t symbols = SuperCoder::symbols(); + uint32_t start = direction_policy::min(0, SuperCoder::symbols()-1); + uint32_t end = direction_policy::max(0, SuperCoder::symbols()-1); - for(uint32_t i = symbols; i --> 0;) + for(direction_policy p(start, end); !p.at_end(); p.advance()) { + uint32_t i = p.index(); + value_type *symbol_i = SuperCoder::symbol_value(i); diff --git a/test/src/test_backward_linear_block_decoder.cpp b/test/src/test_backward_linear_block_decoder.cpp new file mode 100644 index 00000000..7e525f9c --- /dev/null +++ b/test/src/test_backward_linear_block_decoder.cpp @@ -0,0 +1,167 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +/// @file test_forward_linear_block_decoder.cpp Unit tests for the +/// kodo::forward_linear_block_decoder + +#include +#include + +#include +#include + +#include + +#include "basic_api_test_helper.hpp" + +namespace kodo +{ + template + class test_forward_stack + : public // Payload API + // Codec Header API + // Symbol ID API + // Codec API + debug_linear_block_decoder< + linear_block_decoder< + // Coefficient Storage API + coefficient_storage< + coefficient_info< + // Storage API + deep_symbol_storage< + storage_bytes_used< + storage_block_info< + // Finite Field API + finite_field_math::type, + finite_field_info + > > > > > > > > > > + { }; + +} + + +/// Run the tests typical coefficients stack +TEST(TestBackwardLinearBlockDecoder, test_decoder) +{ + typedef fifi::binary field_type; + + kodo::test_forward_stack::factory f(8, 1600); + + auto d = f.build(); + + EXPECT_EQ(d->coefficients_size(), 1U); + + uint8_t coefficients[1]; + + // Create an encoding vector looking like this: 01000010 + coefficients[0] = 0; + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 6, 1U); + + // Create a dummy symbol + std::vector symbol = random_vector(d->symbol_size()); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); + + EXPECT_EQ(d->rank(), 1U); + EXPECT_TRUE(d->symbol_pivot(6)); + + // Create an encoding vector looking like this: 10000010 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 6, 1U); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); + + EXPECT_EQ(d->rank(), 2U); + EXPECT_TRUE(d->symbol_pivot(6)); + EXPECT_TRUE(d->symbol_pivot(1)); + + // Create an encoding vector looking like this: 11100010 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 6, 1U); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); + + EXPECT_EQ(d->rank(), 3U); + EXPECT_TRUE(d->symbol_pivot(6)); + EXPECT_TRUE(d->symbol_pivot(1)); + EXPECT_TRUE(d->symbol_pivot(2)); + + // Create an encoding vector looking like this: 11100010 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 3, 1U); + fifi::set_value(coefficients, 4, 1U); + fifi::set_value(coefficients, 5, 1U); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); + + // Create an encoding vector looking like this: 11100010 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 3, 1U); + fifi::set_value(coefficients, 4, 1U); + fifi::set_value(coefficients, 5, 1U); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); + + // Create an encoding vector looking like this: 11100010 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 3, 1U); + fifi::set_value(coefficients, 5, 1U); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); + + // Create an encoding vector looking like this: 11100010 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 3, 1U); + + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); + + + // Create an encoding vector looking like this: 11100010 + coefficients[0] = 0; + fifi::set_value(coefficients, 3, 1U); + fifi::set_value(coefficients, 4, 1U); + fifi::set_value(coefficients, 7, 1U); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); + + + + EXPECT_EQ(d->rank(), 3U); + EXPECT_TRUE(d->symbol_pivot(6)); + EXPECT_TRUE(d->symbol_pivot(1)); + EXPECT_TRUE(d->symbol_pivot(2)); + + +} + + + diff --git a/test/src/test_backward_linear_block_decoder_policy.cpp b/test/src/test_backward_linear_block_decoder_policy.cpp new file mode 100644 index 00000000..556e3906 --- /dev/null +++ b/test/src/test_backward_linear_block_decoder_policy.cpp @@ -0,0 +1,56 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +/// @file test_backward_linear_block_decoder_policy.cpp Unit test +/// for the backward_linear_block_decoder_policy + +#include +#include + +#include + + +/// Run the tests typical coefficients stack +TEST(TestBackwardLinearBlockDecoderPolicy, test_functions) +{ + + { + EXPECT_EQ(kodo::backward_linear_block_decoder_policy::max(1,2), 1); + EXPECT_EQ(kodo::backward_linear_block_decoder_policy::min(1,2), 2); + + uint32_t start = kodo::backward_linear_block_decoder_policy::min(0,5); + uint32_t stop = kodo::backward_linear_block_decoder_policy::max(0,5); + + kodo::backward_linear_block_decoder_policy policy(start, stop); + EXPECT_EQ(policy.at_end(), false); + EXPECT_EQ(policy.index(), 5U); + policy.advance(); + + EXPECT_EQ(policy.at_end(), false); + EXPECT_EQ(policy.index(), 4U); + policy.advance(); + + EXPECT_EQ(policy.at_end(), false); + EXPECT_EQ(policy.index(), 3U); + policy.advance(); + + EXPECT_EQ(policy.at_end(), false); + EXPECT_EQ(policy.index(), 2U); + policy.advance(); + + EXPECT_EQ(policy.at_end(), false); + EXPECT_EQ(policy.index(), 1U); + policy.advance(); + + EXPECT_EQ(policy.at_end(), false); + EXPECT_EQ(policy.index(), 0U); + policy.advance(); + + EXPECT_EQ(policy.at_end(), true); + } +} + + + diff --git a/test/src/test_forward_linear_block_decoder.cpp b/test/src/test_forward_linear_block_decoder.cpp index 29316996..6e5ffe34 100644 --- a/test/src/test_forward_linear_block_decoder.cpp +++ b/test/src/test_forward_linear_block_decoder.cpp @@ -12,6 +12,8 @@ #include #include +#include + #include "basic_api_test_helper.hpp" namespace kodo @@ -22,7 +24,8 @@ namespace kodo // Codec Header API // Symbol ID API // Codec API - forward_linear_block_decoder< + debug_linear_block_decoder< + linear_block_decoder< // Coefficient Storage API coefficient_storage< coefficient_info< @@ -37,7 +40,7 @@ namespace kodo final_coder_factory_pool< // Final type test_forward_stack - > > > > > > > > > + > > > > > > > > > > { }; } @@ -46,8 +49,9 @@ namespace kodo /// Run the tests typical coefficients stack TEST(TestForwardLinearBlockDecoder, test_decoder) { + typedef fifi::binary field_type; - kodo::test_forward_stack::factory f(8, 1600); + kodo::test_forward_stack::factory f(8, 1600); auto d = f.build(); @@ -57,41 +61,104 @@ TEST(TestForwardLinearBlockDecoder, test_decoder) // Create an encoding vector looking like this: 01000010 coefficients[0] = 0; - fifi::set_value(coefficients, 1, 1U); - fifi::set_value(coefficients, 6, 1U); + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 6, 1U); // Create a dummy symbol std::vector symbol = random_vector(d->symbol_size()); d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); EXPECT_EQ(d->rank(), 1U); + EXPECT_TRUE(d->symbol_pivot(6)); + + // Create an encoding vector looking like this: 10000010 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 6, 1U); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); + + EXPECT_EQ(d->rank(), 2U); + EXPECT_TRUE(d->symbol_pivot(6)); + EXPECT_TRUE(d->symbol_pivot(1)); + + // Create an encoding vector looking like this: 11100010 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 6, 1U); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); + + EXPECT_EQ(d->rank(), 3U); + EXPECT_TRUE(d->symbol_pivot(6)); EXPECT_TRUE(d->symbol_pivot(1)); + EXPECT_TRUE(d->symbol_pivot(2)); + + // Create an encoding vector looking like this: 11100010 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 3, 1U); + fifi::set_value(coefficients, 4, 1U); + fifi::set_value(coefficients, 5, 1U); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); - // // Create an encoding vector looking like this: 10000010 - // coefficients[0] = 0; - // fifi::set_value(coefficients, 0, 1U); - // fifi::set_value(coefficients, 6, 1U); + // Create an encoding vector looking like this: 11100010 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 3, 1U); + fifi::set_value(coefficients, 4, 1U); + fifi::set_value(coefficients, 5, 1U); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); + + // Create an encoding vector looking like this: 11100010 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 3, 1U); + fifi::set_value(coefficients, 5, 1U); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); + + // Create an encoding vector looking like this: 11100010 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 3, 1U); + + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); - // d->decode_symbol(&symbol[0], coefficients); - // EXPECT_EQ(d->rank(), 2U); - // EXPECT_TRUE(d->symbol_pivot(0)); - // EXPECT_TRUE(d->symbol_pivot(1)); + // Create an encoding vector looking like this: 11100010 + coefficients[0] = 0; + fifi::set_value(coefficients, 3, 1U); + fifi::set_value(coefficients, 4, 1U); + fifi::set_value(coefficients, 7, 1U); + + d->decode_symbol(&symbol[0], coefficients); + d->print_decoder_state(std::cout); - // // Create an encoding vector looking like this: 11100010 - // coefficients[0] = 0; - // fifi::set_value(coefficients, 0, 1U); - // fifi::set_value(coefficients, 1, 1U); - // fifi::set_value(coefficients, 2, 1U); - // fifi::set_value(coefficients, 6, 1U); - // d->decode_symbol(&symbol[0], coefficients); - // EXPECT_EQ(d->rank(), 3U); - // EXPECT_TRUE(d->symbol_pivot(0)); - // EXPECT_TRUE(d->symbol_pivot(1)); - // EXPECT_TRUE(d->symbol_pivot(2)); + EXPECT_EQ(d->rank(), 3U); + EXPECT_TRUE(d->symbol_pivot(6)); + EXPECT_TRUE(d->symbol_pivot(1)); + EXPECT_TRUE(d->symbol_pivot(2)); } From 9fcbbec59abae811a8ec4221c1d2415e95191e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Hundeb=C3=B8ll?= Date: Mon, 8 Jul 2013 10:25:57 +0200 Subject: [PATCH 32/83] kodo: support partial decoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To allow decoders to extract decoded symbols before all symbols has been received, the decoding layers needs to keep track of the maximum coefficient index. This patch adds a member variable to hold the maximum index, which is updated whenever a new symbol with more none-zero coefficients is added. To allow users to detect when decoded symbols are available, a public function is added to test wether the block has rank equal to the maximum coefficient index. Signed-off-by: Martin Hundebøll Signed-off-to: Steinwurf ApS --- src/kodo/linear_block_decoder.hpp | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/kodo/linear_block_decoder.hpp b/src/kodo/linear_block_decoder.hpp index d34c524c..75f283f0 100644 --- a/src/kodo/linear_block_decoder.hpp +++ b/src/kodo/linear_block_decoder.hpp @@ -46,7 +46,8 @@ namespace kodo /// Constructor linear_block_decoder() : m_rank(0), - m_maximum_pivot(0) + m_maximum_pivot(0), + m_maximum_coefficient(0) { } /// @copydoc layer::construct(Factory&) @@ -70,6 +71,7 @@ namespace kodo m_rank = 0; m_maximum_pivot = 0; + m_maximum_coefficient = 0; } /// @copydoc layer::decode_symbol(uint8_t*,uint8_t*) @@ -85,6 +87,14 @@ namespace kodo value_type *coefficients = reinterpret_cast(symbol_coefficients); + for (size_t i = 0; i < this->symbols(); i++) { + value_type current_coefficient + = fifi::get_value(coefficients, i); + + if (current_coefficient && i > m_maximum_coefficient) + m_maximum_coefficient = i; + } + decode_coefficients(symbol, coefficients); } @@ -100,6 +110,9 @@ namespace kodo return; } + if (symbol_index > m_maximum_coefficient) + m_maximum_coefficient = symbol_index; + const value_type *symbol = reinterpret_cast( symbol_data ); @@ -139,12 +152,22 @@ namespace kodo return m_rank == SuperCoder::symbols(); } + bool is_partial_complete() const + { + return m_rank == m_maximum_coefficient + 1; + } + /// @copydoc layer::rank() const uint32_t rank() const { return m_rank; } + uint32_t max_coefficient() const + { + return m_maximum_coefficient; + } + /// @copydoc layer::symbol_pivot(uint32_t) const bool symbol_pivot(uint32_t index) const { @@ -572,6 +595,8 @@ namespace kodo /// Stores the current maximum pivot index uint32_t m_maximum_pivot; + uint32_t m_maximum_coefficient; + /// Tracks whether a symbol is contained which /// is fully decoded std::vector m_uncoded; From ad8495079e866ce09875349fdd929d76d1ac23d7 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Mon, 8 Jul 2013 13:44:55 +0200 Subject: [PATCH 33/83] Working on the partial decoding state variables --- src/kodo/linear_block_decoder.hpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/kodo/linear_block_decoder.hpp b/src/kodo/linear_block_decoder.hpp index 75f283f0..aaf93f02 100644 --- a/src/kodo/linear_block_decoder.hpp +++ b/src/kodo/linear_block_decoder.hpp @@ -96,6 +96,15 @@ namespace kodo } decode_coefficients(symbol, coefficients); + + if((m_maximum_pivot + 1) == m_rank) + { + std::cout << "Rank " << m_rank << std::endl; + std::cout << "Max pivot " << m_maximum_pivot << std::endl; + std::cout << "Max coeff. " << m_maximum_coefficient << std::endl; + + //assert(m_maximum_pivot == m_maximum_coefficient); + } } /// @copydoc layer::decode_symbol(uint8_t*, uint32_t) From ba17c4af249b0b8c8ba1425535927a9f3aa7bebf Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Mon, 8 Jul 2013 16:16:09 +0200 Subject: [PATCH 34/83] Intial commit of the partial decoding tracker --- src/kodo/linear_block_decoder.hpp | 36 +--- src/kodo/partial_decoding_tracker.hpp | 167 ++++++++++++++++++ test/src/test_linear_block_decoder.cpp | 191 +++++++++++++++++++++ test/src/test_partial_decoding_tracker.cpp | 128 ++++++++++++++ 4 files changed, 487 insertions(+), 35 deletions(-) create mode 100644 src/kodo/partial_decoding_tracker.hpp create mode 100644 test/src/test_linear_block_decoder.cpp create mode 100644 test/src/test_partial_decoding_tracker.cpp diff --git a/src/kodo/linear_block_decoder.hpp b/src/kodo/linear_block_decoder.hpp index aaf93f02..d34c524c 100644 --- a/src/kodo/linear_block_decoder.hpp +++ b/src/kodo/linear_block_decoder.hpp @@ -46,8 +46,7 @@ namespace kodo /// Constructor linear_block_decoder() : m_rank(0), - m_maximum_pivot(0), - m_maximum_coefficient(0) + m_maximum_pivot(0) { } /// @copydoc layer::construct(Factory&) @@ -71,7 +70,6 @@ namespace kodo m_rank = 0; m_maximum_pivot = 0; - m_maximum_coefficient = 0; } /// @copydoc layer::decode_symbol(uint8_t*,uint8_t*) @@ -87,24 +85,7 @@ namespace kodo value_type *coefficients = reinterpret_cast(symbol_coefficients); - for (size_t i = 0; i < this->symbols(); i++) { - value_type current_coefficient - = fifi::get_value(coefficients, i); - - if (current_coefficient && i > m_maximum_coefficient) - m_maximum_coefficient = i; - } - decode_coefficients(symbol, coefficients); - - if((m_maximum_pivot + 1) == m_rank) - { - std::cout << "Rank " << m_rank << std::endl; - std::cout << "Max pivot " << m_maximum_pivot << std::endl; - std::cout << "Max coeff. " << m_maximum_coefficient << std::endl; - - //assert(m_maximum_pivot == m_maximum_coefficient); - } } /// @copydoc layer::decode_symbol(uint8_t*, uint32_t) @@ -119,9 +100,6 @@ namespace kodo return; } - if (symbol_index > m_maximum_coefficient) - m_maximum_coefficient = symbol_index; - const value_type *symbol = reinterpret_cast( symbol_data ); @@ -161,22 +139,12 @@ namespace kodo return m_rank == SuperCoder::symbols(); } - bool is_partial_complete() const - { - return m_rank == m_maximum_coefficient + 1; - } - /// @copydoc layer::rank() const uint32_t rank() const { return m_rank; } - uint32_t max_coefficient() const - { - return m_maximum_coefficient; - } - /// @copydoc layer::symbol_pivot(uint32_t) const bool symbol_pivot(uint32_t index) const { @@ -604,8 +572,6 @@ namespace kodo /// Stores the current maximum pivot index uint32_t m_maximum_pivot; - uint32_t m_maximum_coefficient; - /// Tracks whether a symbol is contained which /// is fully decoded std::vector m_uncoded; diff --git a/src/kodo/partial_decoding_tracker.hpp b/src/kodo/partial_decoding_tracker.hpp new file mode 100644 index 00000000..dd9867f7 --- /dev/null +++ b/src/kodo/partial_decoding_tracker.hpp @@ -0,0 +1,167 @@ +// Copyright Steinwurf ApS 2011-2012. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +#pragma once + +#include + +#include +#include +#include +#include + +#include + +#include +#include + +namespace kodo +{ + + /// @ingroup codec_layers + /// @brief This layer can be added to a decoding stack to track + /// when partial decoding is possible i.e. when symbols in + /// decoder are fully decoded even though the full data + /// block has not been sent. + /// + /// Keeping track when partial decoding is possible can be done + /// in several ways. This layer will "monitor" the incoming + /// coding coefficients, using the information stored in the + /// it is possible to detect cases where all symbols are fully + /// decoded. + /// + /// Imagine we have observed the following two coding vectors: + /// + /// 1: 1 0 0 0 + /// 2: 0 1 1 0 + /// ^ + /// | + /// +----+ Last non-zero position + /// + /// This matrix has a rank of 2 but since the last non-zero coefficient + /// is found at index 2 we cannot yet decode (packet two is still coded). + /// + /// However if we receive on more packet such that the matrix becomes e.g. + /// + /// 1: 1 0 0 0 + /// 2: 0 1 1 0 + /// 3: 1 0 1 0 + /// ^ + /// | + /// +----+ Last non-zero position + /// + /// We see that the matrix has rank 3 and the last non-zero coefficient is + /// still at index 2. We therefore must be able to decode the three packets. + /// + /// One word of caution, this does of course require that the decoder used + /// will always decode all packets to the degree possible. Some decoders + /// such as the linear_block_decoder_delayed will not perform backward + /// substitution until the matrix reaches full rank. That type of decoder + /// are therefore not compatible with this layer. + template + class partial_decoding_tracker : public SuperCoder + { + public: + + /// @copydoc layer::field_type + typedef typename SuperCoder::field_type field_type; + + /// @copydoc layer::value_type + typedef typename field_type::value_type value_type; + + /// @copydoc layer::factory + typedef typename SuperCoder::factory factory; + + public: + + /// Constructor + partial_decoding_tracker() + : m_largest_nonzero_index(0) + { } + + /// @copydoc layer::initialize(Factory&) + template + void initialize(Factory& the_factory) + { + SuperCoder::initialize(the_factory); + m_largest_nonzero_index = 0; + } + + /// @copydoc layer::decode_symbol(uint8_t*,uint8_t*) + void decode_symbol(uint8_t *symbol_data, + uint8_t *symbol_coefficients) + { + assert(symbol_data != 0); + assert(symbol_coefficients != 0); + + // Inspect the coding coefficients to find the maximum + // position where we have a non-zero value + value_type *coefficients + = reinterpret_cast(symbol_coefficients); + + // We inspect the coefficients from the back i.e. if we + // are given the following coding vector: + // + // 23 0 44 213 0 231 0 + // ^ + // | + // +-----+ max position of non-zero + // coefficient + // + // Starting from the back will in many cases e.g. for uniform + // coding vectors be better, since we are looking for the last + // non-zero element. + assert(SuperCoder::symbols() > m_largest_nonzero_index); + + for (uint32_t i = SuperCoder::symbols(); + i --> m_largest_nonzero_index;) + { + value_type current_coefficient + = fifi::get_value(coefficients, i); + + if (current_coefficient) + { + m_largest_nonzero_index = i; + break; + } + } + + SuperCoder::decode_symbol(symbol_data, symbol_coefficients); + } + + /// @copydoc layer::decode_symbol(uint8_t*, uint32_t) + void decode_symbol(uint8_t *symbol_data, + uint32_t symbol_index) + { + assert(symbol_index < SuperCoder::symbols()); + assert(symbol_data != 0); + + m_largest_nonzero_index = + std::max(symbol_index, m_largest_nonzero_index); + + SuperCoder::decode_symbol(symbol_data, symbol_index); + } + + /// @return True if the matrix should be partially decoded. + bool is_partial_complete() const + { + return SuperCoder::rank() == m_largest_nonzero_index + 1; + } + + /// @return The position of the last seen non-zero coefficients + uint32_t largest_nonzero_index() const + { + return m_largest_nonzero_index; + } + + protected: + + /// The largest index seen with a nonzero coefficient + uint32_t m_largest_nonzero_index; + + }; + +} + diff --git a/test/src/test_linear_block_decoder.cpp b/test/src/test_linear_block_decoder.cpp new file mode 100644 index 00000000..39912440 --- /dev/null +++ b/test/src/test_linear_block_decoder.cpp @@ -0,0 +1,191 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +/// @file test_linear_block_decoder.cpp Unit tests for the +/// kodo::linear_block_decoder + +#include +#include + +#include +#include + +#include + +#include "basic_api_test_helper.hpp" + +namespace kodo +{ + template + class test_linear_stack + : public // Payload API + // Codec Header API + // Symbol ID API + // Codec API + debug_linear_block_decoder< + linear_block_decoder< + // Coefficient Storage API + coefficient_storage< + coefficient_info< + // Storage API + deep_symbol_storage< + storage_bytes_used< + storage_block_info< + // Finite Field API + finite_field_math::type, + finite_field_info + > > > > > > > > > > + { }; + +} + + +/// Run the tests typical coefficients stack +TEST(TestLinearBlockDecoder, test_decoder) +{ + typedef fifi::binary field_type; + + kodo::test_linear_stack::factory f(5, 1600); + + auto d = f.build(); + + EXPECT_EQ(d->coefficients_size(), 1U); + + uint8_t coefficients[1]; + + // Create an encoding vector looking like this: 01100 + coefficients[0] = 0; + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + + // Create a dummy symbol + std::vector symbol = random_vector(d->symbol_size()); + + d->decode_symbol(&symbol[0], coefficients); + // d->print_decoder_state(std::cout); + + EXPECT_EQ(d->rank(), 1U); + + EXPECT_TRUE(d->symbol_pivot(1U)); + + EXPECT_TRUE(d->symbol_coded(1U)); + + EXPECT_FALSE(d->is_complete()); + + // Create an encoding vector looking like this: 11000 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 1, 1U); + + d->decode_symbol(&symbol[0], coefficients); + // d->print_decoder_state(std::cout); + + EXPECT_EQ(d->rank(), 2U); + + EXPECT_TRUE(d->symbol_pivot(0)); + EXPECT_TRUE(d->symbol_pivot(1)); + + EXPECT_TRUE(d->symbol_coded(0)); + EXPECT_TRUE(d->symbol_coded(1)); + + EXPECT_FALSE(d->is_complete()); + + // Create an systematic symbol for position 2 + d->decode_symbol(&symbol[0], 2U); + // d->print_decoder_state(std::cout); + + EXPECT_EQ(d->rank(), 3U); + + EXPECT_TRUE(d->symbol_pivot(0)); + EXPECT_TRUE(d->symbol_pivot(1)); + EXPECT_TRUE(d->symbol_pivot(2)); + + EXPECT_TRUE(d->symbol_coded(0)); + EXPECT_TRUE(d->symbol_coded(1)); + EXPECT_FALSE(d->symbol_coded(2)); + + EXPECT_FALSE(d->is_complete()); + + // Create an encoding vector looking like this: 11101 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 4, 1U); + + d->decode_symbol(&symbol[0], coefficients); + // d->print_decoder_state(std::cout); + + EXPECT_EQ(d->rank(), 4U); + EXPECT_TRUE(d->symbol_pivot(0)); + EXPECT_TRUE(d->symbol_pivot(1)); + EXPECT_TRUE(d->symbol_pivot(2)); + EXPECT_TRUE(d->symbol_pivot(4)); + + EXPECT_TRUE(d->symbol_coded(0)); + EXPECT_TRUE(d->symbol_coded(1)); + EXPECT_FALSE(d->symbol_coded(2)); + EXPECT_TRUE(d->symbol_coded(4)); + + EXPECT_FALSE(d->is_complete()); + + // Create a _linear dependent_ encoding vector looking like this: 11101 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 4, 1U); + + d->decode_symbol(&symbol[0], coefficients); + // d->print_decoder_state(std::cout); + + EXPECT_EQ(d->rank(), 4U); + + EXPECT_TRUE(d->symbol_pivot(0)); + EXPECT_TRUE(d->symbol_pivot(1)); + EXPECT_TRUE(d->symbol_pivot(2)); + EXPECT_TRUE(d->symbol_pivot(4)); + + EXPECT_TRUE(d->symbol_coded(0)); + EXPECT_TRUE(d->symbol_coded(1)); + EXPECT_FALSE(d->symbol_coded(2)); + EXPECT_TRUE(d->symbol_coded(4)); + + EXPECT_FALSE(d->is_complete()); + + // Create an encoding vector looking like this: 01110 + coefficients[0] = 0; + + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 3, 1U); + + d->decode_symbol(&symbol[0], coefficients); + // d->print_decoder_state(std::cout); + + EXPECT_EQ(d->rank(), 5U); + + EXPECT_TRUE(d->symbol_pivot(0)); + EXPECT_TRUE(d->symbol_pivot(1)); + EXPECT_TRUE(d->symbol_pivot(2)); + EXPECT_TRUE(d->symbol_pivot(3)); + EXPECT_TRUE(d->symbol_pivot(4)); + + EXPECT_TRUE(d->symbol_coded(0)); + EXPECT_TRUE(d->symbol_coded(1)); + EXPECT_FALSE(d->symbol_coded(2)); + EXPECT_TRUE(d->symbol_coded(3)); + EXPECT_TRUE(d->symbol_coded(4)); + + EXPECT_TRUE(d->is_complete()); + +} + + + diff --git a/test/src/test_partial_decoding_tracker.cpp b/test/src/test_partial_decoding_tracker.cpp new file mode 100644 index 00000000..448d2a62 --- /dev/null +++ b/test/src/test_partial_decoding_tracker.cpp @@ -0,0 +1,128 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +/// @file test_partial_decoding_tracker.cpp Unit tests for the +/// kodo::partial_decoding_tracker + +#include +#include + +#include +#include +#include + +#include + +#include "basic_api_test_helper.hpp" + +namespace kodo +{ + template + class test_partial_stack + : public // Payload API + // Codec Header API + // Symbol ID API + // Codec API + partial_decoding_tracker< + linear_block_decoder< + // Coefficient Storage API + coefficient_storage< + coefficient_info< + // Storage API + deep_symbol_storage< + storage_bytes_used< + storage_block_info< + // Finite Field API + finite_field_math::type, + finite_field_info + > > > > > > > > > > + { }; + +} + +/// Run the tests typical coefficients stack +TEST(TestPartialDecodingTracker, test_tracker) +{ + typedef fifi::binary field_type; + + kodo::test_partial_stack::factory f(5, 1600); + + auto d = f.build(); + + EXPECT_EQ(d->coefficients_size(), 1U); + + uint8_t coefficients[1]; + + // Create an encoding vector looking like this: 01100 + coefficients[0] = 0; + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + + // Create a dummy symbol + std::vector symbol = random_vector(d->symbol_size()); + + d->decode_symbol(&symbol[0], coefficients); + + EXPECT_EQ(d->rank(), 1U); + EXPECT_EQ(d->largest_nonzero_index(), 2U); + EXPECT_FALSE(d->is_partial_complete()); + + // Create an encoding vector looking like this: 11000 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 1, 1U); + + d->decode_symbol(&symbol[0], coefficients); + + EXPECT_EQ(d->rank(), 2U); + EXPECT_EQ(d->largest_nonzero_index(), 2U); + EXPECT_FALSE(d->is_partial_complete()); + + // Create an encoding vector looking like this: 11100 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + + d->decode_symbol(&symbol[0], coefficients); + + EXPECT_EQ(d->rank(), 3U); + EXPECT_EQ(d->largest_nonzero_index(), 2U); + EXPECT_TRUE(d->is_partial_complete()); + + // Create an encoding vector looking like this: 11101 + coefficients[0] = 0; + fifi::set_value(coefficients, 0, 1U); + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 4, 1U); + + d->decode_symbol(&symbol[0], coefficients); + + EXPECT_EQ(d->rank(), 4U); + EXPECT_EQ(d->largest_nonzero_index(), 4U); + EXPECT_FALSE(d->is_partial_complete()); + + // Create an encoding vector looking like this: 01110 + coefficients[0] = 0; + + fifi::set_value(coefficients, 1, 1U); + fifi::set_value(coefficients, 2, 1U); + fifi::set_value(coefficients, 3, 1U); + + d->decode_symbol(&symbol[0], coefficients); + + EXPECT_EQ(d->rank(), 5U); + EXPECT_EQ(d->largest_nonzero_index(), 4U); + EXPECT_TRUE(d->is_partial_complete()); + +} + + + From 7330b3b0d9af3d888bced56c8d84708b79f825c8 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Mon, 8 Jul 2013 17:15:00 +0200 Subject: [PATCH 35/83] Adding news --- NEWS.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index 4ddcb4d9..7504b36a 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -6,7 +6,12 @@ of every change, see the Git log. Latest ------ -* tbd +* Minor: Added the partial_decoding_tracker layer which "monitors" the + coding vectors being passed to a decoder in order to detect early + decoding opportunities. This means that although not all packets have + yet been sent from the encoder, it might happen that we can decode + anyway. This kind of functionality is useful especially for applications + which require low delay. 11.1.0 ------ From 76195ae1e65f9ee9f2cf227ae3e50f2a3205a296 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Wed, 17 Jul 2013 17:06:31 +0200 Subject: [PATCH 36/83] Working on the revers linear block decoder --- benchmark/throughput/codes.hpp | 8 +- src/kodo/backward_linear_block_decoder.hpp | 7 +- src/kodo/forward_linear_block_decoder.hpp | 612 ------------------ src/kodo/linear_block_decoder.hpp | 1 + .../src/test_forward_linear_block_decoder.cpp | 167 ----- 5 files changed, 10 insertions(+), 785 deletions(-) delete mode 100644 src/kodo/forward_linear_block_decoder.hpp delete mode 100644 test/src/test_forward_linear_block_decoder.cpp diff --git a/benchmark/throughput/codes.hpp b/benchmark/throughput/codes.hpp index abb2fa4d..700a96ed 100644 --- a/benchmark/throughput/codes.hpp +++ b/benchmark/throughput/codes.hpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include namespace kodo @@ -86,7 +86,7 @@ namespace kodo /// RLNC decoder which uses the policy based linear block decoder template - class forward_full_rlnc_decoder + class backward_full_rlnc_decoder : public // Payload API payload_recoder + backward_full_rlnc_decoder > > > > > > > > > > > > > > > { }; diff --git a/src/kodo/backward_linear_block_decoder.hpp b/src/kodo/backward_linear_block_decoder.hpp index 8657faa6..5a11e0d4 100644 --- a/src/kodo/backward_linear_block_decoder.hpp +++ b/src/kodo/backward_linear_block_decoder.hpp @@ -22,6 +22,8 @@ namespace kodo { + + /// @todo document this /// @ingroup codec_layers /// @brief Implements basic linear block decoder. /// @@ -30,8 +32,9 @@ namespace kodo /// coefficients. Using these coefficients the block decoder subtracts /// incoming symbols until the original data has been recreated. template - class backward_linear_block_decoder : public directional_linear_block_decoder< - backward_linear_block_decoder_policy, SuperCoder> + class backward_linear_block_decoder : + public directional_linear_block_decoder + { }; } diff --git a/src/kodo/forward_linear_block_decoder.hpp b/src/kodo/forward_linear_block_decoder.hpp deleted file mode 100644 index 032b3860..00000000 --- a/src/kodo/forward_linear_block_decoder.hpp +++ /dev/null @@ -1,612 +0,0 @@ -// Copyright Steinwurf ApS 2011-2012. -// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". -// See accompanying file LICENSE.rst or -// http://www.steinwurf.com/licensing - -#pragma once - -#include - -#include -#include -#include -#include - -#include - -#include -#include - -#include -#include - -namespace kodo -{ - /// @ingroup codec_layers - /// @brief Implements basic linear block decoder. - /// - /// The linear block decoder - /// expects that an encoded symbol is described by a vector of - /// coefficients. Using these coefficients the block decoder subtracts - /// incoming symbols until the original data has been recreated. - template - class forward_linear_block_decoder : public SuperCoder - { - public: - - /// @copydoc layer::field_type - typedef typename SuperCoder::field_type field_type; - - /// @copydoc layer::value_type - typedef typename field_type::value_type value_type; - - /// @copydoc layer::factory - typedef typename SuperCoder::factory factory; - - typedef backward_linear_block_decoder_policy the_policy; - - public: - - /// Constructor - forward_linear_block_decoder() - : m_rank(0), - m_maximum_pivot(0) - { } - - /// @copydoc layer::construct(Factory&) - template - void construct(Factory &the_factory) - { - SuperCoder::construct(the_factory); - - m_uncoded.resize(the_factory.max_symbols(), false); - m_coded.resize(the_factory.max_symbols(), false); - } - - /// @copydoc layer::initialize(Factory&) - template - void initialize(Factory& the_factory) - { - SuperCoder::initialize(the_factory); - - std::fill_n(m_uncoded.begin(), the_factory.symbols(), false); - std::fill_n(m_coded.begin(), the_factory.symbols(), false); - - m_rank = 0; - - // Depending on the policy we either go from 0 to symbols or - // from symbols to 0. - m_maximum_pivot = the_policy::min(0, the_factory.symbols()-1); - } - - /// @copydoc layer::decode_symbol(uint8_t*,uint8_t*) - void decode_symbol(uint8_t *symbol_data, - uint8_t *symbol_coefficients) - { - assert(symbol_data != 0); - assert(symbol_coefficients != 0); - - value_type *symbol - = reinterpret_cast(symbol_data); - - value_type *coefficients - = reinterpret_cast(symbol_coefficients); - - decode_coefficients(symbol, coefficients); - } - - /// @copydoc layer::decode_symbol(uint8_t*, uint32_t) - void decode_symbol(uint8_t *symbol_data, - uint32_t symbol_index) - { - assert(symbol_index < SuperCoder::symbols()); - assert(symbol_data != 0); - - if(m_uncoded[symbol_index]) - { - return; - } - - const value_type *symbol - = reinterpret_cast( symbol_data ); - - if(m_coded[symbol_index]) - { - swap_decode(symbol, symbol_index); - } - else - { - // Stores the symbol and updates the corresponding - // encoding vector - store_uncoded_symbol(symbol, symbol_index); - - // Backwards substitution - value_type *coefficients = - SuperCoder::coefficients_value(symbol_index); - - backward_substitute(symbol, coefficients, symbol_index); - - // We have increased the rank if we have finished the - // backwards substitution - ++m_rank; - - m_uncoded[ symbol_index ] = true; - - m_maximum_pivot = - the_policy::max(symbol_index, m_maximum_pivot); - - } - } - - /// @copydoc layer::is_complete() const - bool is_complete() const - { - return m_rank == SuperCoder::symbols(); - } - - /// @copydoc layer::rank() const - uint32_t rank() const - { - return m_rank; - } - - /// @copydoc layer::symbol_pivot(uint32_t) const - bool symbol_pivot(uint32_t index) const - { - assert(index < SuperCoder::symbols()); - return m_coded[index] || m_uncoded[index]; - } - - /// @todo Add unit test - /// @copydoc layer::symbol_pivot(uint32_t) const - bool symbol_coded(uint32_t index) const - { - assert(symbol_pivot(index)); - return m_coded[index]; - } - - protected: - - /// Decodes a symbol based on the coefficients - /// @param symbol_data buffer containing the encoding symbol - /// @param symbol_id buffer containing the encoding vector - void decode_coefficients(value_type *symbol_data, - value_type *symbol_coefficients) - { - assert(symbol_data != 0); - assert(symbol_coefficients != 0); - - // See if we can find a pivot - auto pivot_index - = forward_substitute_to_pivot( - symbol_data, symbol_coefficients); - - if(!pivot_index) - return; - - if(!fifi::is_binary::value) - { - // Normalize symbol and vector - normalize( - symbol_data,symbol_coefficients,*pivot_index); - } - - // Reduce the symbol further - forward_substitute_from_pivot( - symbol_data, symbol_coefficients, *pivot_index); - - // Now with the found pivot reduce the existing symbols - backward_substitute( - symbol_data, symbol_coefficients, *pivot_index); - - // Now save the received symbol - store_coded_symbol( - symbol_data, symbol_coefficients, *pivot_index); - - // We have increased the rank - ++m_rank; - - m_coded[ *pivot_index ] = true; - - if(*pivot_index > m_maximum_pivot) - { - m_maximum_pivot = *pivot_index; - } - } - - /// When adding a raw symbol (i.e. uncoded) with a specific - /// pivot id and the decoder already contains a coded symbol - /// in that position this function performs the proper swap - /// between the two symbols. - /// @param symbol_data the data for the raw symbol - /// @param pivot_index the pivot position of the raw symbol - void swap_decode(const value_type *symbol_data, - uint32_t pivot_index) - { - assert(m_coded[pivot_index] == true); - assert(m_uncoded[pivot_index] == false); - - m_coded[pivot_index] = false; - - value_type *symbol_i = - SuperCoder::symbol_value(pivot_index); - - value_type *vector_i = - SuperCoder::coefficients_value(pivot_index); - - value_type value = - fifi::get_value(vector_i, pivot_index); - - assert(value == 1); - - // Subtract the new pivot symbol - fifi::set_value(vector_i, pivot_index, 0); - - SuperCoder::subtract(symbol_i, symbol_data, - SuperCoder::symbol_length()); - - // Now continue our new coded symbol we know that it must - // if found it will contain a pivot id > that the current. - decode_coefficients(symbol_i, vector_i); - - // The previous vector may still be in memory - std::fill_n(vector_i, SuperCoder::coefficients_length(), 0); - - // Stores the symbol and sets the pivot in the vector - store_uncoded_symbol(symbol_data, pivot_index); - - m_uncoded[pivot_index] = true; - - // No need to backwards substitute since we are - // replacing an existing symbol. I.e. backwards - // substitution must already have been done. - } - - /// Iterates the encoding vector from where a pivot has been - /// identified and subtracts existing symbols - /// @param symbol_data the data of the encoded symbol - /// @param symbol_id the data constituting the encoding vector - /// @param pivot_index the index of the found pivot element - void normalize(value_type *symbol_data, - value_type *symbol_id, - uint32_t pivot_index) - { - - assert(symbol_id != 0); - assert(symbol_data != 0); - - assert(pivot_index < SuperCoder::symbols()); - - assert(m_uncoded[pivot_index] == false); - assert(m_coded[pivot_index] == false); - - value_type coefficient = - fifi::get_value(symbol_id, pivot_index); - - assert(coefficient > 0); - - value_type inverted_coefficient = - SuperCoder::invert(coefficient); - - // Update symbol and corresponding vector - SuperCoder::multiply(symbol_id, inverted_coefficient, - SuperCoder::coefficients_length()); - - SuperCoder::multiply(symbol_data, inverted_coefficient, - SuperCoder::symbol_length()); - - } - - /// Iterates the encoding vector and subtracts existing symbols - /// until a pivot element is found. - /// @param symbol_data the data of the encoded symbol - /// @param symbol_id the data constituting the encoding vector - /// @return the pivot index if found. - boost::optional forward_substitute_to_pivot( - value_type *symbol_data, - value_type *symbol_id) - { - assert(symbol_id != 0); - assert(symbol_data != 0); - - uint32_t start = the_policy::min(0, SuperCoder::symbols()-1); - uint32_t end = the_policy::max(0, SuperCoder::symbols()-1); - - // std::cout << start << " " << end << std::endl; - - for(the_policy p(start, end); !p.at_end(); p.advance()) - { - uint32_t i = p.index(); - - value_type current_coefficient - = fifi::get_value(symbol_id, i); - - if( current_coefficient ) - { - // If symbol exists - if( symbol_pivot( i ) ) - { - value_type *vector_i = - SuperCoder::coefficients_value( i ); - - value_type *symbol_i = - SuperCoder::symbol_value( i ); - - if(fifi::is_binary::value) - { - SuperCoder::subtract( - symbol_id, vector_i, - SuperCoder::coefficients_length()); - - SuperCoder::subtract( - symbol_data, symbol_i, - SuperCoder::symbol_length()); - } - else - { - SuperCoder::multiply_subtract( - symbol_id, vector_i, - current_coefficient, - SuperCoder::coefficients_length()); - - SuperCoder::multiply_subtract( - symbol_data, symbol_i, - current_coefficient, - SuperCoder::symbol_length()); - } - } - else - { - return boost::optional( i ); - } - } - } - - return boost::none; - } - - /// Iterates the encoding vector from where a pivot has been - /// identified and subtracts existing symbols - /// @param symbol_data the data of the encoded symbol - /// @param symbol_id the data constituting the encoding vector - /// @param pivot_index the index of the found pivot element - void forward_substitute_from_pivot(value_type *symbol_data, - value_type *symbol_id, - uint32_t pivot_index) - { - // We have received an encoded symbol - described - // by the symbol group. We now normalize the - // the encoding vector according to the symbol id. - // I.e. we make sure the pivot position has a "1" - assert(symbol_id != 0); - assert(symbol_data != 0); - - assert(pivot_index < SuperCoder::symbols()); - - assert(m_uncoded[pivot_index] == false); - assert(m_coded[pivot_index] == false); - - // If this pivot index was smaller than the maximum pivot - // index we have, we might also need to backward - // substitute the higher pivot values into the new packet - uint32_t end = the_policy::max(0, SuperCoder::symbols()-1); - - // std::cout << pivot_index << " " << end << std::endl; - - the_policy p(pivot_index, end); - - // Jump past the pivot_index position - p.advance(); - - for(; !p.at_end(); p.advance()) - { - uint32_t i = p.index(); - - // Do we have a non-zero value here? - value_type value = - fifi::get_value(symbol_id, i); - - - if( !value ) - { - continue; - } - - if( symbol_pivot(i) ) - { - value_type *vector_i = - SuperCoder::coefficients_value(i); - - value_type *symbol_i = - SuperCoder::symbol_value(i); - - if(fifi::is_binary::value) - { - SuperCoder::subtract( - symbol_id, vector_i, - SuperCoder::coefficients_length()); - - SuperCoder::subtract( - symbol_data, symbol_i, - SuperCoder::symbol_length()); - } - else - { - SuperCoder::multiply_subtract( - symbol_id, vector_i, value, - SuperCoder::coefficients_length()); - - SuperCoder::multiply_subtract( - symbol_data, symbol_i, value, - SuperCoder::symbol_length()); - } - } - } - } - - /// Backward substitute the found symbol into the - /// existing symbols. - /// @param symbol_data buffer containing the encoding symbol - /// @param symbol_id buffer containing the encoding vector - /// @param pivot_index the pivot index of the symbol in the - /// buffers symbol_id and symbol_data - void backward_substitute(const value_type *symbol_data, - const value_type *symbol_id, - uint32_t pivot_index) - { - assert(symbol_id != 0); - assert(symbol_data != 0); - - assert(pivot_index < SuperCoder::symbols()); - - uint32_t from = the_policy::min(0, SuperCoder::symbols()-1); - uint32_t to = m_maximum_pivot; - - // std::cout << from << " " << to << std::endl; - - // We found a "1" that nobody else had as pivot, we now - // substract this packet from other coded packets - // - if they have a "1" on our pivot place - for(the_policy p(from, to); !p.at_end(); p.advance()) - { - uint32_t i = p.index(); - - if( m_uncoded[i] ) - { - // We know that we have no non-zero elements - // outside the pivot position. - continue; - } - - if(i == pivot_index) - { - // We cannot backward substitute into ourself - continue; - } - - if( m_coded[i] ) - { - value_type *vector_i = - SuperCoder::coefficients_value(i); - - value_type value = - fifi::get_value( - vector_i, pivot_index); - - if( value ) - { - - value_type *symbol_i = - SuperCoder::symbol_value(i); - - if(fifi::is_binary::value) - { - SuperCoder::subtract( - vector_i, symbol_id, - SuperCoder::coefficients_length()); - - SuperCoder::subtract( - symbol_i, symbol_data, - SuperCoder::symbol_length()); - } - else - { - - // Update symbol and corresponding vector - SuperCoder::multiply_subtract( - vector_i, symbol_id, value, - SuperCoder::coefficients_length()); - - SuperCoder::multiply_subtract( - symbol_i, symbol_data, value, - SuperCoder::symbol_length()); - } - } - } - } - } - - /// Store an encoded symbol and encoding vector with the specified - /// pivot found. - /// @param symbol_data buffer containing the encoding symbol - /// @param symbol_coefficients buffer containing the symbol - /// coefficients - /// @param pivot_index the pivot index - void store_coded_symbol(const value_type *symbol_data, - const value_type *symbol_coefficients, - uint32_t pivot_index) - { - assert(m_uncoded[pivot_index] == false); - assert(m_coded[pivot_index] == false); - assert(symbol_coefficients != 0); - assert(symbol_data != 0); - assert(SuperCoder::is_symbol_available(pivot_index)); - - auto coefficient_storage = - sak::storage(symbol_coefficients, - SuperCoder::coefficients_size()); - - SuperCoder::set_coefficients( - pivot_index, coefficient_storage); - - // Copy it into the symbol storage - sak::mutable_storage dest = - sak::storage(SuperCoder::symbol(pivot_index), - SuperCoder::symbol_size()); - - sak::const_storage src = - sak::storage(symbol_data, SuperCoder::symbol_size()); - - sak::copy_storage(dest, src); - } - - /// Stores an uncoded or fully decoded symbol - /// @param symbol_data the data for the symbol - /// @param pivot_index the pivot index of the symbol - void store_uncoded_symbol(const value_type *symbol_data, - uint32_t pivot_index) - { - assert(symbol_data != 0); - assert(m_uncoded[pivot_index] == false); - assert(m_coded[pivot_index] == false); - assert(SuperCoder::is_symbol_available(pivot_index)); - - // Update the corresponding vector - value_type *vector_dest = - SuperCoder::coefficients_value( pivot_index ); - - // Zero out the memory first - std::fill_n(vector_dest, SuperCoder::coefficients_length(), 0); - - fifi::set_value(vector_dest, pivot_index, 1U); - - // Copy it into the symbol storage - sak::mutable_storage dest = - sak::storage(SuperCoder::symbol(pivot_index), - SuperCoder::symbol_size()); - - sak::const_storage src = - sak::storage(symbol_data, SuperCoder::symbol_size()); - - sak::copy_storage(dest, src); - - } - - protected: - - /// The current rank of the decoder - uint32_t m_rank; - - /// Stores the current maximum pivot index - uint32_t m_maximum_pivot; - - /// Tracks whether a symbol is contained which - /// is fully decoded - std::vector m_uncoded; - - /// Tracks whether a symbol is partially decoded - std::vector m_coded; - }; - -} - diff --git a/src/kodo/linear_block_decoder.hpp b/src/kodo/linear_block_decoder.hpp index e4f41bd5..fd271449 100644 --- a/src/kodo/linear_block_decoder.hpp +++ b/src/kodo/linear_block_decoder.hpp @@ -23,6 +23,7 @@ namespace kodo { + /// @ingroup codec_layers /// @brief Implements basic linear block decoder. /// diff --git a/test/src/test_forward_linear_block_decoder.cpp b/test/src/test_forward_linear_block_decoder.cpp deleted file mode 100644 index 6e5ffe34..00000000 --- a/test/src/test_forward_linear_block_decoder.cpp +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright Steinwurf ApS 2011-2013. -// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". -// See accompanying file LICENSE.rst or -// http://www.steinwurf.com/licensing - -/// @file test_forward_linear_block_decoder.cpp Unit tests for the -/// kodo::forward_linear_block_decoder - -#include -#include - -#include -#include - -#include - -#include "basic_api_test_helper.hpp" - -namespace kodo -{ - template - class test_forward_stack - : public // Payload API - // Codec Header API - // Symbol ID API - // Codec API - debug_linear_block_decoder< - linear_block_decoder< - // Coefficient Storage API - coefficient_storage< - coefficient_info< - // Storage API - deep_symbol_storage< - storage_bytes_used< - storage_block_info< - // Finite Field API - finite_field_math::type, - finite_field_info - > > > > > > > > > > - { }; - -} - - -/// Run the tests typical coefficients stack -TEST(TestForwardLinearBlockDecoder, test_decoder) -{ - typedef fifi::binary field_type; - - kodo::test_forward_stack::factory f(8, 1600); - - auto d = f.build(); - - EXPECT_EQ(d->coefficients_size(), 1U); - - uint8_t coefficients[1]; - - // Create an encoding vector looking like this: 01000010 - coefficients[0] = 0; - fifi::set_value(coefficients, 1, 1U); - fifi::set_value(coefficients, 6, 1U); - - // Create a dummy symbol - std::vector symbol = random_vector(d->symbol_size()); - - d->decode_symbol(&symbol[0], coefficients); - d->print_decoder_state(std::cout); - - EXPECT_EQ(d->rank(), 1U); - EXPECT_TRUE(d->symbol_pivot(6)); - - // Create an encoding vector looking like this: 10000010 - coefficients[0] = 0; - fifi::set_value(coefficients, 0, 1U); - fifi::set_value(coefficients, 6, 1U); - - d->decode_symbol(&symbol[0], coefficients); - d->print_decoder_state(std::cout); - - EXPECT_EQ(d->rank(), 2U); - EXPECT_TRUE(d->symbol_pivot(6)); - EXPECT_TRUE(d->symbol_pivot(1)); - - // Create an encoding vector looking like this: 11100010 - coefficients[0] = 0; - fifi::set_value(coefficients, 0, 1U); - fifi::set_value(coefficients, 1, 1U); - fifi::set_value(coefficients, 2, 1U); - fifi::set_value(coefficients, 6, 1U); - - d->decode_symbol(&symbol[0], coefficients); - d->print_decoder_state(std::cout); - - EXPECT_EQ(d->rank(), 3U); - EXPECT_TRUE(d->symbol_pivot(6)); - EXPECT_TRUE(d->symbol_pivot(1)); - EXPECT_TRUE(d->symbol_pivot(2)); - - // Create an encoding vector looking like this: 11100010 - coefficients[0] = 0; - fifi::set_value(coefficients, 0, 1U); - fifi::set_value(coefficients, 1, 1U); - fifi::set_value(coefficients, 2, 1U); - fifi::set_value(coefficients, 3, 1U); - fifi::set_value(coefficients, 4, 1U); - fifi::set_value(coefficients, 5, 1U); - - d->decode_symbol(&symbol[0], coefficients); - d->print_decoder_state(std::cout); - - // Create an encoding vector looking like this: 11100010 - coefficients[0] = 0; - fifi::set_value(coefficients, 0, 1U); - fifi::set_value(coefficients, 2, 1U); - fifi::set_value(coefficients, 3, 1U); - fifi::set_value(coefficients, 4, 1U); - fifi::set_value(coefficients, 5, 1U); - - d->decode_symbol(&symbol[0], coefficients); - d->print_decoder_state(std::cout); - - // Create an encoding vector looking like this: 11100010 - coefficients[0] = 0; - fifi::set_value(coefficients, 0, 1U); - fifi::set_value(coefficients, 1, 1U); - fifi::set_value(coefficients, 2, 1U); - fifi::set_value(coefficients, 3, 1U); - fifi::set_value(coefficients, 5, 1U); - - d->decode_symbol(&symbol[0], coefficients); - d->print_decoder_state(std::cout); - - // Create an encoding vector looking like this: 11100010 - coefficients[0] = 0; - fifi::set_value(coefficients, 0, 1U); - fifi::set_value(coefficients, 3, 1U); - - - d->decode_symbol(&symbol[0], coefficients); - d->print_decoder_state(std::cout); - - - // Create an encoding vector looking like this: 11100010 - coefficients[0] = 0; - fifi::set_value(coefficients, 3, 1U); - fifi::set_value(coefficients, 4, 1U); - fifi::set_value(coefficients, 7, 1U); - - d->decode_symbol(&symbol[0], coefficients); - d->print_decoder_state(std::cout); - - - - EXPECT_EQ(d->rank(), 3U); - EXPECT_TRUE(d->symbol_pivot(6)); - EXPECT_TRUE(d->symbol_pivot(1)); - EXPECT_TRUE(d->symbol_pivot(2)); - - -} - - - From 42d1ad325fcc855840fa376d45c728121e7d371b Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Sat, 20 Jul 2013 02:33:31 +0200 Subject: [PATCH 37/83] Adding on-the-fly stack --- src/kodo/rlnc/full_vector_codes.hpp | 4 +- src/kodo/rlnc/on_the_fly_codes.hpp | 101 +++++++++ src/kodo/rlnc/seed_codes.hpp | 4 +- src/kodo/seed_symbol_id_writer.hpp | 4 +- test/src/basic_api_test_helper.hpp | 46 ---- test/src/helper_on_the_fly_api.hpp | 109 +++++++++ test/src/helper_recoding_api.hpp | 175 +++++++++++++++ test/src/helper_reuse_api.hpp | 269 +++++++++++++++++++++++ test/src/test_rlnc_full_vector_codes.cpp | 95 +------- test/src/test_rlnc_on_the_fly_codes.cpp | 248 +++++++++++++++++++++ 10 files changed, 907 insertions(+), 148 deletions(-) create mode 100644 src/kodo/rlnc/on_the_fly_codes.hpp create mode 100644 test/src/helper_on_the_fly_api.hpp create mode 100644 test/src/helper_recoding_api.hpp create mode 100644 test/src/helper_reuse_api.hpp create mode 100644 test/src/test_rlnc_on_the_fly_codes.cpp diff --git a/src/kodo/rlnc/full_vector_codes.hpp b/src/kodo/rlnc/full_vector_codes.hpp index 1b6ebfa1..4498d79f 100644 --- a/src/kodo/rlnc/full_vector_codes.hpp +++ b/src/kodo/rlnc/full_vector_codes.hpp @@ -3,8 +3,7 @@ // See accompanying file LICENSE.rst or // http://www.steinwurf.com/licensing -#ifndef KODO_RLNC_FULL_VECTOR_CODES_HPP -#define KODO_RLNC_FULL_VECTOR_CODES_HPP +#pragma once #include @@ -154,5 +153,4 @@ namespace kodo } -#endif diff --git a/src/kodo/rlnc/on_the_fly_codes.hpp b/src/kodo/rlnc/on_the_fly_codes.hpp new file mode 100644 index 00000000..5df337d8 --- /dev/null +++ b/src/kodo/rlnc/on_the_fly_codes.hpp @@ -0,0 +1,101 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +#pragma once + +#include + +#include "full_vector_codes.hpp" +#include "../storage_aware_generator.hpp" + +namespace kodo +{ + + /// @ingroup fec_stacks + /// @brief Complete stack implementing a full RLNC on-the-fly encoder. + /// + /// The on-the-fly encoder has the advantage that symbols can be specified + /// as they arrive at the encoder. This breaks with a traditional block + /// code where all the data has to be available before encoding can start. + /// + /// Implementation of on the fly RLNC encoder uses a storage aware generator + /// and storage aware encoder. The storage aware generator makes sure that + /// we do not generate non-zero coefficients for the missing symbols, the + /// storage aware encoder provides the generator with information about how + /// many symbols have been specified. + template + class on_the_fly_encoder : + public // Payload Codec API + payload_encoder< + // Codec Header API + systematic_encoder< + symbol_id_encoder< + // Symbol ID API + plain_symbol_id_writer< + // Coefficient Generator API + storage_aware_generator< + uniform_generator< + // Codec API + encode_symbol_tracker< + zero_symbol_encoder< + linear_block_encoder< + storage_aware_encoder< + // Coefficient Storage API + coefficient_info< + // Symbol Storage API + deep_symbol_storage< + storage_bytes_used< + storage_block_info< + // Finite Field API + finite_field_math::type, + finite_field_info + > > > > > > > > > > > > > > > > > + { }; + + /// @ingroup fec_stacks + /// @brief Implementation of a complete RLNC decoder + /// + /// This configuration adds the following features (including those + /// described for the encoder): + /// - Recoding using the recoding_stack + /// - Linear block decoder using Gauss-Jordan elimination. + template + class on_the_fly_decoder : + public // Payload API + payload_recoder::type, + finite_field_info + > > > > > > > > > > > > > > > + { }; + +} + + + diff --git a/src/kodo/rlnc/seed_codes.hpp b/src/kodo/rlnc/seed_codes.hpp index a6608576..07065582 100644 --- a/src/kodo/rlnc/seed_codes.hpp +++ b/src/kodo/rlnc/seed_codes.hpp @@ -3,8 +3,7 @@ // See accompanying file LICENSE.rst or // http://www.steinwurf.com/licensing -#ifndef KODO_RLNC_SEED_CODES_HPP -#define KODO_RLNC_SEED_CODES_HPP +#pragma once #include @@ -125,5 +124,4 @@ namespace kodo } -#endif diff --git a/src/kodo/seed_symbol_id_writer.hpp b/src/kodo/seed_symbol_id_writer.hpp index a7d3ef77..811b00be 100644 --- a/src/kodo/seed_symbol_id_writer.hpp +++ b/src/kodo/seed_symbol_id_writer.hpp @@ -3,8 +3,7 @@ // See accompanying file LICENSE.rst or // http://www.steinwurf.com/licensing -#ifndef KODO_SEED_SYMBOL_ID_WRITER_HPP -#define KODO_SEED_SYMBOL_ID_WRITER_HPP +#pragma once #include @@ -63,5 +62,4 @@ namespace kodo } -#endif diff --git a/test/src/basic_api_test_helper.hpp b/test/src/basic_api_test_helper.hpp index beffe3eb..759e541a 100644 --- a/test/src/basic_api_test_helper.hpp +++ b/test/src/basic_api_test_helper.hpp @@ -364,52 +364,6 @@ invoke_systematic(uint32_t symbols, uint32_t symbol_size) } -/// Tests that an encoder support progressively specifying the symbols -template -inline void -invoke_set_symbol(uint32_t symbols, uint32_t symbol_size) -{ - - // Common setting - typename Encoder::factory encoder_factory(symbols, symbol_size); - auto encoder = encoder_factory.build(); - - typename Decoder::factory decoder_factory(symbols, symbol_size); - auto decoder = decoder_factory.build(); - - std::vector payload(encoder->payload_size()); - std::vector data_in = random_vector(encoder->block_size()); - - auto symbol_sequence = sak::split_storage( - sak::storage(data_in), symbol_size); - - // Set the encoder non-systematic - if(kodo::is_systematic_encoder(encoder)) - kodo::set_systematic_off(encoder); - - EXPECT_EQ(encoder->rank(), 0U); - EXPECT_EQ(decoder->rank(), 0U); - - while( !decoder->is_complete() ) - { - encoder->encode( &payload[0] ); - decoder->decode( &payload[0] ); - - if(encoder->rank() < symbols) - { - uint32_t i = rand() % symbols; - encoder->set_symbol(i, symbol_sequence[i]); - } - } - - std::vector data_out(decoder->block_size(), '\0'); - decoder->copy_symbols(sak::storage(data_out)); - - EXPECT_TRUE(std::equal(data_out.begin(), - data_out.end(), - data_in.begin())); -} - diff --git a/test/src/helper_on_the_fly_api.hpp b/test/src/helper_on_the_fly_api.hpp new file mode 100644 index 00000000..ada6f30d --- /dev/null +++ b/test/src/helper_on_the_fly_api.hpp @@ -0,0 +1,109 @@ +// Copyright Steinwurf ApS 2011-2012. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + + +/// @file helper_on_the_fly_api.hpp Unit test helper for the on the fly API. +/// on-the-fly referrers to the fact the we progressively will set symbols +/// on the stack + +#include +#include + + +/// Tests that an encoder support progressively specifying the symbols +template +< + class Encoder, + class Decoder +> +inline void test_on_the_fly(uint32_t symbols, uint32_t symbol_size) +{ + + // Common setting + typename Encoder::factory encoder_factory(symbols, symbol_size); + auto encoder = encoder_factory.build(); + + typename Decoder::factory decoder_factory(symbols, symbol_size); + auto decoder = decoder_factory.build(); + + std::vector payload(encoder->payload_size()); + std::vector data_in = random_vector(encoder->block_size()); + + auto symbol_sequence = sak::split_storage( + sak::storage(data_in), symbol_size); + + // Set the encoder non-systematic + if(kodo::is_systematic_encoder(encoder)) + kodo::set_systematic_off(encoder); + + EXPECT_EQ(encoder->rank(), 0U); + EXPECT_EQ(decoder->rank(), 0U); + + while( !decoder->is_complete() ) + { + encoder->encode( &payload[0] ); + decoder->decode( &payload[0] ); + + if(encoder->rank() < symbols) + { + uint32_t i = rand() % symbols; + encoder->set_symbol(i, symbol_sequence[i]); + } + } + + std::vector data_out(decoder->block_size(), '\0'); + decoder->copy_symbols(sak::storage(data_out)); + + EXPECT_TRUE(std::equal(data_out.begin(), + data_out.end(), + data_in.begin())); +} + + +template +< + template class Encoder, + template class Decoder +> +inline void test_on_the_fly(uint32_t symbols, uint32_t symbol_size) +{ + + test_on_the_fly< + Encoder, + Decoder >( + symbols, symbol_size); + + test_on_the_fly< + Encoder, + Decoder >( + symbols, symbol_size); + + test_on_the_fly< + Encoder, + Decoder >( + symbols, symbol_size); + +} + +template +< + template class Encoder, + template class Decoder +> +inline void test_on_the_fly() +{ + + test_on_the_fly(32, 1600); + test_on_the_fly(1, 1600); + + uint32_t symbols = rand_symbols(); + uint32_t symbol_size = rand_symbol_size(); + + test_on_the_fly(symbols, symbol_size); + +} + + + diff --git a/test/src/helper_recoding_api.hpp b/test/src/helper_recoding_api.hpp new file mode 100644 index 00000000..99dd51ef --- /dev/null +++ b/test/src/helper_recoding_api.hpp @@ -0,0 +1,175 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +#pragma once + +/// @file helper_recoding_api.hpp Unit test helper for testing that recoding +/// works + +#include +#include + +#include + +/// Small helper structure holding the parameters needed for the +/// recoding tests. +struct recoding_parameters +{ + uint32_t m_max_symbols; + uint32_t m_max_symbol_size; + uint32_t m_symbols; + uint32_t m_symbol_size; +}; + +/// Tests that the recoding function works, this is done by using one +/// encoder +/// and two decoders: +/// +/// +------------+ +------------+ +------------+ +/// | encoder |+---->| decoder #1 |+---->| decoder #2 | +/// +------------+ +------------+ +------------+ +/// +/// Where the encoder passes data to the first decoder which then +/// recodes and passes data to the second decoder +/// +/// @param param The recoding parameters to use +template +inline void invoke_recoding(recoding_parameters param) +{ + // Common setting + typename Encoder::factory encoder_factory( + param.m_max_symbols, param.m_max_symbol_size); + + encoder_factory.set_symbols(param.m_symbols); + encoder_factory.set_symbol_size(param.m_symbol_size); + + auto encoder = encoder_factory.build(); + + typename Decoder::factory decoder_factory( + param.m_max_symbols, param.m_max_symbol_size); + + decoder_factory.set_symbols(param.m_symbols); + decoder_factory.set_symbol_size(param.m_symbol_size); + + auto decoder_one = decoder_factory.build(); + auto decoder_two = decoder_factory.build(); + + // If tested with a shallow decoder we have to remember to set the + // buffers to use for the decoding + std::vector buffer_decoder_one(decoder_one->block_size(), '\0'); + std::vector buffer_decoder_two(decoder_two->block_size(), '\0'); + + if(kodo::has_shallow_symbol_storage::value) + { + decoder_one->set_symbols(sak::storage(buffer_decoder_one)); + decoder_two->set_symbols(sak::storage(buffer_decoder_two)); + } + + EXPECT_EQ(encoder->payload_size(), decoder_one->payload_size()); + EXPECT_EQ(encoder->payload_size(), decoder_two->payload_size()); + + std::vector payload(encoder->payload_size()); + std::vector data_in = random_vector(encoder->block_size()); + + encoder->set_symbols(sak::storage(data_in)); + + // Set the encoder non-systematic + if(kodo::is_systematic_encoder(encoder)) + kodo::set_systematic_off(encoder); + + while( !decoder_two->is_complete() ) + { + uint32_t encode_size = encoder->encode( &payload[0] ); + EXPECT_TRUE(encode_size <= payload.size()); + EXPECT_TRUE(encode_size > 0); + + decoder_one->decode( &payload[0] ); + + uint32_t recode_size = decoder_one->recode( &payload[0] ); + EXPECT_TRUE(recode_size <= payload.size()); + EXPECT_TRUE(recode_size > 0); + + decoder_two->decode( &payload[0] ); + } + + std::vector data_out_one(decoder_one->block_size(), '\0'); + std::vector data_out_two(decoder_two->block_size(), '\0'); + + decoder_one->copy_symbols(sak::storage(data_out_one)); + decoder_two->copy_symbols(sak::storage(data_out_two)); + + EXPECT_TRUE(std::equal(data_out_one.begin(), + data_out_one.end(), + data_in.begin())); + + EXPECT_TRUE(std::equal(data_out_two.begin(), + data_out_two.end(), + data_in.begin())); +} + +/// Invokes the recoding API for the Encoder and Decoder with +/// typical field sizes +/// @param param The recoding parameters +template +< + template class Encoder, + template class Decoder +> +inline void test_recoders(recoding_parameters param) +{ + invoke_recoding< + Encoder, + Decoder >(param); + + invoke_recoding< + Encoder, + Decoder >(param); + + invoke_recoding< + Encoder, + Decoder >(param); +} + + +/// Invokes the recoding API for the Encoder and Decoder with +/// typical field sizes +/// @param param The recoding parameters +template +< + template class Encoder, + template class Decoder +> +inline void test_recoders() +{ + recoding_parameters param; + param.m_max_symbols = 32; + param.m_max_symbol_size = 1600; + param.m_symbols = param.m_max_symbols; + param.m_symbol_size = param.m_max_symbol_size; + + test_recoders(param); + + param.m_max_symbols = 1; + param.m_max_symbol_size = 1600; + param.m_symbols = param.m_max_symbols; + param.m_symbol_size = param.m_max_symbol_size; + + test_recoders(param); + + param.m_max_symbols = 8; + param.m_max_symbol_size = 8; + param.m_symbols = param.m_max_symbols; + param.m_symbol_size = param.m_max_symbol_size; + + test_recoders(param); + + param.m_max_symbols = rand_symbols(); + param.m_max_symbol_size = rand_symbol_size(); + param.m_symbols = rand_symbols(param.m_max_symbols); + param.m_symbol_size = rand_symbol_size(param.m_max_symbol_size); + + test_recoders(param); +} + diff --git a/test/src/helper_reuse_api.hpp b/test/src/helper_reuse_api.hpp new file mode 100644 index 00000000..effb0762 --- /dev/null +++ b/test/src/helper_reuse_api.hpp @@ -0,0 +1,269 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +#pragma once + +#include + +#include "basic_api_test_helper.hpp" + +/// Helper for the reuse test, ensures that all encoders and decoders +/// produce valid data +template +inline void test_reuse_helper(Encoder encoder, Decoder decoder) +{ + std::vector payload(encoder->payload_size()); + + std::vector data_in = random_vector(encoder->block_size()); + sak::mutable_storage storage_in = sak::storage(data_in); + + encoder->set_symbols(storage_in); + + // Set the encoder non-systematic + if(kodo::is_systematic_encoder(encoder)) + kodo::set_systematic_off(encoder); + + while( !decoder->is_complete() ) + { + uint32_t payload_used = encoder->encode( &payload[0] ); + EXPECT_TRUE(payload_used <= encoder->payload_size()); + + decoder->decode( &payload[0] ); + } + + std::vector data_out(decoder->block_size(), '\0'); + decoder->copy_symbols(sak::storage(data_out)); + + EXPECT_TRUE(std::equal(data_out.begin(), + data_out.end(), + data_in.begin())); + +} + + +/// Test that instantiates a number of encoders and decoders from +/// the same factories +template +inline void test_reuse(uint32_t symbols, uint32_t symbol_size) +{ + // Common setting + typename Encoder::factory encoder_factory(symbols, symbol_size); + typename Decoder::factory decoder_factory(symbols, symbol_size); + + for(uint32_t i = 0; i < 4; ++i) + { + uint32_t coders = rand_nonzero(10); + + std::vector encoders; + std::vector decoders; + + for(uint32_t j = 0; j < coders; ++j) + { + encoders.push_back(encoder_factory.build()); + decoders.push_back(decoder_factory.build()); + + } + + for(uint32_t j = 0; j < coders; ++j) + { + test_reuse_helper(encoders[j], decoders[j]); + } + + } + + return; + + // Test with differing symbols and symbol sizes + for(uint32_t i = 0; i < 4; ++i) + { + uint32_t coders = rand_nonzero(10); + + std::vector encoders; + std::vector decoders; + + for(uint32_t j = 0; j < coders; ++j) + { + uint32_t s = rand_symbols(encoder_factory.max_symbols()); + uint32_t l = rand_symbol_size(encoder_factory.max_symbol_size()); + + encoder_factory.set_symbols(s); + encoder_factory.set_symbol_size(l); + decoder_factory.set_symbols(s); + decoder_factory.set_symbol_size(l); + + encoders.push_back(encoder_factory.build()); + decoders.push_back(decoder_factory.build()); + } + + for(uint32_t j = 0; j < coders; ++j) + { + test_reuse_helper(encoders[j], decoders[j]); + } + + } + +} + +/// Test reuse function which will invoke the reuse test for the common +/// field sizes +template +< + template class Encoder, + template class Decoder +> +inline void test_reuse(uint32_t symbols, uint32_t symbol_size) +{ + test_reuse + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_reuse + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_reuse + < + Encoder, + Decoder + >(symbols, symbol_size); +} + +template +< + template class Encoder, + template class Decoder +> +inline void test_reuse() +{ + test_reuse(32, 1600); + test_reuse(32, 1600); + + uint32_t symbols = rand_symbols(); + uint32_t symbol_size = rand_symbol_size(); + + test_reuse(symbols, symbol_size); +} + + + +/// Tests that encoders and decoders can be safely reused after incomplete +/// encoding / decoding. +template +inline void test_reuse_incomplete(uint32_t symbols, uint32_t symbol_size) +{ + + bool do_complete; + + typename Encoder::factory encoder_factory(symbols, symbol_size); + typename Decoder::factory decoder_factory(symbols, symbol_size); + + // Use factory a lot of times + for (uint32_t i = 0; i < 100; ++i) + { + // Build coders + auto encoder = encoder_factory.build(); + auto decoder = decoder_factory.build(); + + // Prepare buffers + std::vector payload(encoder->payload_size()); + std::vector data_in(encoder->block_size()); + + // Fill with random data + for (auto &e: data_in) + e = rand() % 256; + + // Put data in encoder + encoder->set_symbols(sak::storage(data_in)); + + if (rand() % 100 > 90) + { + do_complete = false; + } + else + { + do_complete = true; + } + + // Start encoding/decoding + while (!decoder->is_complete()) + { + encoder->encode(&payload[0]); + + // Loose a packet with probability + if (rand() % 100 > 90) + continue; + + decoder->decode(&payload[0]); + + // Stop decoding after a while with probability + if (!do_complete && decoder->rank() == symbols - 2) + break; + } + + // Check if completed decoders are correct + if (decoder->is_complete()) + { + std::vector data_out(decoder->block_size()); + decoder->copy_symbols(sak::storage(data_out)); + + ASSERT_TRUE(sak::equal(sak::storage(data_out), + sak::storage(data_in))); + } + } + +} + + +/// Test reuse function which will invoke the reuse test for the common +/// field sizes +template + < + template class Encoder, + template class Decoder + > +inline void test_reuse_incomplete(uint32_t symbols, uint32_t symbol_size) +{ + test_reuse_incomplete + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_reuse_incomplete + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_reuse_incomplete + < + Encoder, + Decoder + >(symbols, symbol_size); +} + + +template +< + template class Encoder, + template class Decoder +> +inline void test_reuse_incomplete() +{ + test_reuse_incomplete(32, 1600); + test_reuse_incomplete(1, 1600); + + uint32_t symbols = rand_symbols(); + uint32_t symbol_size = rand_symbol_size(); + + test_reuse_incomplete(symbols, symbol_size); +} + + + diff --git a/test/src/test_rlnc_full_vector_codes.cpp b/test/src/test_rlnc_full_vector_codes.cpp index e59191da..438fbca7 100644 --- a/test/src/test_rlnc_full_vector_codes.cpp +++ b/test/src/test_rlnc_full_vector_codes.cpp @@ -93,42 +93,6 @@ namespace kodo > > > > > > > > > > > > > > > > {}; - - /// Implementation of RLNC encoder using a storage aware generator - template - class full_rlnc_encoder_storage_aware - : public // Payload Codec API - payload_encoder< - // Codec Header API - systematic_encoder< - symbol_id_encoder< - // Symbol ID API - plain_symbol_id_writer< - // Coefficient Generator API - storage_aware_generator< - uniform_generator< - // Codec API - encode_symbol_tracker< - zero_symbol_encoder< - linear_block_encoder< - storage_aware_encoder< - // Coefficient Storage API - coefficient_info< - // Symbol Storage API - deep_symbol_storage< - storage_bytes_used< - storage_block_info< - // Finite Field API - finite_field_math::type, - finite_field_info - > > > > > > > > > > > > > > > > > - { }; - - /// Implementation of RLNC encoder using a storage aware generator template class full_rlnc_encoder_shallow : public // Payload Codec API @@ -142,10 +106,10 @@ namespace kodo storage_aware_generator< uniform_generator< // Codec API - storage_aware_encoder< encode_symbol_tracker< zero_symbol_encoder< linear_block_encoder< + storage_aware_encoder< // Coefficient Storage API coefficient_info< // Symbol Storage API @@ -158,7 +122,7 @@ namespace kodo // Factory API final_coder_factory_pool< // Final type - full_rlnc_encoder_storage_aware + full_rlnc_encoder_shallow > > > > > > > > > > > > > > > > > { }; @@ -581,61 +545,6 @@ TEST(TestRlncFullVectorCodes, recoding_simple) } -template - < - template class Encoder, - template class Decoder - > -void test_set_symbol(uint32_t symbols, uint32_t symbol_size) -{ - invoke_set_symbol< - Encoder, - Decoder >( - symbols, symbol_size); - - invoke_set_symbol< - Encoder, - Decoder >( - symbols, symbol_size); - - invoke_set_symbol< - Encoder, - Decoder >( - symbols, symbol_size); - - // invoke_set_symbol< - // Encoder, - // Decoder >( - // symbols, symbol_size); - -} - -void test_set_symbol(uint32_t symbols, uint32_t symbol_size) -{ - - test_set_symbol - < - kodo::full_rlnc_encoder_storage_aware, - kodo::full_rlnc_decoder>(symbols, symbol_size); - - test_set_symbol - < - kodo::full_rlnc_encoder_storage_aware, - kodo::full_rlnc_decoder_delayed>(symbols, symbol_size); -} - -/// Tests that we can progressively set on symbol at-a-time on -/// encoder -TEST(TestRlncFullVectorCodes, set_symbol) -{ - test_set_symbol(32, 1600); - test_set_symbol(1, 1600); - - uint32_t symbols = rand_symbols(); - uint32_t symbol_size = rand_symbol_size(); - - test_set_symbol(symbols, symbol_size); -} void test_reuse(uint32_t symbols, uint32_t symbol_size) diff --git a/test/src/test_rlnc_on_the_fly_codes.cpp b/test/src/test_rlnc_on_the_fly_codes.cpp new file mode 100644 index 00000000..7f07ed5f --- /dev/null +++ b/test/src/test_rlnc_on_the_fly_codes.cpp @@ -0,0 +1,248 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + + +/// @file test_rlnc_on_the_fly_codes.cpp Unit tests for the full +/// vector codes (i.e. Network Coding encoders and decoders). + +#include + +#include + +#include + +#include "basic_api_test_helper.hpp" + +#include "helper_reuse_api.hpp" +#include "helper_recoding_api.hpp" +#include "helper_on_the_fly_api.hpp" + +template +< + template class Encoder, + template class Decoder +> +inline void test_coders(uint32_t symbols, uint32_t symbol_size) +{ + invoke_basic_api + < + Encoder, + Decoder + >(symbols, symbol_size); + + invoke_basic_api + < + Encoder, + Decoder + >(symbols, symbol_size); + + invoke_basic_api + < + Encoder, + Decoder + >(symbols, symbol_size); + + invoke_basic_api + < + Encoder, + Decoder + >(symbols, symbol_size); +} + +inline void test_coders(uint32_t symbols, uint32_t symbol_size) +{ + test_coders< + kodo::on_the_fly_encoder, + kodo::on_the_fly_decoder + >(symbols, symbol_size); +} + +/// Tests the basic API functionality this mean basic encoding +/// and decoding +TEST(TestOnTheFlyCodes, basic_api) +{ + test_coders(32, 1600); + test_coders(1, 1600); + + uint32_t symbols = rand_symbols(); + uint32_t symbol_size = rand_symbol_size(); + + test_coders(symbols, symbol_size); +} + +template +< + template class Encoder, + template class Decoder +> +inline void test_initialize(uint32_t symbols, uint32_t symbol_size) +{ + + invoke_initialize + < + Encoder, + Decoder + >(symbols, symbol_size); + + invoke_initialize + < + Encoder, + Decoder + >(symbols, symbol_size); + + invoke_initialize + < + Encoder, + Decoder + >(symbols, symbol_size); +} + +inline void test_initialize(uint32_t symbols, uint32_t symbol_size) +{ + + test_initialize< + kodo::on_the_fly_encoder, + kodo::on_the_fly_decoder>(symbols, symbol_size); + +} + +/// Test that the encoders and decoders initialize() function can be used +/// to reset the state of an encoder and decoder and that they therefore +/// can be safely reused. +TEST(TestOnTheFlyCodes, initialize_function) +{ + test_initialize(32, 1600); + test_initialize(1, 1600); + + srand(static_cast(time(0))); + + uint32_t symbols = rand_symbols(); + uint32_t symbol_size = rand_symbol_size(); + + test_initialize(symbols, symbol_size); +} + +template +< + template class Encoder, + template class Decoder +> +inline void test_coders_systematic(uint32_t symbols, uint32_t symbol_size) +{ + invoke_systematic< + Encoder, + Decoder + >(symbols, symbol_size); + + invoke_systematic< + Encoder, + Decoder + >(symbols, symbol_size); + + invoke_systematic< + Encoder, + Decoder + >(symbols, symbol_size); +} + + +inline void test_coders_systematic(uint32_t symbols, uint32_t symbol_size) +{ + test_coders_systematic + < + kodo::on_the_fly_encoder, + kodo::on_the_fly_decoder + >(symbols, symbol_size); +} + +/// Tests that an encoder producing systematic packets is handled +/// correctly in the decoder. +TEST(TestOnTheFlyCodes, systematic) +{ + test_coders_systematic(32, 1600); + test_coders_systematic(1, 1600); + + srand(static_cast(time(0))); + + uint32_t symbols = rand_symbols(); + uint32_t symbol_size = rand_symbol_size(); + + test_coders_systematic(symbols, symbol_size); +} + +template +< + template class Encoder, + template class Decoder +> +inline void test_coders_raw(uint32_t symbols, uint32_t symbol_size) +{ + invoke_out_of_order_raw< + Encoder, + Decoder + >(symbols, symbol_size); + + invoke_out_of_order_raw< + Encoder, + Decoder + >(symbols, symbol_size); + + invoke_out_of_order_raw< + Encoder, + Decoder + >(symbols, symbol_size); +} + +inline void test_coders_raw(uint32_t symbols, uint32_t symbol_size) +{ + test_coders_raw< + kodo::on_the_fly_encoder, + kodo::on_the_fly_decoder + >(symbols, symbol_size); +} + +/// Tests whether mixed un-coded and coded packets are correctly handled +/// in the encoder and decoder. +TEST(TestOnTheFlyCodes, raw) +{ + test_coders_raw(32, 1600); + test_coders_raw(1, 1600); + + uint32_t symbols = rand_symbols(); + uint32_t symbol_size = rand_symbol_size(); + + test_coders_raw(symbols, symbol_size); +} + +/// Test the recoding functionality of the stack +TEST(TestOnTheFlyCodes, test_recoders) +{ + test_recoders(); +} + + +/// Tests that we can progressively set on symbol at-a-time on +/// encoder +TEST(TestOnTheFlyCodes, test_on_the_fly) +{ + test_on_the_fly(); +} + +/// Tests that we can progressively set on symbol at-a-time on +/// encoder +TEST(TestOnTheFlyCodes, test_reuse) +{ + test_reuse(); +} + +/// Tests that we can progressively set on symbol at-a-time on +/// encoder +TEST(TestOnTheFlyCodes, test_reuse_incomplete) +{ + test_reuse_incomplete(); +} + + + From 93be979eb87e4b055e8f6303f743f1825277dc31 Mon Sep 17 00:00:00 2001 From: "Morten V. Pedersen" Date: Sat, 20 Jul 2013 11:01:46 +0200 Subject: [PATCH 38/83] Restructuring the tests --- test/src/basic_api_test_helper.hpp | 270 ---------- test/src/helper_test_basic_api.hpp | 153 ++++++ test/src/helper_test_initialize_api.hpp | 113 +++++ test/src/helper_test_mix_uncoded_api.hpp | 127 +++++ ...api.hpp => helper_test_on_the_fly_api.hpp} | 0 ...g_api.hpp => helper_test_recoding_api.hpp} | 0 ...euse_api.hpp => helper_test_reuse_api.hpp} | 1 - test/src/helper_test_systematic_api.hpp | 100 ++++ test/src/test_carousel_codes.cpp | 4 +- test/src/test_debug_xyz.cpp | 4 +- test/src/test_reuse.hpp | 20 +- test/src/test_rlnc_full_vector_codes.cpp | 466 ++---------------- test/src/test_rlnc_on_the_fly_codes.cpp | 203 +------- test/src/test_rlnc_seed_codes.cpp | 95 ++-- test/src/test_rs_reed_solomon_codes.cpp | 1 + 15 files changed, 606 insertions(+), 951 deletions(-) create mode 100644 test/src/helper_test_basic_api.hpp create mode 100644 test/src/helper_test_initialize_api.hpp create mode 100644 test/src/helper_test_mix_uncoded_api.hpp rename test/src/{helper_on_the_fly_api.hpp => helper_test_on_the_fly_api.hpp} (100%) rename test/src/{helper_recoding_api.hpp => helper_test_recoding_api.hpp} (100%) rename test/src/{helper_reuse_api.hpp => helper_test_reuse_api.hpp} (99%) create mode 100644 test/src/helper_test_systematic_api.hpp diff --git a/test/src/basic_api_test_helper.hpp b/test/src/basic_api_test_helper.hpp index 759e541a..bfef78ff 100644 --- a/test/src/basic_api_test_helper.hpp +++ b/test/src/basic_api_test_helper.hpp @@ -99,273 +99,3 @@ inline void run_test(uint32_t symbols, uint32_t symbol_size) } -template -inline void -invoke_basic_api(uint32_t symbols, uint32_t symbol_size) -{ - - // Common setting - typename Encoder::factory encoder_factory(symbols, symbol_size); - auto encoder = encoder_factory.build(); - - typename Decoder::factory decoder_factory(symbols, symbol_size); - auto decoder = decoder_factory.build(); - - EXPECT_TRUE(symbols == encoder_factory.max_symbols()); - EXPECT_TRUE(symbol_size == encoder_factory.max_symbol_size()); - EXPECT_TRUE(symbols == encoder->symbols()); - EXPECT_TRUE(symbol_size == encoder->symbol_size()); - - EXPECT_TRUE(symbols == decoder_factory.max_symbols()); - EXPECT_TRUE(symbol_size == decoder_factory.max_symbol_size()); - EXPECT_TRUE(symbols == decoder->symbols()); - EXPECT_TRUE(symbol_size == decoder->symbol_size()); - - EXPECT_TRUE(encoder->symbol_length() > 0); - EXPECT_TRUE(decoder->symbol_length() > 0); - - EXPECT_TRUE(encoder->block_size() == symbols * symbol_size); - EXPECT_TRUE(decoder->block_size() == symbols * symbol_size); - - EXPECT_TRUE(encoder_factory.max_payload_size() >= - encoder->payload_size()); - - EXPECT_TRUE(decoder_factory.max_payload_size() >= - decoder->payload_size()); - - EXPECT_EQ(encoder_factory.max_payload_size(), - decoder_factory.max_payload_size()); - - // Encode/decode operations - EXPECT_EQ(encoder->payload_size(), decoder->payload_size()); - - std::vector payload(encoder->payload_size()); - - std::vector data_in = random_vector(encoder->block_size()); - std::vector data_in_copy(data_in); - - sak::mutable_storage storage_in = sak::storage(data_in); - sak::mutable_storage storage_in_copy = sak::storage(data_in_copy); - - EXPECT_TRUE(sak::equal(storage_in, storage_in_copy)); - - // Only used for prime fields, lets reconsider how we implement - // this less intrusive - uint32_t prefix = 0; - - if(fifi::is_prime2325::value) - { - // This field only works for multiple of uint32_t - assert((encoder->block_size() % 4) == 0); - - uint32_t block_length = encoder->block_size() / 4; - - fifi::prime2325_binary_search search(block_length); - prefix = search.find_prefix(storage_in_copy); - - // Apply the negated prefix - fifi::apply_prefix(storage_in_copy, ~prefix); - } - - encoder->set_symbols(storage_in_copy); - - // Set the encoder non-systematic - if(kodo::is_systematic_encoder(encoder)) - kodo::set_systematic_off(encoder); - - while( !decoder->is_complete() ) - { - uint32_t payload_used = encoder->encode( &payload[0] ); - EXPECT_TRUE(payload_used <= encoder->payload_size()); - - decoder->decode( &payload[0] ); - } - - std::vector data_out(decoder->block_size(), '\0'); - decoder->copy_symbols(sak::storage(data_out)); - - if(fifi::is_prime2325::value) - { - // Now we have to apply the negated prefix to the decoded data - fifi::apply_prefix(sak::storage(data_out), ~prefix); - } - - EXPECT_TRUE(std::equal(data_out.begin(), - data_out.end(), - data_in.begin())); -} - -template -inline void -invoke_out_of_order_raw(uint32_t symbols, uint32_t symbol_size) -{ - // Common setting - typename Encoder::factory encoder_factory(symbols, symbol_size); - auto encoder = encoder_factory.build(); - - typename Decoder::factory decoder_factory(symbols, symbol_size); - auto decoder = decoder_factory.build(); - - // Encode/decode operations - EXPECT_TRUE(encoder->payload_size() == decoder->payload_size()); - - std::vector payload(encoder->payload_size()); - std::vector data_in = random_vector(encoder->block_size()); - - encoder->set_symbols(sak::storage(data_in)); - - if(kodo::is_systematic_encoder(encoder)) - kodo::set_systematic_off(encoder); - - while( !decoder->is_complete() ) - { - - if((rand() % 100) > 50) - { - encoder->encode( &payload[0] ); - decoder->decode( &payload[0] ); - } - else - { - uint32_t symbol_id = rand() % encoder->symbols(); - - if(decoder->symbol_pivot(symbol_id)) - { - continue; - } - - ASSERT_TRUE(encoder->symbol_size() <= payload.size()); - - encoder->copy_symbol( - symbol_id, sak::storage(payload)); - - const uint8_t *symbol_src = encoder->symbol(symbol_id); - const uint8_t *symbol_dest = &payload[0]; - - EXPECT_TRUE(std::equal(symbol_src, - symbol_src + encoder->symbol_size(), - symbol_dest)); - - - decoder->decode_symbol(&payload[0], symbol_id); - - } - } - - EXPECT_EQ(encoder->block_size(), decoder->block_size()); - - std::vector data_out(decoder->block_size(), '\0'); - decoder->copy_symbols(sak::storage(data_out)); - - EXPECT_TRUE(std::equal(data_out.begin(), - data_out.end(), - data_in.begin())); - -} - -/// Checks that the encoders and decoders are in a clean state after using -/// the initialize function. -template -inline void -invoke_initialize(uint32_t symbols, uint32_t symbol_size) -{ - - // Common setting - typename Encoder::factory encoder_factory(symbols, symbol_size); - auto encoder = encoder_factory.build(); - - typename Decoder::factory decoder_factory(symbols, symbol_size); - auto decoder = decoder_factory.build(); - - - for(uint32_t i = 0; i < 10; ++i) - { - encoder->initialize(encoder_factory); - decoder->initialize(decoder_factory); - - std::vector payload(encoder->payload_size()); - - // Ensure that the we may re-use the encoder also with partial - // data. - uint32_t block_size = rand_nonzero(encoder->block_size()); - - std::vector data_in = random_vector(block_size); - - encoder->set_symbols(sak::storage(data_in)); - - // Set the encoder non-systematic - if(kodo::is_systematic_encoder(encoder)) - kodo::set_systematic_off(encoder); - - while( !decoder->is_complete() ) - { - uint32_t payload_used = encoder->encode( &payload[0] ); - EXPECT_TRUE(payload_used <= encoder->payload_size()); - - decoder->decode( &payload[0] ); - } - - std::vector data_out(block_size, '\0'); - decoder->copy_symbols(sak::storage(data_out)); - - bool data_equal = sak::equal(sak::storage(data_out), - sak::storage(data_in)); - - ASSERT_TRUE(data_equal); - - } - -} - - - -template -inline void -invoke_systematic(uint32_t symbols, uint32_t symbol_size) -{ - - // Common setting - typename Encoder::factory encoder_factory(symbols, symbol_size); - auto encoder = encoder_factory.build(); - - typename Decoder::factory decoder_factory(symbols, symbol_size); - auto decoder = decoder_factory.build(); - - // Encode/decode operations - EXPECT_TRUE(encoder->payload_size() == decoder->payload_size()); - - std::vector payload(encoder->payload_size()); - std::vector data_in = random_vector(encoder->block_size()); - - encoder->set_symbols(sak::storage(data_in)); - - // Ensure encoder systematic - EXPECT_TRUE(kodo::is_systematic_encoder(encoder)); - kodo::set_systematic_on(encoder); - - uint32_t pkg_count = 0; - - while( !decoder->is_complete() ) - { - encoder->encode( &payload[0] ); - decoder->decode( &payload[0] ); - - ++pkg_count; - } - - EXPECT_TRUE(pkg_count == encoder->symbols()); - - std::vector data_out(decoder->block_size(), '\0'); - decoder->copy_symbols(sak::storage(data_out)); - - EXPECT_TRUE(std::equal(data_out.begin(), - data_out.end(), - data_in.begin())); - -} - - - - - - diff --git a/test/src/helper_test_basic_api.hpp b/test/src/helper_test_basic_api.hpp new file mode 100644 index 00000000..748ae8cb --- /dev/null +++ b/test/src/helper_test_basic_api.hpp @@ -0,0 +1,153 @@ +// Copyright Steinwurf ApS 2011-2012. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + +#pragma once + +#include "basic_api_test_helper.hpp" + +template +inline void test_basic_api(uint32_t symbols, uint32_t symbol_size) +{ + + // Common setting + typename Encoder::factory encoder_factory(symbols, symbol_size); + auto encoder = encoder_factory.build(); + + typename Decoder::factory decoder_factory(symbols, symbol_size); + auto decoder = decoder_factory.build(); + + EXPECT_TRUE(symbols == encoder_factory.max_symbols()); + EXPECT_TRUE(symbol_size == encoder_factory.max_symbol_size()); + EXPECT_TRUE(symbols == encoder->symbols()); + EXPECT_TRUE(symbol_size == encoder->symbol_size()); + + EXPECT_TRUE(symbols == decoder_factory.max_symbols()); + EXPECT_TRUE(symbol_size == decoder_factory.max_symbol_size()); + EXPECT_TRUE(symbols == decoder->symbols()); + EXPECT_TRUE(symbol_size == decoder->symbol_size()); + + EXPECT_TRUE(encoder->symbol_length() > 0); + EXPECT_TRUE(decoder->symbol_length() > 0); + + EXPECT_TRUE(encoder->block_size() == symbols * symbol_size); + EXPECT_TRUE(decoder->block_size() == symbols * symbol_size); + + EXPECT_TRUE(encoder_factory.max_payload_size() >= + encoder->payload_size()); + + EXPECT_TRUE(decoder_factory.max_payload_size() >= + decoder->payload_size()); + + EXPECT_EQ(encoder_factory.max_payload_size(), + decoder_factory.max_payload_size()); + + // Encode/decode operations + EXPECT_EQ(encoder->payload_size(), decoder->payload_size()); + + std::vector payload(encoder->payload_size()); + + std::vector data_in = random_vector(encoder->block_size()); + std::vector data_in_copy(data_in); + + sak::mutable_storage storage_in = sak::storage(data_in); + sak::mutable_storage storage_in_copy = sak::storage(data_in_copy); + + EXPECT_TRUE(sak::equal(storage_in, storage_in_copy)); + + // Only used for prime fields, lets reconsider how we implement + // this less intrusive + uint32_t prefix = 0; + + if(fifi::is_prime2325::value) + { + // This field only works for multiple of uint32_t + assert((encoder->block_size() % 4) == 0); + + uint32_t block_length = encoder->block_size() / 4; + + fifi::prime2325_binary_search search(block_length); + prefix = search.find_prefix(storage_in_copy); + + // Apply the negated prefix + fifi::apply_prefix(storage_in_copy, ~prefix); + } + + encoder->set_symbols(storage_in_copy); + + // Set the encoder non-systematic + if(kodo::is_systematic_encoder(encoder)) + kodo::set_systematic_off(encoder); + + while( !decoder->is_complete() ) + { + uint32_t payload_used = encoder->encode( &payload[0] ); + EXPECT_TRUE(payload_used <= encoder->payload_size()); + + decoder->decode( &payload[0] ); + } + + std::vector data_out(decoder->block_size(), '\0'); + decoder->copy_symbols(sak::storage(data_out)); + + if(fifi::is_prime2325::value) + { + // Now we have to apply the negated prefix to the decoded data + fifi::apply_prefix(sak::storage(data_out), ~prefix); + } + + EXPECT_TRUE(std::equal(data_out.begin(), + data_out.end(), + data_in.begin())); +} + +template +< + template class Encoder, + template class Decoder +> +inline void test_basic_api(uint32_t symbols, uint32_t symbol_size) +{ + test_basic_api + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_basic_api + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_basic_api + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_basic_api + < + Encoder, + Decoder + >(symbols, symbol_size); +} + +template +< + template class Encoder, + template class Decoder +> +inline void test_basic_api() +{ + + test_basic_api(32, 1600); + test_basic_api(1, 1600); + + uint32_t symbols = rand_symbols(); + uint32_t symbol_size = rand_symbol_size(); + + test_basic_api(symbols, symbol_size); +} + diff --git a/test/src/helper_test_initialize_api.hpp b/test/src/helper_test_initialize_api.hpp new file mode 100644 index 00000000..a144f9e7 --- /dev/null +++ b/test/src/helper_test_initialize_api.hpp @@ -0,0 +1,113 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + + +/// @file test_rlnc_on_the_fly_codes.cpp Unit tests for the full +/// vector codes (i.e. Network Coding encoders and decoders). + +#include + +#include + +/// Checks that the encoders and decoders are in a clean state after using +/// the initialize function. +template +inline void test_initialize(uint32_t symbols, uint32_t symbol_size) +{ + + // Common setting + typename Encoder::factory encoder_factory(symbols, symbol_size); + auto encoder = encoder_factory.build(); + + typename Decoder::factory decoder_factory(symbols, symbol_size); + auto decoder = decoder_factory.build(); + + + for(uint32_t i = 0; i < 10; ++i) + { + encoder->initialize(encoder_factory); + decoder->initialize(decoder_factory); + + std::vector payload(encoder->payload_size()); + + // Ensure that the we may re-use the encoder also with partial + // data. + uint32_t block_size = rand_nonzero(encoder->block_size()); + + std::vector data_in = random_vector(block_size); + + encoder->set_symbols(sak::storage(data_in)); + + // Set the encoder non-systematic + if(kodo::is_systematic_encoder(encoder)) + kodo::set_systematic_off(encoder); + + while( !decoder->is_complete() ) + { + uint32_t payload_used = encoder->encode( &payload[0] ); + EXPECT_TRUE(payload_used <= encoder->payload_size()); + + decoder->decode( &payload[0] ); + } + + std::vector data_out(block_size, '\0'); + decoder->copy_symbols(sak::storage(data_out)); + + bool data_equal = sak::equal(sak::storage(data_out), + sak::storage(data_in)); + + ASSERT_TRUE(data_equal); + + } + +} + + + +template +< + template class Encoder, + template class Decoder +> +inline void test_initialize(uint32_t symbols, uint32_t symbol_size) +{ + + test_initialize + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_initialize + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_initialize + < + Encoder, + Decoder + >(symbols, symbol_size); +} + +template +< + template class Encoder, + template class Decoder +> +inline void test_initialize() +{ + test_initialize(32, 1600); + test_initialize(1, 1600); + + uint32_t symbols = rand_symbols(); + uint32_t symbol_size = rand_symbol_size(); + + test_initialize(symbols, symbol_size); +} + + + diff --git a/test/src/helper_test_mix_uncoded_api.hpp b/test/src/helper_test_mix_uncoded_api.hpp new file mode 100644 index 00000000..fc244da6 --- /dev/null +++ b/test/src/helper_test_mix_uncoded_api.hpp @@ -0,0 +1,127 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + + +/// @file test_rlnc_on_the_fly_codes.cpp Unit tests for the full +/// vector codes (i.e. Network Coding encoders and decoders). + +#include + +#include + +template +inline void test_mix_uncoded(uint32_t symbols, uint32_t symbol_size) +{ + // Common setting + typename Encoder::factory encoder_factory(symbols, symbol_size); + auto encoder = encoder_factory.build(); + + typename Decoder::factory decoder_factory(symbols, symbol_size); + auto decoder = decoder_factory.build(); + + // Encode/decode operations + EXPECT_TRUE(encoder->payload_size() == decoder->payload_size()); + + std::vector payload(encoder->payload_size()); + std::vector data_in = random_vector(encoder->block_size()); + + encoder->set_symbols(sak::storage(data_in)); + + if(kodo::is_systematic_encoder(encoder)) + kodo::set_systematic_off(encoder); + + while( !decoder->is_complete() ) + { + + if((rand() % 100) > 50) + { + encoder->encode( &payload[0] ); + decoder->decode( &payload[0] ); + } + else + { + uint32_t symbol_id = rand() % encoder->symbols(); + + if(decoder->symbol_pivot(symbol_id)) + { + continue; + } + + ASSERT_TRUE(encoder->symbol_size() <= payload.size()); + + encoder->copy_symbol( + symbol_id, sak::storage(payload)); + + const uint8_t *symbol_src = encoder->symbol(symbol_id); + const uint8_t *symbol_dest = &payload[0]; + + EXPECT_TRUE(std::equal(symbol_src, + symbol_src + encoder->symbol_size(), + symbol_dest)); + + + decoder->decode_symbol(&payload[0], symbol_id); + + } + } + + EXPECT_EQ(encoder->block_size(), decoder->block_size()); + + std::vector data_out(decoder->block_size(), '\0'); + decoder->copy_symbols(sak::storage(data_out)); + + EXPECT_TRUE(std::equal(data_out.begin(), + data_out.end(), + data_in.begin())); + +} + + +template +< + template class Encoder, + template class Decoder +> +inline void test_mix_uncoded(uint32_t symbols, uint32_t symbol_size) +{ + test_mix_uncoded + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_mix_uncoded + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_mix_uncoded + < + Encoder, + Decoder + >(symbols, symbol_size); +} + +template +< + template class Encoder, + template class Decoder +> +inline void test_mix_uncoded() +{ + test_mix_uncoded(32, 1600); + test_mix_uncoded(1, 1600); + + uint32_t symbols = rand_symbols(); + uint32_t symbol_size = rand_symbol_size(); + + test_mix_uncoded(symbols, symbol_size); +} + + + + + diff --git a/test/src/helper_on_the_fly_api.hpp b/test/src/helper_test_on_the_fly_api.hpp similarity index 100% rename from test/src/helper_on_the_fly_api.hpp rename to test/src/helper_test_on_the_fly_api.hpp diff --git a/test/src/helper_recoding_api.hpp b/test/src/helper_test_recoding_api.hpp similarity index 100% rename from test/src/helper_recoding_api.hpp rename to test/src/helper_test_recoding_api.hpp diff --git a/test/src/helper_reuse_api.hpp b/test/src/helper_test_reuse_api.hpp similarity index 99% rename from test/src/helper_reuse_api.hpp rename to test/src/helper_test_reuse_api.hpp index effb0762..8be0007d 100644 --- a/test/src/helper_reuse_api.hpp +++ b/test/src/helper_test_reuse_api.hpp @@ -42,7 +42,6 @@ inline void test_reuse_helper(Encoder encoder, Decoder decoder) } - /// Test that instantiates a number of encoders and decoders from /// the same factories template diff --git a/test/src/helper_test_systematic_api.hpp b/test/src/helper_test_systematic_api.hpp new file mode 100644 index 00000000..8306fd22 --- /dev/null +++ b/test/src/helper_test_systematic_api.hpp @@ -0,0 +1,100 @@ +// Copyright Steinwurf ApS 2011-2013. +// Distributed under the "STEINWURF RESEARCH LICENSE 1.0". +// See accompanying file LICENSE.rst or +// http://www.steinwurf.com/licensing + + +/// @file test_rlnc_on_the_fly_codes.cpp Unit tests for the full +/// vector codes (i.e. Network Coding encoders and decoders). + +#include + +#include + + +template +inline void test_systematic(uint32_t symbols, uint32_t symbol_size) +{ + // Common setting + typename Encoder::factory encoder_factory(symbols, symbol_size); + auto encoder = encoder_factory.build(); + + typename Decoder::factory decoder_factory(symbols, symbol_size); + auto decoder = decoder_factory.build(); + + // Encode/decode operations + EXPECT_TRUE(encoder->payload_size() == decoder->payload_size()); + + std::vector payload(encoder->payload_size()); + std::vector data_in = random_vector(encoder->block_size()); + + encoder->set_symbols(sak::storage(data_in)); + + // Ensure encoder systematic + EXPECT_TRUE(kodo::is_systematic_encoder(encoder)); + kodo::set_systematic_on(encoder); + + uint32_t pkg_count = 0; + + while( !decoder->is_complete() ) + { + encoder->encode( &payload[0] ); + decoder->decode( &payload[0] ); + + ++pkg_count; + } + + EXPECT_TRUE(pkg_count == encoder->symbols()); + + std::vector data_out(decoder->block_size(), '\0'); + decoder->copy_symbols(sak::storage(data_out)); + + EXPECT_TRUE(std::equal(data_out.begin(), + data_out.end(), + data_in.begin())); + +} + +template +< + template class Encoder, + template class Decoder +> +inline void test_systematic(uint32_t symbols, uint32_t symbol_size) +{ + test_systematic + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_systematic + < + Encoder, + Decoder + >(symbols, symbol_size); + + test_systematic + < + Encoder, + Decoder + >(symbols, symbol_size); +} + +template +< + template class Encoder, + template class Decoder +> +inline void test_systematic() +{ + test_systematic(32, 1600); + test_systematic(1, 1600); + + uint32_t symbols = rand_symbols(); + uint32_t symbol_size = rand_symbol_size(); + + test_systematic(symbols, symbol_size); +} + + diff --git a/test/src/test_carousel_codes.cpp b/test/src/test_carousel_codes.cpp index ed203556..1d4d2eda 100644 --- a/test/src/test_carousel_codes.cpp +++ b/test/src/test_carousel_codes.cpp @@ -11,12 +11,12 @@ #include -#include "basic_api_test_helper.hpp" +#include "helper_test_basic_api.hpp" static void test_coders(uint32_t symbols, uint32_t symbol_size) { - invoke_basic_api + test_basic_api < kodo::nocode_carousel_encoder, kodo::nocode_carousel_decoder diff --git a/test/src/test_debug_xyz.cpp b/test/src/test_debug_xyz.cpp index 6fe3b886..8bbfaf69 100644 --- a/test/src/test_debug_xyz.cpp +++ b/test/src/test_debug_xyz.cpp @@ -12,7 +12,7 @@ #include #include -#include "basic_api_test_helper.hpp" +#include "helper_test_basic_api.hpp" /// Here we define the stacks which should be tested. namespace kodo @@ -58,7 +58,7 @@ TEST(TestDebugXYZ, test_debug_linear_block_decoder) uint32_t symbols = rand_symbols(); uint32_t symbol_size = rand_symbol_size(); - invoke_basic_api + test_basic_api < kodo::full_rlnc_encoder, kodo::debug_decoder diff --git a/test/src/test_reuse.hpp b/test/src/test_reuse.hpp index cfe16eab..bd0b5f6e 100644 --- a/test/src/test_reuse.hpp +++ b/test/src/test_reuse.hpp @@ -12,8 +12,7 @@ /// Helper for the reuse test, ensures that all encoders and decoders /// produce valid data template -inline void -invoke_reuse_helper(Encoder encoder, Decoder decoder) +inline void test_reuse_helper(Encoder encoder, Decoder decoder) { std::vector payload(encoder->payload_size()); @@ -47,8 +46,7 @@ invoke_reuse_helper(Encoder encoder, Decoder decoder) /// Test that instantiates a number of encoders and decoders from /// the same factories template -inline void -invoke_reuse(uint32_t symbols, uint32_t symbol_size) +inline void test_reuse(uint32_t symbols, uint32_t symbol_size) { // Common setting typename Encoder::factory encoder_factory(symbols, symbol_size); @@ -70,7 +68,7 @@ invoke_reuse(uint32_t symbols, uint32_t symbol_size) for(uint32_t j = 0; j < coders; ++j) { - invoke_reuse_helper(encoders[j], decoders[j]); + test_reuse_helper(encoders[j], decoders[j]); } } @@ -101,7 +99,7 @@ invoke_reuse(uint32_t symbols, uint32_t symbol_size) for(uint32_t j = 0; j < coders; ++j) { - invoke_reuse_helper(encoders[j], decoders[j]); + test_reuse_helper(encoders[j], decoders[j]); } } @@ -111,25 +109,25 @@ invoke_reuse(uint32_t symbols, uint32_t symbol_size) /// Test reuse function which will invoke the reuse test for the common /// field sizes template - < +< template class Encoder, template class Decoder - > +> inline void test_reuse(uint32_t symbols, uint32_t symbol_size) { - invoke_reuse + test_reuse < Encoder, Decoder >(symbols, symbol_size); - invoke_reuse + test_reuse < Encoder, Decoder >(symbols, symbol_size); - invoke_reuse + test_reuse < Encoder, Decoder diff --git a/test/src/test_rlnc_full_vector_codes.cpp b/test/src/test_rlnc_full_vector_codes.cpp index 438fbca7..a5b5b893 100644 --- a/test/src/test_rlnc_full_vector_codes.cpp +++ b/test/src/test_rlnc_full_vector_codes.cpp @@ -22,7 +22,14 @@ #include #include "basic_api_test_helper.hpp" -#include "test_reuse.hpp" + +#include "helper_test_reuse_api.hpp" +#include "helper_test_recoding_api.hpp" +#include "helper_test_on_the_fly_api.hpp" +#include "helper_test_basic_api.hpp" +#include "helper_test_initialize_api.hpp" +#include "helper_test_systematic_api.hpp" +#include "helper_test_mix_uncoded_api.hpp" namespace kodo { @@ -129,489 +136,98 @@ namespace kodo } -template