diff --git a/.outline-sync.json b/.outline-sync.json index dd34e79c..f8ebae67 100644 --- a/.outline-sync.json +++ b/.outline-sync.json @@ -18,17 +18,13 @@ "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/contributing/00-who-are-we.md": "747223f36677931db50fa7e539dafecb01a10834", "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/contributing/01-the-stack.md": "5b18c75e6ee188adb465a1c2d2fab4024eb714f6", "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/contributing/03-best-practices.md": "197df8c11f46748b758a0aea1c4805020c57c7fc", - "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/contributing/04-deploy-app-release-process/index.md": "a8aa2e7e532fdb97fe0be8e8307f04684e1aab3a", - "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/contributing/04-deploy-app-release-process/1-submodule-development-process.md": "8c51a0966a71ae734e80472ca0b04685cb0e8f1f", - "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/contributing/04-deploy-app-release-process/2-submodule-merge-process.md": "6c5a719524dd6bba823e8dbe9ed5b56805948bd7", - "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/contributing/04-deploy-app-release-process/3-integration-repo-merge-process.md": "5777b11945f58798f67605573b544ff23e101ced", "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/contributing/05-bugs-vulns.md": "6168d833ad91d07374415452db4d9ea3b263c5dd", "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/contributing/06-contribution-agreement.md": "c71da81dea6d82f059a2365abe91b62178551461", "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/specs/01-deploy-app/index.md": "a2a7b6e303bf3eacdb0cbda40f33a3a786e58c98", "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/specs/03-battlelog.md": "030d71cae9eb93e9c133ea13a7380e935a211377", "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/roadmap/01-pvarki.md": "73af6b939f4c6cd5447991cc81aa947354bf449a", "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/roadmap/02-deploy-app.md": "30fdf62740d3790c129bcdac2220c809bda61d3b", - "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/roadmap/03-tak.md": "176f18e80b5b57b97a728da0cc5842f70bc985d4", + "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/roadmap/03-tak.md": "0bced3136be8f5be3714def62f408dfaf5e1c227", "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/setupguide/preparations.md": "a7473d176bc68d8ea14943da2870dd82d0c714c6", "/Users/bg/PVARKI-projekti/esitesivu/Docusaurus-docs/docs/dev/specs/01-deploy-app/00-integration-spec-overview.md": "77e40364b86e342ec4009b0d8c0afe7e2241c688", "/Users/bg/PVARKI-projekti/esitesivu/Docusaurus-docs/docs/dev/specs/01-deploy-app/01-integration-spec-compose.md": "02c7914fe7a9bd784580ca6d2fad006f6b62254b", @@ -50,6 +46,10 @@ "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/specs/01-deploy-app/product-integrations/02-battlelog-integration.md": "23c4753f98a5e99afc145b4919a2b1afd024fde6", "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/specs/01-deploy-app/product-integrations/03-mediamtx-video-streaming-integration.md": "494dc44bb2284c614f59326fe5d4255d5fc6b11a", "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/specs/01-deploy-app/product-integrations/04-docs-integration.md": "a89388a5ec380ae211273b3ea3383d5956009f62", - "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/specs/02-docs.md": "3435a2d908f7f92dc3800bf4bb0f38d7a969b564" + "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/specs/02-docs.md": "3435a2d908f7f92dc3800bf4bb0f38d7a969b564", + "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/specs/01-deploy-app/deploy-app-release-process/index.md": "e431790720ac93f90c87981c27c5a773ad4e4e98", + "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/specs/01-deploy-app/deploy-app-release-process/1-submodule-development-process.md": "8c51a0966a71ae734e80472ca0b04685cb0e8f1f", + "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/specs/01-deploy-app/deploy-app-release-process/2-submodule-merge-process.md": "6c5a719524dd6bba823e8dbe9ed5b56805948bd7", + "/home/runner/work/Docusaurus-docs/Docusaurus-docs/docs/dev/specs/01-deploy-app/deploy-app-release-process/3-integration-repo-merge-process.md": "5777b11945f58798f67605573b544ff23e101ced" } } diff --git a/docs/dev/integrationrepo/index.md b/docs/dev/integrationrepo/index.md index ad90ecb6..00b14bc5 100644 --- a/docs/dev/integrationrepo/index.md +++ b/docs/dev/integrationrepo/index.md @@ -4,6 +4,12 @@ title: "Repositories & README’s" Below are the integration repo versions: +- [1.15.2](./v/1.15.2/index.md) +- [1.15.1](./v/1.15.1/index.md) +- [1.15.0](./v/1.15.0/index.md) +- [1.14.0](./v/1.14.0/index.md) +- [1.13.0](./v/1.13.0/index.md) +- [1.12.1](./v/1.12.1/index.md) - [1.12.0](./v/1.12.0/index.md) - [1.11.1](./v/1.11.1/index.md) - [1.11.0](./v/1.11.0/index.md) @@ -18,9 +24,3 @@ Below are the integration repo versions: - [1.6.0](./v/1.6.0/index.md) - [1.5.1](./v/1.5.1/index.md) - [1.5.0](./v/1.5.0/index.md) -- [1.4.1](./v/1.4.1/index.md) -- [1.4.0](./v/1.4.0/index.md) -- [1.3.0](./v/1.3.0/index.md) -- [1.2.1](./v/1.2.1/index.md) -- [1.2.0](./v/1.2.0/index.md) -- [1.1.1](./v/1.1.1/index.md) diff --git a/docs/dev/integrationrepo/v/1.10.0/_category_.json b/docs/dev/integrationrepo/v/1.10.0/_category_.json index a437cdf9..7d6c346c 100644 --- a/docs/dev/integrationrepo/v/1.10.0/_category_.json +++ b/docs/dev/integrationrepo/v/1.10.0/_category_.json @@ -1,4 +1,4 @@ { "label": "1.10.0", - "position": 5 + "position": 11 } diff --git a/docs/dev/integrationrepo/v/1.10.0/submodules/api.md b/docs/dev/integrationrepo/v/1.10.0/submodules/api.md index 0944439e..07da61e2 100644 --- a/docs/dev/integrationrepo/v/1.10.0/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.10.0/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.10.0/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.10.0/submodules/battlelog.md index 36ff2b17..e89ade78 100644 --- a/docs/dev/integrationrepo/v/1.10.0/submodules/battlelog.md +++ b/docs/dev/integrationrepo/v/1.10.0/submodules/battlelog.md @@ -6,7 +6,7 @@ title: "pvarki/typescript-liveloki-app – README" > **Repo:** git@github.com:pvarki/typescript-liveloki-app.git > **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/197f6b74507fb81fc442fa1522088f0cb7ed3939 -# livelogi +# BattleLog ## Install diff --git a/docs/dev/integrationrepo/v/1.10.0/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.10.0/submodules/fpintegration.md index 885e3c88..1c4751d6 100644 --- a/docs/dev/integrationrepo/v/1.10.0/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.10.0/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.10.1/_category_.json b/docs/dev/integrationrepo/v/1.10.1/_category_.json index 80c69627..64f1c6a6 100644 --- a/docs/dev/integrationrepo/v/1.10.1/_category_.json +++ b/docs/dev/integrationrepo/v/1.10.1/_category_.json @@ -1,4 +1,4 @@ { "label": "1.10.1", - "position": 4 + "position": 10 } diff --git a/docs/dev/integrationrepo/v/1.10.1/submodules/api.md b/docs/dev/integrationrepo/v/1.10.1/submodules/api.md index 6b3152aa..5eb10ec6 100644 --- a/docs/dev/integrationrepo/v/1.10.1/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.10.1/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.10.1/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.10.1/submodules/battlelog.md index 449668ab..4e2f35fc 100644 --- a/docs/dev/integrationrepo/v/1.10.1/submodules/battlelog.md +++ b/docs/dev/integrationrepo/v/1.10.1/submodules/battlelog.md @@ -6,7 +6,7 @@ title: "pvarki/typescript-liveloki-app – README" > **Repo:** git@github.com:pvarki/typescript-liveloki-app.git > **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/197f6b74507fb81fc442fa1522088f0cb7ed3939 -# livelogi +# BattleLog ## Install diff --git a/docs/dev/integrationrepo/v/1.10.1/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.10.1/submodules/fpintegration.md index a8e54e26..51971d5e 100644 --- a/docs/dev/integrationrepo/v/1.10.1/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.10.1/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.11.0/_category_.json b/docs/dev/integrationrepo/v/1.11.0/_category_.json index b06e9ea6..f88ae652 100644 --- a/docs/dev/integrationrepo/v/1.11.0/_category_.json +++ b/docs/dev/integrationrepo/v/1.11.0/_category_.json @@ -1,4 +1,4 @@ { "label": "1.11.0", - "position": 3 + "position": 9 } diff --git a/docs/dev/integrationrepo/v/1.11.0/submodules/api.md b/docs/dev/integrationrepo/v/1.11.0/submodules/api.md index a12fb870..b467ab58 100644 --- a/docs/dev/integrationrepo/v/1.11.0/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.11.0/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.11.0/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.11.0/submodules/battlelog.md index f9571dd9..bb71cfa9 100644 --- a/docs/dev/integrationrepo/v/1.11.0/submodules/battlelog.md +++ b/docs/dev/integrationrepo/v/1.11.0/submodules/battlelog.md @@ -6,7 +6,7 @@ title: "pvarki/typescript-liveloki-app – README" > **Repo:** git@github.com:pvarki/typescript-liveloki-app.git > **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/197f6b74507fb81fc442fa1522088f0cb7ed3939 -# livelogi +# BattleLog ## Install diff --git a/docs/dev/integrationrepo/v/1.11.0/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.11.0/submodules/fpintegration.md index dd2ab796..9874bc0d 100644 --- a/docs/dev/integrationrepo/v/1.11.0/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.11.0/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.11.1/_category_.json b/docs/dev/integrationrepo/v/1.11.1/_category_.json index 679d9763..cb97326a 100644 --- a/docs/dev/integrationrepo/v/1.11.1/_category_.json +++ b/docs/dev/integrationrepo/v/1.11.1/_category_.json @@ -1,4 +1,4 @@ { "label": "1.11.1", - "position": 2 + "position": 8 } diff --git a/docs/dev/integrationrepo/v/1.11.1/submodules/api.md b/docs/dev/integrationrepo/v/1.11.1/submodules/api.md index 5c405da3..b92f322e 100644 --- a/docs/dev/integrationrepo/v/1.11.1/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.11.1/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.11.1/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.11.1/submodules/battlelog.md index 5bc39fde..d5c7431b 100644 --- a/docs/dev/integrationrepo/v/1.11.1/submodules/battlelog.md +++ b/docs/dev/integrationrepo/v/1.11.1/submodules/battlelog.md @@ -6,7 +6,7 @@ title: "pvarki/typescript-liveloki-app – README" > **Repo:** git@github.com:pvarki/typescript-liveloki-app.git > **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/197f6b74507fb81fc442fa1522088f0cb7ed3939 -# livelogi +# BattleLog ## Install diff --git a/docs/dev/integrationrepo/v/1.11.1/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.11.1/submodules/fpintegration.md index 8dec3780..be1da3e0 100644 --- a/docs/dev/integrationrepo/v/1.11.1/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.11.1/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.12.0/_category_.json b/docs/dev/integrationrepo/v/1.12.0/_category_.json index 71876d5b..693bdc5a 100644 --- a/docs/dev/integrationrepo/v/1.12.0/_category_.json +++ b/docs/dev/integrationrepo/v/1.12.0/_category_.json @@ -1,4 +1,4 @@ { "label": "1.12.0", - "position": 1 + "position": 7 } diff --git a/docs/dev/integrationrepo/v/1.12.0/submodules/api.md b/docs/dev/integrationrepo/v/1.12.0/submodules/api.md index 229ae98c..cc01f20f 100644 --- a/docs/dev/integrationrepo/v/1.12.0/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.12.0/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.12.0/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.12.0/submodules/battlelog.md index 2c977217..6e16c435 100644 --- a/docs/dev/integrationrepo/v/1.12.0/submodules/battlelog.md +++ b/docs/dev/integrationrepo/v/1.12.0/submodules/battlelog.md @@ -6,7 +6,7 @@ title: "pvarki/typescript-liveloki-app – README" > **Repo:** git@github.com:pvarki/typescript-liveloki-app.git > **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/197f6b74507fb81fc442fa1522088f0cb7ed3939 -# livelogi +# BattleLog ## Install diff --git a/docs/dev/integrationrepo/v/1.12.0/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.12.0/submodules/fpintegration.md index fccdf0b6..6588a63c 100644 --- a/docs/dev/integrationrepo/v/1.12.0/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.12.0/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.12.0/submodules/mtxauthz.md b/docs/dev/integrationrepo/v/1.12.0/submodules/mtxauthz.md index d1fc0cff..7d579c2f 100644 --- a/docs/dev/integrationrepo/v/1.12.0/submodules/mtxauthz.md +++ b/docs/dev/integrationrepo/v/1.12.0/submodules/mtxauthz.md @@ -39,9 +39,17 @@ and Linux: Build image, create container and start it: docker build --ssh default --target devel_shell -t rmmtxauthz:devel_shell . - docker create --name rmmtxauthz_devel -v "$(pwd):/app" -it $(echo $DOCKER_SSHAGENT) rmmtxauthz:devel_shell + docker create --name rmmtxauthz_devel -v "$(pwd)/rune/output/rune.json:/opt/templates/mediamtx.json" -v "$(pwd):/app" -it $(echo $DOCKER_SSHAGENT) rmmtxauthz:devel_shell docker start -i rmmtxauthz_devel +To rebuild the documentation inside the container run: + + rune rune/src json >/opt/templates/mediamtx.json + +Outside of container use: + + rune rune/src json >rune/output/rune.json + ### pre-commit considerations If working in Docker instead of native env you need to run the @@ -106,3 +114,9 @@ Running "pre-commit run --all-files" and "py.test -v" regularly during development and especially before committing will save you some headache. +### RUNE instructions compile + +tldr: + + rune rune/src json >rune/output/mediamtx.json + diff --git a/docs/dev/integrationrepo/v/1.12.1/_category_.json b/docs/dev/integrationrepo/v/1.12.1/_category_.json new file mode 100644 index 00000000..6c51ef42 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "1.12.1", + "position": 6 +} diff --git a/docs/dev/integrationrepo/v/1.12.1/index.md b/docs/dev/integrationrepo/v/1.12.1/index.md new file mode 100644 index 00000000..e8d65b0f --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/index.md @@ -0,0 +1,4 @@ +--- +title: "1.12.1" +--- + diff --git a/docs/dev/integrationrepo/v/1.12.1/integration.md b/docs/dev/integrationrepo/v/1.12.1/integration.md new file mode 100644 index 00000000..cd39fff8 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/integration.md @@ -0,0 +1,304 @@ +--- +title: "pvarki/docker-rasenmaeher-integration – README" +--- + +> **Integration tag:** `1.12.1` +> **Repo:** https://github.com/pvarki/docker-rasenmaeher-integration + +![Build Status](https://github.com/pvarki/docker-rasenmaeher-integration/actions/workflows/build.yml/badge.svg) + +# Deploy App + +"One Ring to rule them all, One Ring to find them, One Ring to bring +them all and in the darkness bind them." + +Docker compositions, helpers etc to bring it all together into something +resembling grand old ones. + +Codename RASENMAEHER because infantry jokes. + +## WTF is this anyway ? + +This [Disobey24 +talk](https://www.youtube.com/watch?v=m3xd7uygpaY&list=PLLvAhAn5sGfiB9AlEt2KD7H9Dnr6kbd64&index=23) +explains a lot. + +## Running Deploy App in your own docker environment + +### Needed DNS Records + +These need to point to your WAN address. + +> - domain +> - kc.domain +> - tak.domain +> - bl.domain +> - mtls.domain +> - mtls.tak.domain +> - mtls.kc.domain +> - mtls.bl.domain +> - kc.tak.domain + +When more products are added to the deployment they will follow the same +naming pattern, you will need subdomains for all products listed in the +composition for miniwerk service variable MW_PRODUCTS and "kc" for +Keycloak. + +### Needed ports open + +And redirected to the server if behind NAT or similar. + +> - 80 +> - 443 +> - 8443 (TAK) +> - 8446 (TAK) +> - 9446 (Keycloak) +> - 4626 (Product integration APIs port) +> - 4666 (Battlelog API/UI port) + +### Downloading and composing Deploy App + +Be mindfull on where you download the repository, you will need to +perform rest of the commands inside the downloaded repository. + +Getting the repository from github (on Windows **first** see "Windows +notes" below): + + git clone --recurse-submodules -j8 git@github.com:pvarki/docker-rasenmaeher-integration.git + +Create `.env` file that defines environmental variables for Deploy App +setup. File must be located inside downloaded repository and file type +must be named literally `.env` (not `something.env`) to work. + +The original example file is: +[https://github.com/pvarki/docker-rasenmaeher-integration/blob/main/example_env.sh](https://github.com/pvarki/docker-rasenmaeher-integration/blob/main/example_env.sh) + +Example .env-file with the minimal information needed: + + KEYCLOAK_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RM_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + POSTGRES_PASSWORD="input-secure-password" # pragma: allowlist secret + LDAP_ADMIN_PASSWORD="input-secure-password" # pragma: allowlist secret + KEYCLOAK_ADMIN_PASSWORD="input-secure-password" # pragma: allowlist secret + TAK_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + SERVER_DOMAIN="input-domain" + CFSSL_CA_NAME="input-ca-name" + MW_LE_EMAIL="input-email-for-lets-encrypt" + MW_LE_TEST="false" + TAKSERVER_CERT_PASS="input-secure-password" # pragma: allowlist secret + TAK_CA_PASS="input-secure-password" # pragma: allowlist secret + VITE_ASSET_SET="${VITE_ASSET_SET:-neutral}" + KEYCLOAK_PROFILEROOT_UUID="input-uuid" + KEYCLOAK_HTTPS_KEY_STORE_PASSWORD="input-secure-password" # pragma: allowlist secret + KEYCLOAK_HTTPS_TRUST_STORE_PASSWORD="input-secure-password" # pragma: allowlist secret + BL_POSTGRES_PASSWORD="input-secure-password" # pragma: allowlist secret + RMMTX_POSTGRES_PASSWORD="input-secure-password" # pragma: allowlist secret + +Replace "intput-secure-password" with a good passphrase that is unique +for each replacment. + +If you wish to use one deployment for longer than the *design lifetime* +of 1-2 months you can change the following env variables. But do +understand that this is **not recommended** and has **security +implications**. If you do this **you** take **responsibility** to go +through all Dockerfiles and compositions to understand **exactly** how +things are done and how apply security updates into the containers: + + CFSSL_CA_EXPIRY="8800h" # Input time in hours = xxh, 8800h is 366.. days + CFSSL_SIGN_DEFAULT_EXPIRY="8800h" # Input time in hours = xxh, 8800h is 366.. days + +Starting the services: + + docker compose up -d + +Updating the repository from github: + + git submodule update + +!DO NOT DO! Deleting the services. Deletes the certificates etc you will +need to add all users etc again: + + docker compose down -v + +Getting the admin login invite code for first admin: + + docker compose exec -it rmapi /bin/bash -c "rasenmaeher_api addcode" + +If you get "no such service -d", make sure whatever you copy-pasted the +command from did not render the dash (ASCII 0x2D) as some other +codepoint with a glyph that looks deceptively similar. When in doubt +write the commands yourself instead of copy-pasting. + +### Services + +Deploy App login page: + + https://domain (example.com) + +Deploy App home page: + + https://mtls.domain (mtls.example.com) + +Takserver Admin UI: + + https://tak.domain:8443/ (tak.example.com:8443/) + +Keycloack Admin UI. (Later group management will be withing Deploy App): + + https://kc.domain:9443/admin/RASENMAEHER/console/ (kc.example.com:9443/admin/RASENMAEHER/console/) + +OTA update server inside takserver. Is located in the loaded repository, +location depends on where you downloaded it: + + /home/user/docker-rasenmaeher-integration/takserver/update + +### Using the Deploy App service + +1. Login with first admin code. Create your admin account by typing + your first admin invite code and inputting desired admin callsign. +2. Create invite code for other users. Share the invite code. Go to + Manage Users -\> Add Users -\> Create New Invite. Share link, qr + code or invite code and domain. +3. Approve users in Deploy App. Open approvement link or scan qr code + from users and approve the user. You can also go to Approve Users + -\> Select Waiting User and input the users approvement code. +4. If desired promote some of the added users as admins. Go to Manage + Users -\> Manage Users -\> Select user and select Promote. You can + also Demote Admins or Delete users altogether. + +### Using Deploy App TAK in EUD + +EUD=End User Device + +1. Login to Deploy App. Go to [https://mtls.domain](https://mtls.domain) and select TAK. +2. Download Client Package. Select tak package for desired software + "Android ATAK or Windows WinTAK" or "iOS iTAK". Select Download + Client Package. +3. Go to EUD's TAK Software. Import downloaded package. Device is + connected to server. +4. You should also read Quickstart and Usage Guides. + +## Git submodules + +When cloning for the first time use: + + git clone --recurse-submodules -j8 git@github.com:pvarki/docker-rasenmaeher-integration.git + +When updating or checking out branches use: + + git submodule update --init --recursive + +And if you forgot to --recurse-submodules run the update command above. + +The submodules are repos in their own right, if you plan to make changes +into them change to the directory and create new branch, commit and push +changes as usual under that directory. + +### Directories that are submodules + +> - api [https://github.com/pvarki/python-rasenmaeher-api](https://github.com/pvarki/python-rasenmaeher-api) +> - cfssl [https://github.com/pvarki/docker-rasenmaeher-cfssl](https://github.com/pvarki/docker-rasenmaeher-cfssl) +> - fpintegration [https://github.com/pvarki/python-rasenmaeher-rmfpapi](https://github.com/pvarki/python-rasenmaeher-rmfpapi) +> - keycloak [https://github.com/pvarki/docker-keycloak](https://github.com/pvarki/docker-keycloak) +> - kw_product_init +> [https://github.com/pvarki/golang-kraftwerk-init-helper-cli](https://github.com/pvarki/golang-kraftwerk-init-helper-cli) +> - openldap [https://github.com/pvarki/docker-openldap](https://github.com/pvarki/docker-openldap) +> - miniwerk [https://github.com/pvarki/python-rasenmaeher-miniwerk](https://github.com/pvarki/python-rasenmaeher-miniwerk) +> - ui [https://github.com/pvarki/rasenmaeher-ui](https://github.com/pvarki/rasenmaeher-ui) +> - takserver [https://github.com/pvarki/docker-atak-server](https://github.com/pvarki/docker-atak-server) +> - takintegration [https://github.com/pvarki/python-tak-rmapi](https://github.com/pvarki/python-tak-rmapi) +> - battlelog [https://github.com/pvarki/typescript-liveloki-app](https://github.com/pvarki/typescript-liveloki-app) + +## Autogenerated (mostly API) docs + +> - Module API docs: +> [https://pvarki.github.io/docker-rasenmaeher-integration/docs/](https://pvarki.github.io/docker-rasenmaeher-integration/docs/) +> - Swagger definition for Deploy App API: +> [https://pvarki.github.io/docker-rasenmaeher-integration/](https://pvarki.github.io/docker-rasenmaeher-integration/) + +## Running in local development mode + +### Windows notes + +> 1. Do **NOT** use git-bash, it will cause *weirdest* issues with +> Docker containers +> +> 2. Use WSL, see +> [best_practises](https://github.com/pvarki/markdown-pvarki-best_practises) +> -repo for instructions on how to set it up. +> +> 3. Make sure whatever git client or IDE you use it does not mess with +> line-endings, for CLI client this does the trick: +> +> git config --global core.eol lf +> git config --global core.autocrlf false + +### Compositions + +Generally start with "rmlocal", it corresponds best to a real running +environment. "rmdev" starts a bunch of things in development mode which +does make developing more convenient but also introduces extra +variability to how things work. + +Make sure to always check your changes work correctly in rmlocal mode +where assets are minified and baked in. + +TLDR: + + alias rmlocal="docker compose -p rmlocal -f docker-compose-local.yml" + rmlocal build takinit + rmlocal build + rmlocal up + +or: + + alias rmdev="docker compose -p rmdev -f docker-compose-local.yml -f docker-compose-dev.yml" + rmdev build takinit + rmdev build + rmdev up + +OpenLDAP and keycloak-init sometimes fail on first start, just run up +again. + +IMPORTANT: Only keep either rmlocal or rmdev created at one time or you +may have weird network issues run "down" for one env before starting the +other. + +Remember to run "down -v" if you want to reset the persistent volumes, +or if you have weird issues when switching between environments. + +The dev version launches all the services and runs rasenmaeher-api in +uvicorn reload mode so any edits you make under /api will soon be +reflected in the running instance. + +If rasenmaeher-ui devel server complains make sure to delete +`ui/node_modules` -directory from host first. The docker NodeJS +distribution probably is not compatible with whatever you have installed +on the host. + +### Gaining first admin access in dev and production mode + +In dev mode: + + docker exec -it rmdev-rmapi-1 /bin/bash -c "source /.venv/bin/activate && rasenmaeher_api addcode" + +Under dev mode, the UI runs at [https://localmaeher.dev.pvarki.fi:4439](https://localmaeher.dev.pvarki.fi:4439). + +In VM production mode: + + docker exec -it rmvm-rmapi-1 /bin/bash -c "rasenmaeher_api addcode" + +## pre-commit notes + +Use "pre-commit run --all-files" liberally (and make sure you have run +"pre-commit install --install-hooks"). If you get complaints about +missing environment variables run "source example_env.sh" + +## Integration tests + +Pytest is used to handle the integration tests, the requirements are in +tests/requirements.txt. NOTE: The tests have side-effects and expect a +clean database to start with so always make sure to run "down -v" for +the composition first, then bring it back up before running integration +tests. + diff --git a/docs/dev/integrationrepo/v/1.12.1/submodules/api.md b/docs/dev/integrationrepo/v/1.12.1/submodules/api.md new file mode 100644 index 00000000..c028d99a --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/submodules/api.md @@ -0,0 +1,240 @@ +--- +title: "pvarki/python-rasenmaeher-api – README" +--- + +> **Integration tag:** `1.12.1` · **Submodule commit:** `a7e2a5e966e2957ebb2f108d40ebaa32bfc3b164` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-api.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-api/tree/a7e2a5e966e2957ebb2f108d40ebaa32bfc3b164 + +# python-rasenmaeher-api + +python-rasenmaeher-api + +## Configuration + +This application can be configured with environment variables. Below is +a example list of available variables. You can find all available +variables here +[https://github.com/pvarki/python-rasenmaeher-api/blob/main/src/rasenmaeher_api/settings.py](https://github.com/pvarki/python-rasenmaeher-api/blob/main/src/rasenmaeher_api/settings.py). + +| ENV VAR | Default value | Info / Usage | +|---------------------------|---------------------------------------------|---------------------------------------------------------------| +| RM_PORT | 8000 | Docker API listen port | +| RM_WORKERS_COUNT | 2 | Number of uvicorn workers | +| RM_RELOAD | False | Reload service on code change | +| RM_ENVIRONMENT | dev | Run dev / prod environment | +| RM_CFSSL_HOST | None | CFSSL host url | +| RM_CFSSL_PORT | None | CFSSL service port | +| RM_KEYCLOAK_SERVER_URL | None | Keycloak server url ([http://1234:8080/auth](http://1234:8080/auth)) | +| RM_KEYCLOAK_CLIENT_ID | None | Keycloak client id | +| RM_KEYCLOAK_REALM_NAME | None | Keycloak realm name | +| RM_KEYCLOAK_CLIENT_SECRET | None | Keycloak secert | +| RM_LDAP_CONN_STRING | None | LDAP conn string | +| RM_LDAP_USERNAME | None | LDAP username | +| RM_LDAP_CLIENT_SECRET | None | LDAP connection secret | +| RM_SQLITE_FILEPATH_PROD | /opt/rasenmaher/persistent/sqlite/rm_db.sql | location for sqlite database file in "prod" | +| RM_SQLITE_FILEPATH_DEV | /tmp/rm_db.sql | location for sqlite database file in "dev", local development | + +Container Variables + +You can create .env file in the root +directory and place all environment variables here. + +All environment variables should start with +RM\_ prefix. + +For example if you see in your "rasenmaeher_api/settings.py" a variable +named like random_parameter, you should +provide the "RM_RANDOM_PARAMETER" variable to configure the value. This +behaviour can be changed by overriding +env_prefix property in +rasenmaeher_api.settings.Settings.Config. + +An example of .env file: +`` `bash RM_RELOAD="True" RM_PORT="8000" RM_ENVIRONMENT="dev" ``\` + +You can read more about BaseSettings class here: +[https://pydantic-docs.helpmanual.io/usage/settings/](https://pydantic-docs.helpmanual.io/usage/settings/) + +## Backend services manifest + +Integration APIs are always https in port 4625. Default location for the +manifest is /pvarki/kraftwerk-rasenmaeher-init.json + +`` `json { "dns": "sleepy-sloth.pvarki.fi", "products": { "tak": "tak.sleepy-sloth.pvarki.fi", "ptt": "ptt.sleepy-sloth.pvarki.fi" } } ``\` + +## Api endpoints and usage + +| State | Method | URI | Request JSON | Response JSON | Api description . | +|---------|--------|------------------------------------|--------------------------------------------------------------|-------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| Dummy | GET | /api/v1/enroll/status/{work_id} | NA | {'status':'None/Processing/Denied/WaitingForAcceptance/ReadyForDelivery/Delivered'} | Check the situation of enrollment process, None = no enrollment started, this work_id is free to use. | +| Dummy | POST | /api/v1/enroll/init | {'work_id':'{work_id}'} | {'work_id':'{work_id}', 'id_hash':{id_string} } | Start service access enrollment for given {work_id} | +| Dummy | GET | /api/v1/enroll/deliver/{id_string} | NA | {'dl_link':"{[http://here.be/zip](http://here.be/zip)}"} | Deliver download link for enrollment zip | +| Dummy | POST | /api/v1/enroll/accept | { 'permit_string':'{permit_string}, 'id_hash':'{id_hash}' '} | { 'access':'granted/denied/None', 'work_id':'{work_id}' } | Accept the enrollment request | +| Testing | POST | /api/v1/product/sign_csr | { 'TO':'DO'} | { 'TO':'DO'} | Accept the enrollment request | +| Testing | POST | /api/v1/enroll/accept | { 'permit_string':'{permit_string}, 'id_hash':'{id_hash}' '} | { 'access':'granted/denied/None', 'work_id':'{work_id}' } | Accept the enrollment request | + +API vimpain + +## Example usage + +\# REQUEST A NEW CERTIFICATE USING CSR (requires cfssl backend for the +api container) + +> `` `curl -L -H "Content-Type: application/json" -d '{"csr": "-----BEGIN CERTIFICATE REQUEST-----\nMIIBUjCB+QIBADBqMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLZXhhbXBsZS5jb20x\nFjAUBgNVBAcTDVNhbiBGcmFuY2lzY28xEzARBgNVBAgTCkNhbGlmb3JuaWExGDAW\nBgNVBAMTD3d3dy5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA\nBK/CtZaQ4VliKE+DLIVGLwtSxJgtUKRzGvN1EwI3HRgKDQ3l3urBIzHtUcdMq6HZ\nb8jX0O9fXYUOf4XWggrLk1agLTArBgkqhkiG9w0BCQ4xHjAcMBoGA1UdEQQTMBGC\nD3d3dy5leGFtcGxlLmNvbTAKBggqhkjOPQQDAgNIADBFAiAcvfhXnsLtzep2sKSa\n36W7G9PRbHh8zVGlw3Hph8jR1QIhAKfrgplKwXcUctU5grjQ8KXkJV8RxQUo5KKs\ngFnXYtkb\n-----END CERTIFICATE REQUEST-----\n"}' 127.0.0.1:8000/api/v1/takreg | jq ``\` + +\# REQUEST A NEW CERTIFICATE WITHOUT CSR (requires cfssl backend for the +api container) + +> `` `curl -L -H "Content-Type: application/json" -d '{ "request": {"hosts":["harjoitus1.pvarki.fi"], "names":[{"C":"FI", "ST":"Jyvaskyla", "L":"KeskiSuomi", "O":"harjoitus1.pvarki.fi"}], "CN": "harjoitus1.pvarki.fi"}, "bundle":true, "profile":"client"}' 127.0.0.1:8000/takreg | jq ``\` + +\# LIST CFSSL CRL LIST + +> `` `curl -L -H "Content-Type: application/json" -d '{ "request": {"hosts":["harjoitus1.pvarki.fi"], "names":[{"C":"FI", "ST":"Jyvaskyla", "L":"KeskiSuomi", "O":"harjoitus1.pvarki.fi"}], "CN": "harjoitus1.pvarki.fi"}, "bundle":true, "profile":"client"}' 127.0.0.1:8000/takreg | jq ``\` + +The cfssl used behind API listens this kind of stuff +[https://github.com/cloudflare/cfssl/blob/master/doc/api/endpoint_newcert.txt](https://github.com/cloudflare/cfssl/blob/master/doc/api/endpoint_newcert.txt) + +\# Enrollment - Enroll a new work_id + +> `` `curl -H "Content-Type: application/json" -d '{"work_id":"porakoira666"}' http://127.0.0.1:8000/api/v1/enrollment/init ``\` + +\# Enrollment - Check the status and availability of work_id + +> `` `curl http://127.0.0.1:8000/api/v1/enrollment/status/koira ``\` + +\# Enrollment - Request the download link using the provided work_id_hash +`` `curl http://127.0.0.1:8000/api/v1/enrollment/deliver/zxzxzxzxzxzxzxxzzx ``\` + +\# Enrollment - Accept enrollment using permit_str +`` `curl -H "Content-Type: application/json" -d '{"enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/accept ``\` + +\# Enrollment - Set download link for enrollment +`` `curl -H "Content-Type: application/json" -d '{"download_link":"https://kuvaton.com","enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/set-dl-link ``\` + +\# Enrollment - Set state for enrollment +`` `curl -H "Content-Type: application/json" -d '{"state":"ReadyForDelivery","enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/set-state ``\` + +\# Enrollment - Add new permit_str +`` `curl -H "Content-Type: application/json" -d '{"permissions_str":"all", "new_permit_hash":"too_short","permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/add-manager ``\` + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rasenmaeher_api:devel_shell . + docker create --name rasenmaeher_api_devel -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` rasenmaeher_api:devel_shell + docker start -i rasenmaeher_api_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rasenmaeher_api_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rasenmaeher_api_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rasenmaeher_api:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rasenmaeher_api:tox . + docker run --rm -it --network host -v /var/run/docker.sock:/var/run/docker.sock -v `pwd`":/app" `echo $DOCKER_SSHAGENT` rasenmaeher_api:tox + +NOTE: This will not work from the git submodule directory in the +integration repo, docker tests must be run from the original repo. + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rasenmaeher_api:amd64-latest . + docker run --rm -it --name rasenmaeher_openapijson rasenmaeher_api:amd64-latest rasenmaeher_api openapi + docker run -it --name rasenmaeher_api rasenmaeher_api:amd64-latest + +There is also a specific target for just dumping the openapi.json: + + docker build --ssh default --target openapi -t rasenmaeher_api:amd64-openapi . + docker run --rm -it --name rasenmaeher_openapijson rasenmaeher_api:amd64-openapi + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.12.1/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.12.1/submodules/battlelog.md new file mode 100644 index 00000000..4a9671ef --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/submodules/battlelog.md @@ -0,0 +1,53 @@ +--- +title: "pvarki/typescript-liveloki-app – README" +--- + +> **Integration tag:** `1.12.1` · **Submodule commit:** `197f6b74507fb81fc442fa1522088f0cb7ed3939` +> **Repo:** git@github.com:pvarki/typescript-liveloki-app.git +> **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/197f6b74507fb81fc442fa1522088f0cb7ed3939 + +# BattleLog + +## Install + +1. Install docker + compose +2. Rename .env_example --> .env and modify as you like. +3. Run docker compose build --no-cache +4. Run docker compose up -d +5. Navigate to localhost:3000 + +## Frontend development + +The frontend is bundled with [Vite](https://vitejs.dev/). + +Once the backend is running (on port 3000), you can navigate to `frontend/`, +run `npm i` and `npm run dev` to run the Vite development server that has +hot reload and all that jazz. + +## Info + +Database is currently preseeded with test data from preseed/preseed.csv + +## Migrations + +To add new migration, locally run: `./node_modules/.bin/node-pg-migrate create ` and modify created file in `/migrations` directory. +Migrations are run (if needed) when docker container starts. `wait-for-it.sh` will ensure that psql container is up and accepting connections before running migrations. + +## JWT testing + +Remove comments from "jwt-test-network" for local jwt testing, to allow joining containers from 2 different compose runs to same docker network. + +## OpenAPI Specification + +https://pvarki.github.io/typescript-liveloki-app/ + +## Running tests + +In directory `tests`, use + +```shell +npm run test:integration +``` + +with the server running in `localhost:3000` as it does by default after `docker compose up -d`. + diff --git a/docs/dev/integrationrepo/v/1.12.1/submodules/cfssl.md b/docs/dev/integrationrepo/v/1.12.1/submodules/cfssl.md new file mode 100644 index 00000000..1c28e26e --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/submodules/cfssl.md @@ -0,0 +1,91 @@ +--- +title: "pvarki/docker-rasenmaeher-cfssl – README" +--- + +> **Integration tag:** `1.12.1` · **Submodule commit:** `87f2c2b4b22a0d25d7dcc900d0750373b24fec4b` +> **Repo:** git@github.com:pvarki/docker-rasenmaeher-cfssl.git +> **Browse at this commit:** https://github.com/pvarki/docker-rasenmaeher-cfssl/tree/87f2c2b4b22a0d25d7dcc900d0750373b24fec4b + +# cfssl Submodule + +\![Build +Status\]([https://github.com/pvarki/docker-rasenmaeher-cfssl/actions/workflows/build.yml/badge.svg](https://github.com/pvarki/docker-rasenmaeher-cfssl/actions/workflows/build.yml/badge.svg)) + +## Used as git submodule + +This repo is used as submodule in +[https://github.com/pvarki/docker-rasenmaeher-integration](https://github.com/pvarki/docker-rasenmaeher-integration) it is +probably a good idea to handle all development via it because it has +docker composition for bringin up all the other services rasenmaeher-api +depends on + +## Development + +The cfssl itself we can't do much about but the FastAPI thing uses +poetry and in any case use pre-commit: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t ocsprest:devel_shell . + docker create --name ocsprest_devel -v `pwd`/../":/app" -it `echo $DOCKER_SSHAGENT` ocsprest:devel_shell + docker start -i ocsprest_devel + +Or just pwd if working under separate checkout instead of the +integration repo. + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i ocsprest_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i ocsprest_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" ocsprest:devel_shell -c "pre-commit run --all-files" + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target ocsprest -t ocsprest:amd64-latest . + docker run -it --name ocsprest ocsprest:amd64-latest + +There is also a specific target for just dumping the openapi.json: + + docker build --ssh default --target openapi -t ocsprest:amd64-openapi . + docker run --rm -it --name rasenmaeher_openapijson ocsprest:amd64-openapi + diff --git a/docs/dev/integrationrepo/v/1.12.1/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.12.1/submodules/fpintegration.md new file mode 100644 index 00000000..abdcb3e4 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/submodules/fpintegration.md @@ -0,0 +1,106 @@ +--- +title: "pvarki/python-rasenmaeher-rmfpapi – README" +--- + +> **Integration tag:** `1.12.1` · **Submodule commit:** `7b24c8155518a4e11dd46b1f8d210b850973004c` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-rmfpapi.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-rmfpapi/tree/7b24c8155518a4e11dd46b1f8d210b850973004c + +# rmfpapi + +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rmfpapi:devel_shell . + docker create --name rmfpapi_devel -p 4625:4625 -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` rmfpapi:devel_shell + docker start -i rmfpapi_devel + +In the shell you can start the uvicorn devel server with (binding to +0.0.0.0 is important!): + + uvicorn "rmfpapi.app:get_app" --factory --host 0.0.0.0 --port 4625 --reload --log-level debug + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmfpapi_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rmfpapi_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rmfpapi:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rmfpapi:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` rmfpapi:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rmfpapi:latest . + docker run -it --name rmfpapi -p 4625:4625 rmfpapi:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.12.1/submodules/keycloak2.md b/docs/dev/integrationrepo/v/1.12.1/submodules/keycloak2.md new file mode 100644 index 00000000..26f3e7db --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/submodules/keycloak2.md @@ -0,0 +1,31 @@ +--- +title: "pvarki/docker-keycloak – README" +--- + +> **Integration tag:** `1.12.1` · **Submodule commit:** `8b2efdf6a4d90fd91d2074617983c086ebe537c5` +> **Repo:** git@github.com:pvarki/docker-keycloak.git +> **Browse at this commit:** https://github.com/pvarki/docker-keycloak/tree/8b2efdf6a4d90fd91d2074617983c086ebe537c5 + +# KeyCloak + +Templates, init container etc to setup keycloak automagically with ENV +variables. + +## Used as git submodule + +This repo is used as submodule in +[https://github.com/pvarki/docker-rasenmaeher-integration](https://github.com/pvarki/docker-rasenmaeher-integration) it is +probably a good idea to handle all development via it because it has +docker composition for bringin up all the other services rasenmaeher-api +depends on + +## pre-commit notes + +Make sure pre-commit is installed: + + pre-commit install --install-hooks + +And it's a good idea to run it regularly before committing: + + pre-commit run --all-files + diff --git a/docs/dev/integrationrepo/v/1.12.1/submodules/kw_product_init.md b/docs/dev/integrationrepo/v/1.12.1/submodules/kw_product_init.md new file mode 100644 index 00000000..3625f8c3 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/submodules/kw_product_init.md @@ -0,0 +1,78 @@ +--- +title: "pvarki/golang-kraftwerk-init-helper-cli – README" +--- + +> **Integration tag:** `1.12.1` · **Submodule commit:** `0dee1f8f397ba37280f6bf393a3b866d93387e8c` +> **Repo:** git@github.com:pvarki/golang-kraftwerk-init-helper-cli.git +> **Browse at this commit:** https://github.com/pvarki/golang-kraftwerk-init-helper-cli/tree/0dee1f8f397ba37280f6bf393a3b866d93387e8c + +golang-kraftwerk-init-helper-cli =============================== + +Tool for products to create their certificates from +KRAFTWERK manifests. + +# Development + +## Prerequisites + +**Enable** +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/) + +```bash +export DOCKER_BUILDKIT=1 +``` + +**Forward SSH-agent to running instance:** + +**OSX:** + +```bash +export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" +``` + +**Linux:** + +```bash +export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" +``` + +## Create & start development container + +Build the image, create a container and start the container + +```bash +docker build --ssh default --target dev_shell -t kraftwerk_init_helper:dev_shell . +``` + +```bash +docker create --name kraftwerk_init_helper -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` kraftwerk_init_helper:dev_shell +``` + +```bash +docker start -i kraftwerk_init_helper +``` + +## pre-commit initialization + +Once inside the container, run: + +```bash +pre-commit install && pre-commit run --all-files +``` + +That's it, now you have the development environment up & running. + +# Production + +Build the production image: + +```bash +docker build --ssh default --target production -t kraftwerk_init_helper:latest . +``` + +Run the image: + +```bash +docker run --rm -it --name kraftwerk_helper kraftwerk_init_helper:latest +``` + diff --git a/docs/dev/integrationrepo/v/1.12.1/submodules/miniwerk.md b/docs/dev/integrationrepo/v/1.12.1/submodules/miniwerk.md new file mode 100644 index 00000000..43b24cb0 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/submodules/miniwerk.md @@ -0,0 +1,104 @@ +--- +title: "pvarki/python-rasenmaeher-miniwerk – README" +--- + +> **Integration tag:** `1.12.1` · **Submodule commit:** `5ce93b4c918bff4c2731fb08529a7d307995e6d1` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-miniwerk.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-miniwerk/tree/5ce93b4c918bff4c2731fb08529a7d307995e6d1 + +# miniwerk + +Minimal KRAFTWERK amulation to be able to run a RASENMAEHER+products +deployment on any VM + +There are some required ENV configs check out example_env.sh for them +(.env file is also supported) + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t miniwerk:devel_shell . + docker create --name miniwerk_devel -v `pwd`":/app" -p 80:80 -it `echo $DOCKER_SSHAGENT` miniwerk:devel_shell + docker start -i miniwerk_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i miniwerk_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i miniwerk_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" miniwerk:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t miniwerk:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` miniwerk:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t miniwerk:amd64-latest . + docker run -it --name miniwerk miniwerk:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.12.1/submodules/mtxauthz.md b/docs/dev/integrationrepo/v/1.12.1/submodules/mtxauthz.md new file mode 100644 index 00000000..2eeab0cc --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/submodules/mtxauthz.md @@ -0,0 +1,122 @@ +--- +title: "pvarki/python-mediamtx-rmmtxauthz – README" +--- + +> **Integration tag:** `1.12.1` · **Submodule commit:** `fac84b6aa5a5ce01a66264118407894d3b3ff3d0` +> **Repo:** git@github.com:pvarki/python-mediamtx-rmmtxauthz.git +> **Browse at this commit:** https://github.com/pvarki/python-mediamtx-rmmtxauthz/tree/fac84b6aa5a5ce01a66264118407894d3b3ff3d0 + +# rmmtxauthz + +Do HTTP based authz for MediaMTX + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rmmtxauthz:devel_shell . + docker create --name rmmtxauthz_devel -v "$(pwd)/rune/output/rune.json:/opt/templates/mediamtx.json" -v "$(pwd):/app" -it $(echo $DOCKER_SSHAGENT) rmmtxauthz:devel_shell + docker start -i rmmtxauthz_devel + +To rebuild the documentation inside the container run: + + rune rune/src json >/opt/templates/mediamtx.json + +Outside of container use: + + rune rune/src json >rune/output/rune.json + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmmtxauthz_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rmmtxauthz_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v "$(pwd):/app" rmmtxauthz:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rmmtxauthz:tox . + docker run --rm -it -v "$(pwd):/app" $(echo $DOCKER_SSHAGENT) rmmtxauthz:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rmmtxauthz:amd64-latest . + docker run -it --name rmmtxauthz rmmtxauthz:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p $(which python3.11) my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + git add poetry.lock + pre-commit install --install-hooks + pre-commit run --all-files + +If you get weird errors about missing packages from pre-commit try +running it with "poetry run pre-commit". + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + +Running "pre-commit run --all-files" and "py.test -v" regularly during +development and especially before committing will save you some +headache. + +### RUNE instructions compile + +tldr: + + rune rune/src json >rune/output/mediamtx.json + diff --git a/docs/dev/integrationrepo/v/1.12.1/submodules/takintegration.md b/docs/dev/integrationrepo/v/1.12.1/submodules/takintegration.md new file mode 100644 index 00000000..24d83fbe --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/submodules/takintegration.md @@ -0,0 +1,100 @@ +--- +title: "pvarki/python-tak-rmapi – README" +--- + +> **Integration tag:** `1.12.1` · **Submodule commit:** `56eedefe48b12c5a3394ac675aa33245d0dce23b` +> **Repo:** git@github.com:pvarki/python-tak-rmapi.git +> **Browse at this commit:** https://github.com/pvarki/python-tak-rmapi/tree/56eedefe48b12c5a3394ac675aa33245d0dce23b + +# takrmapi + +RASENMAEHER integration API for TAK server + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t takrmapi:devel_shell . + docker create --name takrmapi_devel -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` takrmapi:devel_shell + docker start -i takrmapi_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i takrmapi_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i takrmapi_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" takrmapi:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t takrmapi:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` takrmapi:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t takrmapi:amd64-latest . + docker run -it --name takrmapi takrmapi:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.12.1/submodules/takserver.md b/docs/dev/integrationrepo/v/1.12.1/submodules/takserver.md new file mode 100644 index 00000000..c35c5dc6 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/submodules/takserver.md @@ -0,0 +1,68 @@ +--- +title: "pvarki/docker-atak-server – README" +--- + +> **Integration tag:** `1.12.1` · **Submodule commit:** `4562ecd9f0614f2b3ee946cdf0238d10cbbf6e87` +> **Repo:** git@github.com:pvarki/docker-atak-server.git +> **Browse at this commit:** https://github.com/pvarki/docker-atak-server/tree/4562ecd9f0614f2b3ee946cdf0238d10cbbf6e87 + +# Run TAK Java server in container + +tldr: + + cp takserver.env.example takserver.env + # edit the env + # export the variables gomplate uses (see the .tpl files) + docker compose pull --include-deps --ignore-pull-failures + docker compose -p tak up -d + +or use docker compose.local.yml without gomplate for local dev +(rebuilding containers): + + export DOCKER_TAG_EXTRA="-dev" + docker build --no-cache --progress=plain -t takserver:latest${DOCKER_TAG_EXTRA} -t takserver:4.7-RELEASE-32${DOCKER_TAG_EXTRA} -t pvarki/takserver:4.7-RELEASE-32${DOCKER_TAG_EXTRA} . + cp takserver.env.example takserver.env + # edit the env + docker compose -f docker-compose.local.yml -p tak up + +Note, for things that live in the volumes (like TAK certs) you must nuke +the volumes to see changes: + + docker compose -f docker-compose.local.yml -p tak down -v ; docker compose -f docker-compose.local.yml -p tak rm -vf + +## Creating client packages locally + +Using the REST API is probably nicer though + +Create client package: + + docker compose -p tak exec takserver_api /bin/bash -c 'CLIENT_CERT_NAME=replaceme /opt/scripts/make_client_zip.sh' + +Then get /opt/tak/certs/files/clientpkgs/replaceme.zip out of the +container: + + docker compose -p tak exec taktakserver_apiserver /bin/bash -c 'base64 /opt/tak/certs/files/clientpkgs/replaceme.zip' | base64 -id >replaceme.zip + +This approach also works for recovering the admin cert +(/opt/tak/certs/files/admin.p12 unless you changed the ADMIN_CERT_NAME +ENV) + +## Creating new admin users + +Create the user on the takserver container: + + docker compose -p tak exec takserver_api /bin/bash -c 'cd /opt/tak/data/certs/ && CAPASS=$CA_PASS PASS=replaceme_user_cert_pass ./makeCert.sh client replaceme_username && ADMIN_CERT_NAME=replaceme_username /opt/scripts/enable_admin.sh' + +See above about the hard way of getting the cert file, or use the REST +API. + +## Gradle builds + +Build the distribution: + + mkdir outputs + docker build --progress=plain -f Dockerfile_build --target files -t atakbuild:files . + docker run --rm -it -v `pwd`/outputs:/output atakbuild:files + +Now you have the build artefacts in outputs -directory. + diff --git a/docs/dev/integrationrepo/v/1.12.1/submodules/ui.md b/docs/dev/integrationrepo/v/1.12.1/submodules/ui.md new file mode 100644 index 00000000..ac424343 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.12.1/submodules/ui.md @@ -0,0 +1,116 @@ +--- +title: "pvarki/rasenmaeher-ui – README" +--- + +> **Integration tag:** `1.12.1` · **Submodule commit:** `eb2d303774fa7791ba5410080cd55316fe8b32c1` +> **Repo:** git@github.com:pvarki/rasenmaeher-ui.git +> **Browse at this commit:** https://github.com/pvarki/rasenmaeher-ui/tree/eb2d303774fa7791ba5410080cd55316fe8b32c1 + +# UI For RASENMAEHER + +React, TS etc + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rasenmaeher_ui:devel_shell . + docker create --name rmui_devel -v `pwd`":/app" -p 8002:8002 -it `echo $DOCKER_SSHAGENT` rasenmaeher_ui:devel_shell + docker start -i rmui_devel + +Note: due to the volume mapping if you already had run "npm install" on +the host you need to delete the node_modules directory and in any case +you need to run "npm install" inside the container. + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmui_devel /bin/bash -c "pre-commit install" + docker exec -i rmui_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rasenmaeher_ui:devel_shell -c "pre-commit run --all-files" + +### Production docker + +This will just deliver the compiled files to a mounted directory + +> docker build --ssh default --target production -t +> rasenmaeher_ui:amd64-latest . docker run -it --rm -v +> pwd/rmui_files:/deliver +> rasenmaeher_ui:amd64-latest + +Then you need to serve things so that index.html is the default +controller for things, nginx example: + + location / { + index index.html; + root /rmui_files; + try_files $uri $uri/ /index.html =404; + } + +## Development + +TLDR: + +- Create and activate a Node 18 virtualenv: + + FIXME: Insert instructions + +- change to a branch: + + git checkout -b my_branch + +- Install project deps and pre-commit hooks: + + npm install + pre-commit install + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo. + +### Asset set + +Asset set is loaded according to VITE_ASSET_SET enviroment variable. +Changing asset set requires reloading vite. Folder assetSetStore +contains files for for different sets that are copied to src/assets/set/ +folder on startup. Current set is tracked with setName.txt file. + +When you want to use dynamic assets, you want to make sure your dynamic +asset has equivalent for sets (both "fdf" and "neutral", so on). If the +assets are images, you want to then import them to your components from +/assets/set/\* where the env-defined assets are placed by VITE on +startup. If the assets are translation keys, make sure keys are present +in locales under assetSetStore, and then use the "dynamic" namespace for +accessing them. + diff --git a/docs/dev/integrationrepo/v/1.13.0/_category_.json b/docs/dev/integrationrepo/v/1.13.0/_category_.json new file mode 100644 index 00000000..271f9953 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "1.13.0", + "position": 5 +} diff --git a/docs/dev/integrationrepo/v/1.13.0/index.md b/docs/dev/integrationrepo/v/1.13.0/index.md new file mode 100644 index 00000000..0a49e5f5 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/index.md @@ -0,0 +1,4 @@ +--- +title: "1.13.0" +--- + diff --git a/docs/dev/integrationrepo/v/1.13.0/integration.md b/docs/dev/integrationrepo/v/1.13.0/integration.md new file mode 100644 index 00000000..fcae9fcf --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/integration.md @@ -0,0 +1,319 @@ +--- +title: "pvarki/docker-rasenmaeher-integration – README" +--- + +> **Integration tag:** `1.13.0` +> **Repo:** https://github.com/pvarki/docker-rasenmaeher-integration + +![Build Status](https://github.com/pvarki/docker-rasenmaeher-integration/actions/workflows/build.yml/badge.svg) + +# Deploy App + +"One Ring to rule them all, One Ring to find them, One Ring to bring +them all and in the darkness bind them." + +Docker compositions, helpers etc to bring it all together into something +resembling grand old ones. + +Codename RASENMAEHER because infantry jokes. + +## WTF is this anyway ? + +This [Disobey24 +talk](https://www.youtube.com/watch?v=m3xd7uygpaY&list=PLLvAhAn5sGfiB9AlEt2KD7H9Dnr6kbd64&index=23) +explains a lot. + +## Running Deploy App in your own docker environment + +### Needed DNS Records + +These need to point to your WAN address. + +> - domain +> - kc.domain +> - tak.domain +> - bl.domain +> - mtx.domain +> - mtls.domain +> - mtls.kc.domain +> - mtls.tak.domain +> - mtls.kc.domain +> - mtls.bl.domain +> - mtls.mtx.domain + +When more products are added to the deployment they will follow the same +naming pattern, you will need subdomains for all products listed in the +composition for miniwerk service variable MW_PRODUCTS and "kc" for +Keycloak. + +### Needed ports open + +And redirected to the server if behind NAT or similar. + +> - 80 +> - 443 +> - 8443 (TAK) +> - 8446 (TAK) +> - 9446 (Keycloak) +> - 4626 (Product integration APIs port) +> - 4666 (Battlelog API/UI port) +> - 1936 RTMPS +> - 8322 RTSPS +> - 8890 SRT +> - 9888 HSL +> - 9889 WebRTC +> - 9996 Playback for recorded streams/videos + +### Downloading and composing Deploy App + +Be mindfull on where you download the repository, you will need to +perform rest of the commands inside the downloaded repository. + +Getting the repository from github (on Windows **first** see "Windows +notes" below): + + git clone --recurse-submodules -j8 git@github.com:pvarki/docker-rasenmaeher-integration.git + +Create `.env` file that defines environmental variables for Deploy App +setup. File must be located inside downloaded repository and file type +must be named literally `.env` (not `something.env`) to work. + +The original example file is: +[https://github.com/pvarki/docker-rasenmaeher-integration/blob/main/example_env.sh](https://github.com/pvarki/docker-rasenmaeher-integration/blob/main/example_env.sh) + +Example .env-file with the minimal information needed: + + KEYCLOAK_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RM_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + POSTGRES_PASSWORD="input-secure-password" # pragma: allowlist secret + LDAP_ADMIN_PASSWORD="input-secure-password" # pragma: allowlist secret + KEYCLOAK_ADMIN_PASSWORD="input-secure-password" # pragma: allowlist secret + TAK_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + SERVER_DOMAIN="input-domain" + CFSSL_CA_NAME="input-ca-name" + MW_LE_EMAIL="input-email-for-lets-encrypt" + MW_LE_TEST="false" + TAKSERVER_CERT_PASS="input-secure-password" # pragma: allowlist secret + TAK_CA_PASS="input-secure-password" # pragma: allowlist secret + VITE_ASSET_SET="${VITE_ASSET_SET:-neutral}" + KEYCLOAK_PROFILEROOT_UUID="input-uuid" + KEYCLOAK_HTTPS_KEY_STORE_PASSWORD="input-secure-password" # pragma: allowlist secret + KEYCLOAK_HTTPS_TRUST_STORE_PASSWORD="input-secure-password" # pragma: allowlist secret + BL_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RMMTX_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RMMTX_API_PASSWORD="input-secure-password" # pragma: allowlist secret + +Replace "intput-secure-password" with a good passphrase that is unique +for each replacment. You can generate an UUID with: + + python -c 'import uuid; print(uuid.uuid4())' + +If you wish to use one deployment for longer than the *design lifetime* +of 1-2 months you can change the following env variables. But do +understand that this is **not recommended** and has **security +implications**. If you do this **you** take **responsibility** to go +through all Dockerfiles and compositions to understand **exactly** how +things are done and how apply security updates into the containers: + + CFSSL_CA_EXPIRY="8800h" # Input time in hours = xxh, 8800h is 366.. days + CFSSL_SIGN_DEFAULT_EXPIRY="8800h" # Input time in hours = xxh, 8800h is 366.. days + +Starting the services: + + docker compose up -d + +Updating the repository from github: + + git pull + git submodule update --init --recursive + docker compose pull + docker compose up -d + +!DO NOT DO! Deleting the services. Deletes the certificates etc you will +need to add all users etc again: + + docker compose down -v + +Getting the admin login invite code for first admin: + + docker compose exec -it rmapi /bin/bash -c "rasenmaeher_api addcode" + +If you get "no such service -d", make sure whatever you copy-pasted the +command from did not render the dash (ASCII 0x2D) as some other +codepoint with a glyph that looks deceptively similar. When in doubt +write the commands yourself instead of copy-pasting. + +### Services + +Deploy App login page: + + https://domain (example.com) + +Deploy App home page: + + https://mtls.domain (mtls.example.com) + +Takserver Admin UI: + + https://tak.domain:8443/ (tak.example.com:8443/) + +Keycloack Admin UI. (Later group management will be withing Deploy App): + + https://kc.domain:9443/admin/RASENMAEHER/console/ (kc.example.com:9443/admin/RASENMAEHER/console/) + +OTA update server inside takserver. Is located in the loaded repository, +location depends on where you downloaded it: + + /home/user/docker-rasenmaeher-integration/takserver/update + +### Using the Deploy App service + +1. Login with first admin code. Create your admin account by typing + your first admin invite code and inputting desired admin callsign. +2. Create invite code for other users. Share the invite code. Go to + Manage Users -\> Add Users -\> Create New Invite. Share link, qr + code or invite code and domain. +3. Approve users in Deploy App. Open approvement link or scan qr code + from users and approve the user. You can also go to Approve Users + -\> Select Waiting User and input the users approvement code. +4. If desired promote some of the added users as admins. Go to Manage + Users -\> Manage Users -\> Select user and select Promote. You can + also Demote Admins or Delete users altogether. + +### Using Deploy App TAK in EUD + +EUD=End User Device + +1. Login to Deploy App. Go to [https://mtls.domain](https://mtls.domain) and select TAK. +2. Download Client Package. Select tak package for desired software + "Android ATAK or Windows WinTAK" or "iOS iTAK". Select Download + Client Package. +3. Go to EUD's TAK Software. Import downloaded package. Device is + connected to server. +4. You should also read Quickstart and Usage Guides. + +## Git submodules + +When cloning for the first time use: + + git clone --recurse-submodules -j8 git@github.com:pvarki/docker-rasenmaeher-integration.git + +When updating or checking out branches use: + + git submodule update --init --recursive + +And if you forgot to --recurse-submodules run the update command above. + +The submodules are repos in their own right, if you plan to make changes +into them change to the directory and create new branch, commit and push +changes as usual under that directory. + +### Directories that are submodules + +> - api [https://github.com/pvarki/python-rasenmaeher-api](https://github.com/pvarki/python-rasenmaeher-api) +> - cfssl [https://github.com/pvarki/docker-rasenmaeher-cfssl](https://github.com/pvarki/docker-rasenmaeher-cfssl) +> - fpintegration [https://github.com/pvarki/python-rasenmaeher-rmfpapi](https://github.com/pvarki/python-rasenmaeher-rmfpapi) +> - keycloak [https://github.com/pvarki/docker-keycloak](https://github.com/pvarki/docker-keycloak) +> - kw_product_init +> [https://github.com/pvarki/golang-kraftwerk-init-helper-cli](https://github.com/pvarki/golang-kraftwerk-init-helper-cli) +> - openldap [https://github.com/pvarki/docker-openldap](https://github.com/pvarki/docker-openldap) +> - miniwerk [https://github.com/pvarki/python-rasenmaeher-miniwerk](https://github.com/pvarki/python-rasenmaeher-miniwerk) +> - ui [https://github.com/pvarki/rasenmaeher-ui](https://github.com/pvarki/rasenmaeher-ui) +> - takserver [https://github.com/pvarki/docker-atak-server](https://github.com/pvarki/docker-atak-server) +> - takintegration [https://github.com/pvarki/python-tak-rmapi](https://github.com/pvarki/python-tak-rmapi) +> - battlelog [https://github.com/pvarki/typescript-liveloki-app](https://github.com/pvarki/typescript-liveloki-app) +> - mtxauthz [https://github.com/pvarki/python-mediamtx-rmmtxauthz](https://github.com/pvarki/python-mediamtx-rmmtxauthz) + +## Autogenerated (mostly API) docs + +> - Module API docs: +> [https://pvarki.github.io/docker-rasenmaeher-integration/docs/](https://pvarki.github.io/docker-rasenmaeher-integration/docs/) +> - Swagger definition for Deploy App API: +> [https://pvarki.github.io/docker-rasenmaeher-integration/](https://pvarki.github.io/docker-rasenmaeher-integration/) + +## Running in local development mode + +### Windows notes + +> 1. Do **NOT** use git-bash, it will cause *weirdest* issues with +> Docker containers +> +> 2. Use WSL, see +> [best_practises](https://github.com/pvarki/markdown-pvarki-best_practises) +> -repo for instructions on how to set it up. +> +> 3. Make sure whatever git client or IDE you use it does not mess with +> line-endings, for CLI client this does the trick: +> +> git config --global core.eol lf +> git config --global core.autocrlf false + +### Compositions + +Generally start with "rmlocal", it corresponds best to a real running +environment. "rmdev" starts a bunch of things in development mode which +does make developing more convenient but also introduces extra +variability to how things work. + +Make sure to always check your changes work correctly in rmlocal mode +where assets are minified and baked in. + +TLDR: + + alias rmlocal="docker compose -p rmlocal -f docker-compose-local.yml" + rmlocal build takinit miniwerk --pull + rmlocal build --pull + rmlocal up + +or: + + alias rmdev="docker compose -p rmdev -f docker-compose-local.yml -f docker-compose-dev.yml" + rmdev build takinit miniwerk --pull + rmdev build --pull + rmdev up + +OpenLDAP and keycloak-init sometimes fail on first start, just run up +again. + +IMPORTANT: Only keep either rmlocal or rmdev created at one time or you +may have weird network issues run "down" for one env before starting the +other. + +Remember to run "down -v" if you want to reset the persistent volumes, +or if you have weird issues when switching between environments. + +The dev version launches all the services and runs rasenmaeher-api in +uvicorn reload mode so any edits you make under /api will soon be +reflected in the running instance. + +If rasenmaeher-ui devel server complains make sure to delete +`ui/node_modules` -directory from host first. The docker NodeJS +distribution probably is not compatible with whatever you have installed +on the host. + +### Gaining first admin access in dev and production mode + +In dev mode: + + docker exec -it rmdev-rmapi-1 /bin/bash -c "source /.venv/bin/activate && rasenmaeher_api addcode" + +Under dev mode, the UI runs at [https://localmaeher.dev.pvarki.fi:4439](https://localmaeher.dev.pvarki.fi:4439). + +In VM production mode: + + docker exec -it rmvm-rmapi-1 /bin/bash -c "rasenmaeher_api addcode" + +## pre-commit notes + +Use "pre-commit run --all-files" liberally (and make sure you have run +"pre-commit install --install-hooks"). If you get complaints about +missing environment variables run "source example_env.sh" + +## Integration tests + +Pytest is used to handle the integration tests, the requirements are in +tests/requirements.txt. NOTE: The tests have side-effects and expect a +clean database to start with so always make sure to run "down -v" for +the composition first, then bring it back up before running integration +tests. + diff --git a/docs/dev/integrationrepo/v/1.13.0/submodules/api.md b/docs/dev/integrationrepo/v/1.13.0/submodules/api.md new file mode 100644 index 00000000..dc1d6a7c --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/submodules/api.md @@ -0,0 +1,240 @@ +--- +title: "pvarki/python-rasenmaeher-api – README" +--- + +> **Integration tag:** `1.13.0` · **Submodule commit:** `9c54071450ac027dd5687ade5ac63e1497cc1ac2` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-api.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-api/tree/9c54071450ac027dd5687ade5ac63e1497cc1ac2 + +# python-rasenmaeher-api + +python-rasenmaeher-api + +## Configuration + +This application can be configured with environment variables. Below is +a example list of available variables. You can find all available +variables here +[https://github.com/pvarki/python-rasenmaeher-api/blob/main/src/rasenmaeher_api/settings.py](https://github.com/pvarki/python-rasenmaeher-api/blob/main/src/rasenmaeher_api/settings.py). + +| ENV VAR | Default value | Info / Usage | +|---------------------------|---------------------------------------------|---------------------------------------------------------------| +| RM_PORT | 8000 | Docker API listen port | +| RM_WORKERS_COUNT | 2 | Number of uvicorn workers | +| RM_RELOAD | False | Reload service on code change | +| RM_ENVIRONMENT | dev | Run dev / prod environment | +| RM_CFSSL_HOST | None | CFSSL host url | +| RM_CFSSL_PORT | None | CFSSL service port | +| RM_KEYCLOAK_SERVER_URL | None | Keycloak server url ([http://1234:8080/auth](http://1234:8080/auth)) | +| RM_KEYCLOAK_CLIENT_ID | None | Keycloak client id | +| RM_KEYCLOAK_REALM_NAME | None | Keycloak realm name | +| RM_KEYCLOAK_CLIENT_SECRET | None | Keycloak secert | +| RM_LDAP_CONN_STRING | None | LDAP conn string | +| RM_LDAP_USERNAME | None | LDAP username | +| RM_LDAP_CLIENT_SECRET | None | LDAP connection secret | +| RM_SQLITE_FILEPATH_PROD | /opt/rasenmaher/persistent/sqlite/rm_db.sql | location for sqlite database file in "prod" | +| RM_SQLITE_FILEPATH_DEV | /tmp/rm_db.sql | location for sqlite database file in "dev", local development | + +Container Variables + +You can create .env file in the root +directory and place all environment variables here. + +All environment variables should start with +RM\_ prefix. + +For example if you see in your "rasenmaeher_api/settings.py" a variable +named like random_parameter, you should +provide the "RM_RANDOM_PARAMETER" variable to configure the value. This +behaviour can be changed by overriding +env_prefix property in +rasenmaeher_api.settings.Settings.Config. + +An example of .env file: +`` `bash RM_RELOAD="True" RM_PORT="8000" RM_ENVIRONMENT="dev" ``\` + +You can read more about BaseSettings class here: +[https://pydantic-docs.helpmanual.io/usage/settings/](https://pydantic-docs.helpmanual.io/usage/settings/) + +## Backend services manifest + +Integration APIs are always https in port 4625. Default location for the +manifest is /pvarki/kraftwerk-rasenmaeher-init.json + +`` `json { "dns": "sleepy-sloth.pvarki.fi", "products": { "tak": "tak.sleepy-sloth.pvarki.fi", "ptt": "ptt.sleepy-sloth.pvarki.fi" } } ``\` + +## Api endpoints and usage + +| State | Method | URI | Request JSON | Response JSON | Api description . | +|---------|--------|------------------------------------|--------------------------------------------------------------|-------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| Dummy | GET | /api/v1/enroll/status/{work_id} | NA | {'status':'None/Processing/Denied/WaitingForAcceptance/ReadyForDelivery/Delivered'} | Check the situation of enrollment process, None = no enrollment started, this work_id is free to use. | +| Dummy | POST | /api/v1/enroll/init | {'work_id':'{work_id}'} | {'work_id':'{work_id}', 'id_hash':{id_string} } | Start service access enrollment for given {work_id} | +| Dummy | GET | /api/v1/enroll/deliver/{id_string} | NA | {'dl_link':"{[http://here.be/zip](http://here.be/zip)}"} | Deliver download link for enrollment zip | +| Dummy | POST | /api/v1/enroll/accept | { 'permit_string':'{permit_string}, 'id_hash':'{id_hash}' '} | { 'access':'granted/denied/None', 'work_id':'{work_id}' } | Accept the enrollment request | +| Testing | POST | /api/v1/product/sign_csr | { 'TO':'DO'} | { 'TO':'DO'} | Accept the enrollment request | +| Testing | POST | /api/v1/enroll/accept | { 'permit_string':'{permit_string}, 'id_hash':'{id_hash}' '} | { 'access':'granted/denied/None', 'work_id':'{work_id}' } | Accept the enrollment request | + +API vimpain + +## Example usage + +\# REQUEST A NEW CERTIFICATE USING CSR (requires cfssl backend for the +api container) + +> `` `curl -L -H "Content-Type: application/json" -d '{"csr": "-----BEGIN CERTIFICATE REQUEST-----\nMIIBUjCB+QIBADBqMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLZXhhbXBsZS5jb20x\nFjAUBgNVBAcTDVNhbiBGcmFuY2lzY28xEzARBgNVBAgTCkNhbGlmb3JuaWExGDAW\nBgNVBAMTD3d3dy5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA\nBK/CtZaQ4VliKE+DLIVGLwtSxJgtUKRzGvN1EwI3HRgKDQ3l3urBIzHtUcdMq6HZ\nb8jX0O9fXYUOf4XWggrLk1agLTArBgkqhkiG9w0BCQ4xHjAcMBoGA1UdEQQTMBGC\nD3d3dy5leGFtcGxlLmNvbTAKBggqhkjOPQQDAgNIADBFAiAcvfhXnsLtzep2sKSa\n36W7G9PRbHh8zVGlw3Hph8jR1QIhAKfrgplKwXcUctU5grjQ8KXkJV8RxQUo5KKs\ngFnXYtkb\n-----END CERTIFICATE REQUEST-----\n"}' 127.0.0.1:8000/api/v1/takreg | jq ``\` + +\# REQUEST A NEW CERTIFICATE WITHOUT CSR (requires cfssl backend for the +api container) + +> `` `curl -L -H "Content-Type: application/json" -d '{ "request": {"hosts":["harjoitus1.pvarki.fi"], "names":[{"C":"FI", "ST":"Jyvaskyla", "L":"KeskiSuomi", "O":"harjoitus1.pvarki.fi"}], "CN": "harjoitus1.pvarki.fi"}, "bundle":true, "profile":"client"}' 127.0.0.1:8000/takreg | jq ``\` + +\# LIST CFSSL CRL LIST + +> `` `curl -L -H "Content-Type: application/json" -d '{ "request": {"hosts":["harjoitus1.pvarki.fi"], "names":[{"C":"FI", "ST":"Jyvaskyla", "L":"KeskiSuomi", "O":"harjoitus1.pvarki.fi"}], "CN": "harjoitus1.pvarki.fi"}, "bundle":true, "profile":"client"}' 127.0.0.1:8000/takreg | jq ``\` + +The cfssl used behind API listens this kind of stuff +[https://github.com/cloudflare/cfssl/blob/master/doc/api/endpoint_newcert.txt](https://github.com/cloudflare/cfssl/blob/master/doc/api/endpoint_newcert.txt) + +\# Enrollment - Enroll a new work_id + +> `` `curl -H "Content-Type: application/json" -d '{"work_id":"porakoira666"}' http://127.0.0.1:8000/api/v1/enrollment/init ``\` + +\# Enrollment - Check the status and availability of work_id + +> `` `curl http://127.0.0.1:8000/api/v1/enrollment/status/koira ``\` + +\# Enrollment - Request the download link using the provided work_id_hash +`` `curl http://127.0.0.1:8000/api/v1/enrollment/deliver/zxzxzxzxzxzxzxxzzx ``\` + +\# Enrollment - Accept enrollment using permit_str +`` `curl -H "Content-Type: application/json" -d '{"enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/accept ``\` + +\# Enrollment - Set download link for enrollment +`` `curl -H "Content-Type: application/json" -d '{"download_link":"https://kuvaton.com","enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/set-dl-link ``\` + +\# Enrollment - Set state for enrollment +`` `curl -H "Content-Type: application/json" -d '{"state":"ReadyForDelivery","enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/set-state ``\` + +\# Enrollment - Add new permit_str +`` `curl -H "Content-Type: application/json" -d '{"permissions_str":"all", "new_permit_hash":"too_short","permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/add-manager ``\` + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rasenmaeher_api:devel_shell . + docker create --name rasenmaeher_api_devel -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` rasenmaeher_api:devel_shell + docker start -i rasenmaeher_api_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rasenmaeher_api_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rasenmaeher_api_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rasenmaeher_api:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rasenmaeher_api:tox . + docker run --rm -it --network host -v /var/run/docker.sock:/var/run/docker.sock -v `pwd`":/app" `echo $DOCKER_SSHAGENT` rasenmaeher_api:tox + +NOTE: This will not work from the git submodule directory in the +integration repo, docker tests must be run from the original repo. + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rasenmaeher_api:amd64-latest . + docker run --rm -it --name rasenmaeher_openapijson rasenmaeher_api:amd64-latest rasenmaeher_api openapi + docker run -it --name rasenmaeher_api rasenmaeher_api:amd64-latest + +There is also a specific target for just dumping the openapi.json: + + docker build --ssh default --target openapi -t rasenmaeher_api:amd64-openapi . + docker run --rm -it --name rasenmaeher_openapijson rasenmaeher_api:amd64-openapi + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.13.0/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.13.0/submodules/battlelog.md new file mode 100644 index 00000000..79e5f682 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/submodules/battlelog.md @@ -0,0 +1,53 @@ +--- +title: "pvarki/typescript-liveloki-app – README" +--- + +> **Integration tag:** `1.13.0` · **Submodule commit:** `197f6b74507fb81fc442fa1522088f0cb7ed3939` +> **Repo:** git@github.com:pvarki/typescript-liveloki-app.git +> **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/197f6b74507fb81fc442fa1522088f0cb7ed3939 + +# BattleLog + +## Install + +1. Install docker + compose +2. Rename .env_example --> .env and modify as you like. +3. Run docker compose build --no-cache +4. Run docker compose up -d +5. Navigate to localhost:3000 + +## Frontend development + +The frontend is bundled with [Vite](https://vitejs.dev/). + +Once the backend is running (on port 3000), you can navigate to `frontend/`, +run `npm i` and `npm run dev` to run the Vite development server that has +hot reload and all that jazz. + +## Info + +Database is currently preseeded with test data from preseed/preseed.csv + +## Migrations + +To add new migration, locally run: `./node_modules/.bin/node-pg-migrate create ` and modify created file in `/migrations` directory. +Migrations are run (if needed) when docker container starts. `wait-for-it.sh` will ensure that psql container is up and accepting connections before running migrations. + +## JWT testing + +Remove comments from "jwt-test-network" for local jwt testing, to allow joining containers from 2 different compose runs to same docker network. + +## OpenAPI Specification + +https://pvarki.github.io/typescript-liveloki-app/ + +## Running tests + +In directory `tests`, use + +```shell +npm run test:integration +``` + +with the server running in `localhost:3000` as it does by default after `docker compose up -d`. + diff --git a/docs/dev/integrationrepo/v/1.13.0/submodules/cfssl.md b/docs/dev/integrationrepo/v/1.13.0/submodules/cfssl.md new file mode 100644 index 00000000..abe46cb7 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/submodules/cfssl.md @@ -0,0 +1,91 @@ +--- +title: "pvarki/docker-rasenmaeher-cfssl – README" +--- + +> **Integration tag:** `1.13.0` · **Submodule commit:** `87f2c2b4b22a0d25d7dcc900d0750373b24fec4b` +> **Repo:** git@github.com:pvarki/docker-rasenmaeher-cfssl.git +> **Browse at this commit:** https://github.com/pvarki/docker-rasenmaeher-cfssl/tree/87f2c2b4b22a0d25d7dcc900d0750373b24fec4b + +# cfssl Submodule + +\![Build +Status\]([https://github.com/pvarki/docker-rasenmaeher-cfssl/actions/workflows/build.yml/badge.svg](https://github.com/pvarki/docker-rasenmaeher-cfssl/actions/workflows/build.yml/badge.svg)) + +## Used as git submodule + +This repo is used as submodule in +[https://github.com/pvarki/docker-rasenmaeher-integration](https://github.com/pvarki/docker-rasenmaeher-integration) it is +probably a good idea to handle all development via it because it has +docker composition for bringin up all the other services rasenmaeher-api +depends on + +## Development + +The cfssl itself we can't do much about but the FastAPI thing uses +poetry and in any case use pre-commit: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t ocsprest:devel_shell . + docker create --name ocsprest_devel -v `pwd`/../":/app" -it `echo $DOCKER_SSHAGENT` ocsprest:devel_shell + docker start -i ocsprest_devel + +Or just pwd if working under separate checkout instead of the +integration repo. + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i ocsprest_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i ocsprest_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" ocsprest:devel_shell -c "pre-commit run --all-files" + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target ocsprest -t ocsprest:amd64-latest . + docker run -it --name ocsprest ocsprest:amd64-latest + +There is also a specific target for just dumping the openapi.json: + + docker build --ssh default --target openapi -t ocsprest:amd64-openapi . + docker run --rm -it --name rasenmaeher_openapijson ocsprest:amd64-openapi + diff --git a/docs/dev/integrationrepo/v/1.13.0/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.13.0/submodules/fpintegration.md new file mode 100644 index 00000000..4831b590 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/submodules/fpintegration.md @@ -0,0 +1,106 @@ +--- +title: "pvarki/python-rasenmaeher-rmfpapi – README" +--- + +> **Integration tag:** `1.13.0` · **Submodule commit:** `7b24c8155518a4e11dd46b1f8d210b850973004c` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-rmfpapi.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-rmfpapi/tree/7b24c8155518a4e11dd46b1f8d210b850973004c + +# rmfpapi + +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rmfpapi:devel_shell . + docker create --name rmfpapi_devel -p 4625:4625 -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` rmfpapi:devel_shell + docker start -i rmfpapi_devel + +In the shell you can start the uvicorn devel server with (binding to +0.0.0.0 is important!): + + uvicorn "rmfpapi.app:get_app" --factory --host 0.0.0.0 --port 4625 --reload --log-level debug + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmfpapi_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rmfpapi_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rmfpapi:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rmfpapi:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` rmfpapi:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rmfpapi:latest . + docker run -it --name rmfpapi -p 4625:4625 rmfpapi:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.13.0/submodules/keycloak2.md b/docs/dev/integrationrepo/v/1.13.0/submodules/keycloak2.md new file mode 100644 index 00000000..c0533b83 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/submodules/keycloak2.md @@ -0,0 +1,31 @@ +--- +title: "pvarki/docker-keycloak – README" +--- + +> **Integration tag:** `1.13.0` · **Submodule commit:** `8b2efdf6a4d90fd91d2074617983c086ebe537c5` +> **Repo:** git@github.com:pvarki/docker-keycloak.git +> **Browse at this commit:** https://github.com/pvarki/docker-keycloak/tree/8b2efdf6a4d90fd91d2074617983c086ebe537c5 + +# KeyCloak + +Templates, init container etc to setup keycloak automagically with ENV +variables. + +## Used as git submodule + +This repo is used as submodule in +[https://github.com/pvarki/docker-rasenmaeher-integration](https://github.com/pvarki/docker-rasenmaeher-integration) it is +probably a good idea to handle all development via it because it has +docker composition for bringin up all the other services rasenmaeher-api +depends on + +## pre-commit notes + +Make sure pre-commit is installed: + + pre-commit install --install-hooks + +And it's a good idea to run it regularly before committing: + + pre-commit run --all-files + diff --git a/docs/dev/integrationrepo/v/1.13.0/submodules/kw_product_init.md b/docs/dev/integrationrepo/v/1.13.0/submodules/kw_product_init.md new file mode 100644 index 00000000..80574db5 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/submodules/kw_product_init.md @@ -0,0 +1,78 @@ +--- +title: "pvarki/golang-kraftwerk-init-helper-cli – README" +--- + +> **Integration tag:** `1.13.0` · **Submodule commit:** `0dee1f8f397ba37280f6bf393a3b866d93387e8c` +> **Repo:** git@github.com:pvarki/golang-kraftwerk-init-helper-cli.git +> **Browse at this commit:** https://github.com/pvarki/golang-kraftwerk-init-helper-cli/tree/0dee1f8f397ba37280f6bf393a3b866d93387e8c + +golang-kraftwerk-init-helper-cli =============================== + +Tool for products to create their certificates from +KRAFTWERK manifests. + +# Development + +## Prerequisites + +**Enable** +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/) + +```bash +export DOCKER_BUILDKIT=1 +``` + +**Forward SSH-agent to running instance:** + +**OSX:** + +```bash +export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" +``` + +**Linux:** + +```bash +export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" +``` + +## Create & start development container + +Build the image, create a container and start the container + +```bash +docker build --ssh default --target dev_shell -t kraftwerk_init_helper:dev_shell . +``` + +```bash +docker create --name kraftwerk_init_helper -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` kraftwerk_init_helper:dev_shell +``` + +```bash +docker start -i kraftwerk_init_helper +``` + +## pre-commit initialization + +Once inside the container, run: + +```bash +pre-commit install && pre-commit run --all-files +``` + +That's it, now you have the development environment up & running. + +# Production + +Build the production image: + +```bash +docker build --ssh default --target production -t kraftwerk_init_helper:latest . +``` + +Run the image: + +```bash +docker run --rm -it --name kraftwerk_helper kraftwerk_init_helper:latest +``` + diff --git a/docs/dev/integrationrepo/v/1.13.0/submodules/miniwerk.md b/docs/dev/integrationrepo/v/1.13.0/submodules/miniwerk.md new file mode 100644 index 00000000..3ebbbe31 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/submodules/miniwerk.md @@ -0,0 +1,104 @@ +--- +title: "pvarki/python-rasenmaeher-miniwerk – README" +--- + +> **Integration tag:** `1.13.0` · **Submodule commit:** `073f866ad4b47dc7d1eab0cc06ecaf45c5fc5c4d` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-miniwerk.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-miniwerk/tree/073f866ad4b47dc7d1eab0cc06ecaf45c5fc5c4d + +# miniwerk + +Minimal KRAFTWERK amulation to be able to run a RASENMAEHER+products +deployment on any VM + +There are some required ENV configs check out example_env.sh for them +(.env file is also supported) + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t miniwerk:devel_shell . + docker create --name miniwerk_devel -v `pwd`":/app" -p 80:80 -it `echo $DOCKER_SSHAGENT` miniwerk:devel_shell + docker start -i miniwerk_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i miniwerk_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i miniwerk_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" miniwerk:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t miniwerk:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` miniwerk:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t miniwerk:amd64-latest . + docker run -it --name miniwerk miniwerk:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.13.0/submodules/mtxauthz.md b/docs/dev/integrationrepo/v/1.13.0/submodules/mtxauthz.md new file mode 100644 index 00000000..db2637cc --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/submodules/mtxauthz.md @@ -0,0 +1,122 @@ +--- +title: "pvarki/python-mediamtx-rmmtxauthz – README" +--- + +> **Integration tag:** `1.13.0` · **Submodule commit:** `55921125444547f67089706f64432e5e3aef7763` +> **Repo:** git@github.com:pvarki/python-mediamtx-rmmtxauthz.git +> **Browse at this commit:** https://github.com/pvarki/python-mediamtx-rmmtxauthz/tree/55921125444547f67089706f64432e5e3aef7763 + +# rmmtxauthz + +Do HTTP based authz for MediaMTX + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rmmtxauthz:devel_shell . + docker create --name rmmtxauthz_devel -v "$(pwd)/rune/output/rune.json:/opt/templates/mediamtx.json" -v "$(pwd):/app" -it $(echo $DOCKER_SSHAGENT) rmmtxauthz:devel_shell + docker start -i rmmtxauthz_devel + +To rebuild the documentation inside the container run: + + rune rune/src json >/opt/templates/mediamtx.json + +Outside of container use: + + rune rune/src json >rune/output/rune.json + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmmtxauthz_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rmmtxauthz_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v "$(pwd):/app" rmmtxauthz:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rmmtxauthz:tox . + docker run --rm -it -v "$(pwd):/app" $(echo $DOCKER_SSHAGENT) rmmtxauthz:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rmmtxauthz:amd64-latest . + docker run -it --name rmmtxauthz rmmtxauthz:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p $(which python3.11) my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + git add poetry.lock + pre-commit install --install-hooks + pre-commit run --all-files + +If you get weird errors about missing packages from pre-commit try +running it with "poetry run pre-commit". + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + +Running "pre-commit run --all-files" and "py.test -v" regularly during +development and especially before committing will save you some +headache. + +### RUNE instructions compile + +tldr: + + rune rune/src json >rune/output/mediamtx.json + diff --git a/docs/dev/integrationrepo/v/1.13.0/submodules/takintegration.md b/docs/dev/integrationrepo/v/1.13.0/submodules/takintegration.md new file mode 100644 index 00000000..62b9bd83 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/submodules/takintegration.md @@ -0,0 +1,100 @@ +--- +title: "pvarki/python-tak-rmapi – README" +--- + +> **Integration tag:** `1.13.0` · **Submodule commit:** `0f3021ec2771c0d3846b46a302692f97fa7f2a19` +> **Repo:** git@github.com:pvarki/python-tak-rmapi.git +> **Browse at this commit:** https://github.com/pvarki/python-tak-rmapi/tree/0f3021ec2771c0d3846b46a302692f97fa7f2a19 + +# takrmapi + +RASENMAEHER integration API for TAK server + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t takrmapi:devel_shell . + docker create --name takrmapi_devel -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` takrmapi:devel_shell + docker start -i takrmapi_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i takrmapi_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i takrmapi_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" takrmapi:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t takrmapi:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` takrmapi:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t takrmapi:amd64-latest . + docker run -it --name takrmapi takrmapi:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.13.0/submodules/takserver.md b/docs/dev/integrationrepo/v/1.13.0/submodules/takserver.md new file mode 100644 index 00000000..b940b56f --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/submodules/takserver.md @@ -0,0 +1,68 @@ +--- +title: "pvarki/docker-atak-server – README" +--- + +> **Integration tag:** `1.13.0` · **Submodule commit:** `7a73302fef52d0e4b76b11bbda92afefaab43732` +> **Repo:** git@github.com:pvarki/docker-atak-server.git +> **Browse at this commit:** https://github.com/pvarki/docker-atak-server/tree/7a73302fef52d0e4b76b11bbda92afefaab43732 + +# Run TAK Java server in container + +tldr: + + cp takserver.env.example takserver.env + # edit the env + # export the variables gomplate uses (see the .tpl files) + docker compose pull --include-deps --ignore-pull-failures + docker compose -p tak up -d + +or use docker compose.local.yml without gomplate for local dev +(rebuilding containers): + + export DOCKER_TAG_EXTRA="-dev" + docker build --no-cache --progress=plain -t takserver:latest${DOCKER_TAG_EXTRA} -t takserver:4.7-RELEASE-32${DOCKER_TAG_EXTRA} -t pvarki/takserver:4.7-RELEASE-32${DOCKER_TAG_EXTRA} . + cp takserver.env.example takserver.env + # edit the env + docker compose -f docker-compose.local.yml -p tak up + +Note, for things that live in the volumes (like TAK certs) you must nuke +the volumes to see changes: + + docker compose -f docker-compose.local.yml -p tak down -v ; docker compose -f docker-compose.local.yml -p tak rm -vf + +## Creating client packages locally + +Using the REST API is probably nicer though + +Create client package: + + docker compose -p tak exec takserver_api /bin/bash -c 'CLIENT_CERT_NAME=replaceme /opt/scripts/make_client_zip.sh' + +Then get /opt/tak/certs/files/clientpkgs/replaceme.zip out of the +container: + + docker compose -p tak exec taktakserver_apiserver /bin/bash -c 'base64 /opt/tak/certs/files/clientpkgs/replaceme.zip' | base64 -id >replaceme.zip + +This approach also works for recovering the admin cert +(/opt/tak/certs/files/admin.p12 unless you changed the ADMIN_CERT_NAME +ENV) + +## Creating new admin users + +Create the user on the takserver container: + + docker compose -p tak exec takserver_api /bin/bash -c 'cd /opt/tak/data/certs/ && CAPASS=$CA_PASS PASS=replaceme_user_cert_pass ./makeCert.sh client replaceme_username && ADMIN_CERT_NAME=replaceme_username /opt/scripts/enable_admin.sh' + +See above about the hard way of getting the cert file, or use the REST +API. + +## Gradle builds + +Build the distribution: + + mkdir outputs + docker build --progress=plain -f Dockerfile_build --target files -t atakbuild:files . + docker run --rm -it -v `pwd`/outputs:/output atakbuild:files + +Now you have the build artefacts in outputs -directory. + diff --git a/docs/dev/integrationrepo/v/1.13.0/submodules/ui.md b/docs/dev/integrationrepo/v/1.13.0/submodules/ui.md new file mode 100644 index 00000000..82db9fd8 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.13.0/submodules/ui.md @@ -0,0 +1,116 @@ +--- +title: "pvarki/rasenmaeher-ui – README" +--- + +> **Integration tag:** `1.13.0` · **Submodule commit:** `eb2d303774fa7791ba5410080cd55316fe8b32c1` +> **Repo:** git@github.com:pvarki/rasenmaeher-ui.git +> **Browse at this commit:** https://github.com/pvarki/rasenmaeher-ui/tree/eb2d303774fa7791ba5410080cd55316fe8b32c1 + +# UI For RASENMAEHER + +React, TS etc + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rasenmaeher_ui:devel_shell . + docker create --name rmui_devel -v `pwd`":/app" -p 8002:8002 -it `echo $DOCKER_SSHAGENT` rasenmaeher_ui:devel_shell + docker start -i rmui_devel + +Note: due to the volume mapping if you already had run "npm install" on +the host you need to delete the node_modules directory and in any case +you need to run "npm install" inside the container. + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmui_devel /bin/bash -c "pre-commit install" + docker exec -i rmui_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rasenmaeher_ui:devel_shell -c "pre-commit run --all-files" + +### Production docker + +This will just deliver the compiled files to a mounted directory + +> docker build --ssh default --target production -t +> rasenmaeher_ui:amd64-latest . docker run -it --rm -v +> pwd/rmui_files:/deliver +> rasenmaeher_ui:amd64-latest + +Then you need to serve things so that index.html is the default +controller for things, nginx example: + + location / { + index index.html; + root /rmui_files; + try_files $uri $uri/ /index.html =404; + } + +## Development + +TLDR: + +- Create and activate a Node 18 virtualenv: + + FIXME: Insert instructions + +- change to a branch: + + git checkout -b my_branch + +- Install project deps and pre-commit hooks: + + npm install + pre-commit install + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo. + +### Asset set + +Asset set is loaded according to VITE_ASSET_SET enviroment variable. +Changing asset set requires reloading vite. Folder assetSetStore +contains files for for different sets that are copied to src/assets/set/ +folder on startup. Current set is tracked with setName.txt file. + +When you want to use dynamic assets, you want to make sure your dynamic +asset has equivalent for sets (both "fdf" and "neutral", so on). If the +assets are images, you want to then import them to your components from +/assets/set/\* where the env-defined assets are placed by VITE on +startup. If the assets are translation keys, make sure keys are present +in locales under assetSetStore, and then use the "dynamic" namespace for +accessing them. + diff --git a/docs/dev/integrationrepo/v/1.14.0/_category_.json b/docs/dev/integrationrepo/v/1.14.0/_category_.json new file mode 100644 index 00000000..6a374478 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "1.14.0", + "position": 4 +} diff --git a/docs/dev/integrationrepo/v/1.14.0/index.md b/docs/dev/integrationrepo/v/1.14.0/index.md new file mode 100644 index 00000000..9b37bfd0 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/index.md @@ -0,0 +1,4 @@ +--- +title: "1.14.0" +--- + diff --git a/docs/dev/integrationrepo/v/1.14.0/integration.md b/docs/dev/integrationrepo/v/1.14.0/integration.md new file mode 100644 index 00000000..76a14011 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/integration.md @@ -0,0 +1,319 @@ +--- +title: "pvarki/docker-rasenmaeher-integration – README" +--- + +> **Integration tag:** `1.14.0` +> **Repo:** https://github.com/pvarki/docker-rasenmaeher-integration + +![Build Status](https://github.com/pvarki/docker-rasenmaeher-integration/actions/workflows/build.yml/badge.svg) + +# Deploy App + +"One Ring to rule them all, One Ring to find them, One Ring to bring +them all and in the darkness bind them." + +Docker compositions, helpers etc to bring it all together into something +resembling grand old ones. + +Codename RASENMAEHER because infantry jokes. + +## WTF is this anyway ? + +This [Disobey24 +talk](https://www.youtube.com/watch?v=m3xd7uygpaY&list=PLLvAhAn5sGfiB9AlEt2KD7H9Dnr6kbd64&index=23) +explains a lot. + +## Running Deploy App in your own docker environment + +### Needed DNS Records + +These need to point to your WAN address. + +> - domain +> - kc.domain +> - tak.domain +> - bl.domain +> - mtx.domain +> - mtls.domain +> - mtls.kc.domain +> - mtls.tak.domain +> - mtls.kc.domain +> - mtls.bl.domain +> - mtls.mtx.domain + +When more products are added to the deployment they will follow the same +naming pattern, you will need subdomains for all products listed in the +composition for miniwerk service variable MW_PRODUCTS and "kc" for +Keycloak. + +### Needed ports open + +And redirected to the server if behind NAT or similar. + +> - 80 +> - 443 +> - 8443 (TAK) +> - 8446 (TAK) +> - 9446 (Keycloak) +> - 4626 (Product integration APIs port) +> - 4666 (Battlelog API/UI port) +> - 1936 RTMPS +> - 8322 RTSPS +> - 8890 SRT +> - 9888 HSL +> - 9889 WebRTC +> - 9996 Playback for recorded streams/videos + +### Downloading and composing Deploy App + +Be mindfull on where you download the repository, you will need to +perform rest of the commands inside the downloaded repository. + +Getting the repository from github (on Windows **first** see "Windows +notes" below): + + git clone --recurse-submodules -j8 git@github.com:pvarki/docker-rasenmaeher-integration.git + +Create `.env` file that defines environmental variables for Deploy App +setup. File must be located inside downloaded repository and file type +must be named literally `.env` (not `something.env`) to work. + +The original example file is: +[https://github.com/pvarki/docker-rasenmaeher-integration/blob/main/example_env.sh](https://github.com/pvarki/docker-rasenmaeher-integration/blob/main/example_env.sh) + +Example .env-file with the minimal information needed: + + KEYCLOAK_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RM_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + POSTGRES_PASSWORD="input-secure-password" # pragma: allowlist secret + LDAP_ADMIN_PASSWORD="input-secure-password" # pragma: allowlist secret + KEYCLOAK_ADMIN_PASSWORD="input-secure-password" # pragma: allowlist secret + TAK_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + SERVER_DOMAIN="input-domain" + CFSSL_CA_NAME="input-ca-name" + MW_LE_EMAIL="input-email-for-lets-encrypt" + MW_LE_TEST="false" + TAKSERVER_CERT_PASS="input-secure-password" # pragma: allowlist secret + TAK_CA_PASS="input-secure-password" # pragma: allowlist secret + VITE_ASSET_SET="${VITE_ASSET_SET:-neutral}" + KEYCLOAK_PROFILEROOT_UUID="input-uuid" + KEYCLOAK_HTTPS_KEY_STORE_PASSWORD="input-secure-password" # pragma: allowlist secret + KEYCLOAK_HTTPS_TRUST_STORE_PASSWORD="input-secure-password" # pragma: allowlist secret + BL_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RMMTX_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RMMTX_API_PASSWORD="input-secure-password" # pragma: allowlist secret + +Replace "intput-secure-password" with a good passphrase that is unique +for each replacment. You can generate an UUID with: + + python -c 'import uuid; print(uuid.uuid4())' + +If you wish to use one deployment for longer than the *design lifetime* +of 1-2 months you can change the following env variables. But do +understand that this is **not recommended** and has **security +implications**. If you do this **you** take **responsibility** to go +through all Dockerfiles and compositions to understand **exactly** how +things are done and how apply security updates into the containers: + + CFSSL_CA_EXPIRY="8800h" # Input time in hours = xxh, 8800h is 366.. days + CFSSL_SIGN_DEFAULT_EXPIRY="8800h" # Input time in hours = xxh, 8800h is 366.. days + +Starting the services: + + docker compose up -d + +Updating the repository from github: + + git pull + git submodule update --init --recursive + docker compose pull + docker compose up -d + +!DO NOT DO! Deleting the services. Deletes the certificates etc you will +need to add all users etc again: + + docker compose down -v + +Getting the admin login invite code for first admin: + + docker compose exec -it rmapi /bin/bash -c "rasenmaeher_api addcode" + +If you get "no such service -d", make sure whatever you copy-pasted the +command from did not render the dash (ASCII 0x2D) as some other +codepoint with a glyph that looks deceptively similar. When in doubt +write the commands yourself instead of copy-pasting. + +### Services + +Deploy App login page: + + https://domain (example.com) + +Deploy App home page: + + https://mtls.domain (mtls.example.com) + +Takserver Admin UI: + + https://tak.domain:8443/ (tak.example.com:8443/) + +Keycloack Admin UI. (Later group management will be withing Deploy App): + + https://kc.domain:9443/admin/RASENMAEHER/console/ (kc.example.com:9443/admin/RASENMAEHER/console/) + +OTA update server inside takserver. Is located in the loaded repository, +location depends on where you downloaded it: + + /home/user/docker-rasenmaeher-integration/takserver/update + +### Using the Deploy App service + +1. Login with first admin code. Create your admin account by typing + your first admin invite code and inputting desired admin callsign. +2. Create invite code for other users. Share the invite code. Go to + Manage Users -\> Add Users -\> Create New Invite. Share link, qr + code or invite code and domain. +3. Approve users in Deploy App. Open approvement link or scan qr code + from users and approve the user. You can also go to Approve Users + -\> Select Waiting User and input the users approvement code. +4. If desired promote some of the added users as admins. Go to Manage + Users -\> Manage Users -\> Select user and select Promote. You can + also Demote Admins or Delete users altogether. + +### Using Deploy App TAK in EUD + +EUD=End User Device + +1. Login to Deploy App. Go to [https://mtls.domain](https://mtls.domain) and select TAK. +2. Download Client Package. Select tak package for desired software + "Android ATAK or Windows WinTAK" or "iOS iTAK". Select Download + Client Package. +3. Go to EUD's TAK Software. Import downloaded package. Device is + connected to server. +4. You should also read Quickstart and Usage Guides. + +## Git submodules + +When cloning for the first time use: + + git clone --recurse-submodules -j8 git@github.com:pvarki/docker-rasenmaeher-integration.git + +When updating or checking out branches use: + + git submodule update --init --recursive + +And if you forgot to --recurse-submodules run the update command above. + +The submodules are repos in their own right, if you plan to make changes +into them change to the directory and create new branch, commit and push +changes as usual under that directory. + +### Directories that are submodules + +> - api [https://github.com/pvarki/python-rasenmaeher-api](https://github.com/pvarki/python-rasenmaeher-api) +> - cfssl [https://github.com/pvarki/docker-rasenmaeher-cfssl](https://github.com/pvarki/docker-rasenmaeher-cfssl) +> - fpintegration [https://github.com/pvarki/python-rasenmaeher-rmfpapi](https://github.com/pvarki/python-rasenmaeher-rmfpapi) +> - keycloak [https://github.com/pvarki/docker-keycloak](https://github.com/pvarki/docker-keycloak) +> - kw_product_init +> [https://github.com/pvarki/golang-kraftwerk-init-helper-cli](https://github.com/pvarki/golang-kraftwerk-init-helper-cli) +> - openldap [https://github.com/pvarki/docker-openldap](https://github.com/pvarki/docker-openldap) +> - miniwerk [https://github.com/pvarki/python-rasenmaeher-miniwerk](https://github.com/pvarki/python-rasenmaeher-miniwerk) +> - ui [https://github.com/pvarki/rasenmaeher-ui](https://github.com/pvarki/rasenmaeher-ui) +> - takserver [https://github.com/pvarki/docker-atak-server](https://github.com/pvarki/docker-atak-server) +> - takintegration [https://github.com/pvarki/python-tak-rmapi](https://github.com/pvarki/python-tak-rmapi) +> - battlelog [https://github.com/pvarki/typescript-liveloki-app](https://github.com/pvarki/typescript-liveloki-app) +> - mtxauthz [https://github.com/pvarki/python-mediamtx-rmmtxauthz](https://github.com/pvarki/python-mediamtx-rmmtxauthz) + +## Autogenerated (mostly API) docs + +> - Module API docs: +> [https://pvarki.github.io/docker-rasenmaeher-integration/docs/](https://pvarki.github.io/docker-rasenmaeher-integration/docs/) +> - Swagger definition for Deploy App API: +> [https://pvarki.github.io/docker-rasenmaeher-integration/](https://pvarki.github.io/docker-rasenmaeher-integration/) + +## Running in local development mode + +### Windows notes + +> 1. Do **NOT** use git-bash, it will cause *weirdest* issues with +> Docker containers +> +> 2. Use WSL, see +> [best_practises](https://github.com/pvarki/markdown-pvarki-best_practises) +> -repo for instructions on how to set it up. +> +> 3. Make sure whatever git client or IDE you use it does not mess with +> line-endings, for CLI client this does the trick: +> +> git config --global core.eol lf +> git config --global core.autocrlf false + +### Compositions + +Generally start with "rmlocal", it corresponds best to a real running +environment. "rmdev" starts a bunch of things in development mode which +does make developing more convenient but also introduces extra +variability to how things work. + +Make sure to always check your changes work correctly in rmlocal mode +where assets are minified and baked in. + +TLDR: + + alias rmlocal="docker compose -p rmlocal -f docker-compose-local.yml" + rmlocal build takinit miniwerk --pull + rmlocal build --pull + rmlocal up + +or: + + alias rmdev="docker compose -p rmdev -f docker-compose-local.yml -f docker-compose-dev.yml" + rmdev build takinit miniwerk --pull + rmdev build --pull + rmdev up + +OpenLDAP and keycloak-init sometimes fail on first start, just run up +again. + +IMPORTANT: Only keep either rmlocal or rmdev created at one time or you +may have weird network issues run "down" for one env before starting the +other. + +Remember to run "down -v" if you want to reset the persistent volumes, +or if you have weird issues when switching between environments. + +The dev version launches all the services and runs rasenmaeher-api in +uvicorn reload mode so any edits you make under /api will soon be +reflected in the running instance. + +If rasenmaeher-ui devel server complains make sure to delete +`ui/node_modules` -directory from host first. The docker NodeJS +distribution probably is not compatible with whatever you have installed +on the host. + +### Gaining first admin access in dev and production mode + +In dev mode: + + docker exec -it rmdev-rmapi-1 /bin/bash -c "source /.venv/bin/activate && rasenmaeher_api addcode" + +Under dev mode, the UI runs at [https://localmaeher.dev.pvarki.fi:4439](https://localmaeher.dev.pvarki.fi:4439). + +In VM production mode: + + docker exec -it rmvm-rmapi-1 /bin/bash -c "rasenmaeher_api addcode" + +## pre-commit notes + +Use "pre-commit run --all-files" liberally (and make sure you have run +"pre-commit install --install-hooks"). If you get complaints about +missing environment variables run "source example_env.sh" + +## Integration tests + +Pytest is used to handle the integration tests, the requirements are in +tests/requirements.txt. NOTE: The tests have side-effects and expect a +clean database to start with so always make sure to run "down -v" for +the composition first, then bring it back up before running integration +tests. + diff --git a/docs/dev/integrationrepo/v/1.14.0/submodules/api.md b/docs/dev/integrationrepo/v/1.14.0/submodules/api.md new file mode 100644 index 00000000..4080c197 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/submodules/api.md @@ -0,0 +1,240 @@ +--- +title: "pvarki/python-rasenmaeher-api – README" +--- + +> **Integration tag:** `1.14.0` · **Submodule commit:** `0b51a68dda421693c8081505175b1250403bc595` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-api.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-api/tree/0b51a68dda421693c8081505175b1250403bc595 + +# python-rasenmaeher-api + +python-rasenmaeher-api + +## Configuration + +This application can be configured with environment variables. Below is +a example list of available variables. You can find all available +variables here +[https://github.com/pvarki/python-rasenmaeher-api/blob/main/src/rasenmaeher_api/settings.py](https://github.com/pvarki/python-rasenmaeher-api/blob/main/src/rasenmaeher_api/settings.py). + +| ENV VAR | Default value | Info / Usage | +|---------------------------|---------------------------------------------|---------------------------------------------------------------| +| RM_PORT | 8000 | Docker API listen port | +| RM_WORKERS_COUNT | 2 | Number of uvicorn workers | +| RM_RELOAD | False | Reload service on code change | +| RM_ENVIRONMENT | dev | Run dev / prod environment | +| RM_CFSSL_HOST | None | CFSSL host url | +| RM_CFSSL_PORT | None | CFSSL service port | +| RM_KEYCLOAK_SERVER_URL | None | Keycloak server url ([http://1234:8080/auth](http://1234:8080/auth)) | +| RM_KEYCLOAK_CLIENT_ID | None | Keycloak client id | +| RM_KEYCLOAK_REALM_NAME | None | Keycloak realm name | +| RM_KEYCLOAK_CLIENT_SECRET | None | Keycloak secert | +| RM_LDAP_CONN_STRING | None | LDAP conn string | +| RM_LDAP_USERNAME | None | LDAP username | +| RM_LDAP_CLIENT_SECRET | None | LDAP connection secret | +| RM_SQLITE_FILEPATH_PROD | /opt/rasenmaher/persistent/sqlite/rm_db.sql | location for sqlite database file in "prod" | +| RM_SQLITE_FILEPATH_DEV | /tmp/rm_db.sql | location for sqlite database file in "dev", local development | + +Container Variables + +You can create .env file in the root +directory and place all environment variables here. + +All environment variables should start with +RM\_ prefix. + +For example if you see in your "rasenmaeher_api/settings.py" a variable +named like random_parameter, you should +provide the "RM_RANDOM_PARAMETER" variable to configure the value. This +behaviour can be changed by overriding +env_prefix property in +rasenmaeher_api.settings.Settings.Config. + +An example of .env file: +`` `bash RM_RELOAD="True" RM_PORT="8000" RM_ENVIRONMENT="dev" ``\` + +You can read more about BaseSettings class here: +[https://pydantic-docs.helpmanual.io/usage/settings/](https://pydantic-docs.helpmanual.io/usage/settings/) + +## Backend services manifest + +Integration APIs are always https in port 4625. Default location for the +manifest is /pvarki/kraftwerk-rasenmaeher-init.json + +`` `json { "dns": "sleepy-sloth.pvarki.fi", "products": { "tak": "tak.sleepy-sloth.pvarki.fi", "ptt": "ptt.sleepy-sloth.pvarki.fi" } } ``\` + +## Api endpoints and usage + +| State | Method | URI | Request JSON | Response JSON | Api description . | +|---------|--------|------------------------------------|--------------------------------------------------------------|-------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| Dummy | GET | /api/v1/enroll/status/{work_id} | NA | {'status':'None/Processing/Denied/WaitingForAcceptance/ReadyForDelivery/Delivered'} | Check the situation of enrollment process, None = no enrollment started, this work_id is free to use. | +| Dummy | POST | /api/v1/enroll/init | {'work_id':'{work_id}'} | {'work_id':'{work_id}', 'id_hash':{id_string} } | Start service access enrollment for given {work_id} | +| Dummy | GET | /api/v1/enroll/deliver/{id_string} | NA | {'dl_link':"{[http://here.be/zip](http://here.be/zip)}"} | Deliver download link for enrollment zip | +| Dummy | POST | /api/v1/enroll/accept | { 'permit_string':'{permit_string}, 'id_hash':'{id_hash}' '} | { 'access':'granted/denied/None', 'work_id':'{work_id}' } | Accept the enrollment request | +| Testing | POST | /api/v1/product/sign_csr | { 'TO':'DO'} | { 'TO':'DO'} | Accept the enrollment request | +| Testing | POST | /api/v1/enroll/accept | { 'permit_string':'{permit_string}, 'id_hash':'{id_hash}' '} | { 'access':'granted/denied/None', 'work_id':'{work_id}' } | Accept the enrollment request | + +API vimpain + +## Example usage + +\# REQUEST A NEW CERTIFICATE USING CSR (requires cfssl backend for the +api container) + +> `` `curl -L -H "Content-Type: application/json" -d '{"csr": "-----BEGIN CERTIFICATE REQUEST-----\nMIIBUjCB+QIBADBqMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLZXhhbXBsZS5jb20x\nFjAUBgNVBAcTDVNhbiBGcmFuY2lzY28xEzARBgNVBAgTCkNhbGlmb3JuaWExGDAW\nBgNVBAMTD3d3dy5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA\nBK/CtZaQ4VliKE+DLIVGLwtSxJgtUKRzGvN1EwI3HRgKDQ3l3urBIzHtUcdMq6HZ\nb8jX0O9fXYUOf4XWggrLk1agLTArBgkqhkiG9w0BCQ4xHjAcMBoGA1UdEQQTMBGC\nD3d3dy5leGFtcGxlLmNvbTAKBggqhkjOPQQDAgNIADBFAiAcvfhXnsLtzep2sKSa\n36W7G9PRbHh8zVGlw3Hph8jR1QIhAKfrgplKwXcUctU5grjQ8KXkJV8RxQUo5KKs\ngFnXYtkb\n-----END CERTIFICATE REQUEST-----\n"}' 127.0.0.1:8000/api/v1/takreg | jq ``\` + +\# REQUEST A NEW CERTIFICATE WITHOUT CSR (requires cfssl backend for the +api container) + +> `` `curl -L -H "Content-Type: application/json" -d '{ "request": {"hosts":["harjoitus1.pvarki.fi"], "names":[{"C":"FI", "ST":"Jyvaskyla", "L":"KeskiSuomi", "O":"harjoitus1.pvarki.fi"}], "CN": "harjoitus1.pvarki.fi"}, "bundle":true, "profile":"client"}' 127.0.0.1:8000/takreg | jq ``\` + +\# LIST CFSSL CRL LIST + +> `` `curl -L -H "Content-Type: application/json" -d '{ "request": {"hosts":["harjoitus1.pvarki.fi"], "names":[{"C":"FI", "ST":"Jyvaskyla", "L":"KeskiSuomi", "O":"harjoitus1.pvarki.fi"}], "CN": "harjoitus1.pvarki.fi"}, "bundle":true, "profile":"client"}' 127.0.0.1:8000/takreg | jq ``\` + +The cfssl used behind API listens this kind of stuff +[https://github.com/cloudflare/cfssl/blob/master/doc/api/endpoint_newcert.txt](https://github.com/cloudflare/cfssl/blob/master/doc/api/endpoint_newcert.txt) + +\# Enrollment - Enroll a new work_id + +> `` `curl -H "Content-Type: application/json" -d '{"work_id":"porakoira666"}' http://127.0.0.1:8000/api/v1/enrollment/init ``\` + +\# Enrollment - Check the status and availability of work_id + +> `` `curl http://127.0.0.1:8000/api/v1/enrollment/status/koira ``\` + +\# Enrollment - Request the download link using the provided work_id_hash +`` `curl http://127.0.0.1:8000/api/v1/enrollment/deliver/zxzxzxzxzxzxzxxzzx ``\` + +\# Enrollment - Accept enrollment using permit_str +`` `curl -H "Content-Type: application/json" -d '{"enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/accept ``\` + +\# Enrollment - Set download link for enrollment +`` `curl -H "Content-Type: application/json" -d '{"download_link":"https://kuvaton.com","enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/set-dl-link ``\` + +\# Enrollment - Set state for enrollment +`` `curl -H "Content-Type: application/json" -d '{"state":"ReadyForDelivery","enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/set-state ``\` + +\# Enrollment - Add new permit_str +`` `curl -H "Content-Type: application/json" -d '{"permissions_str":"all", "new_permit_hash":"too_short","permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/add-manager ``\` + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rasenmaeher_api:devel_shell . + docker create --name rasenmaeher_api_devel -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` rasenmaeher_api:devel_shell + docker start -i rasenmaeher_api_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rasenmaeher_api_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rasenmaeher_api_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rasenmaeher_api:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rasenmaeher_api:tox . + docker run --rm -it --network host -v /var/run/docker.sock:/var/run/docker.sock -v `pwd`":/app" `echo $DOCKER_SSHAGENT` rasenmaeher_api:tox + +NOTE: This will not work from the git submodule directory in the +integration repo, docker tests must be run from the original repo. + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rasenmaeher_api:amd64-latest . + docker run --rm -it --name rasenmaeher_openapijson rasenmaeher_api:amd64-latest rasenmaeher_api openapi + docker run -it --name rasenmaeher_api rasenmaeher_api:amd64-latest + +There is also a specific target for just dumping the openapi.json: + + docker build --ssh default --target openapi -t rasenmaeher_api:amd64-openapi . + docker run --rm -it --name rasenmaeher_openapijson rasenmaeher_api:amd64-openapi + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.14.0/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.14.0/submodules/battlelog.md new file mode 100644 index 00000000..9dd2eb40 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/submodules/battlelog.md @@ -0,0 +1,53 @@ +--- +title: "pvarki/typescript-liveloki-app – README" +--- + +> **Integration tag:** `1.14.0` · **Submodule commit:** `197f6b74507fb81fc442fa1522088f0cb7ed3939` +> **Repo:** git@github.com:pvarki/typescript-liveloki-app.git +> **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/197f6b74507fb81fc442fa1522088f0cb7ed3939 + +# BattleLog + +## Install + +1. Install docker + compose +2. Rename .env_example --> .env and modify as you like. +3. Run docker compose build --no-cache +4. Run docker compose up -d +5. Navigate to localhost:3000 + +## Frontend development + +The frontend is bundled with [Vite](https://vitejs.dev/). + +Once the backend is running (on port 3000), you can navigate to `frontend/`, +run `npm i` and `npm run dev` to run the Vite development server that has +hot reload and all that jazz. + +## Info + +Database is currently preseeded with test data from preseed/preseed.csv + +## Migrations + +To add new migration, locally run: `./node_modules/.bin/node-pg-migrate create ` and modify created file in `/migrations` directory. +Migrations are run (if needed) when docker container starts. `wait-for-it.sh` will ensure that psql container is up and accepting connections before running migrations. + +## JWT testing + +Remove comments from "jwt-test-network" for local jwt testing, to allow joining containers from 2 different compose runs to same docker network. + +## OpenAPI Specification + +https://pvarki.github.io/typescript-liveloki-app/ + +## Running tests + +In directory `tests`, use + +```shell +npm run test:integration +``` + +with the server running in `localhost:3000` as it does by default after `docker compose up -d`. + diff --git a/docs/dev/integrationrepo/v/1.14.0/submodules/cfssl.md b/docs/dev/integrationrepo/v/1.14.0/submodules/cfssl.md new file mode 100644 index 00000000..f7c6bead --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/submodules/cfssl.md @@ -0,0 +1,91 @@ +--- +title: "pvarki/docker-rasenmaeher-cfssl – README" +--- + +> **Integration tag:** `1.14.0` · **Submodule commit:** `87f2c2b4b22a0d25d7dcc900d0750373b24fec4b` +> **Repo:** git@github.com:pvarki/docker-rasenmaeher-cfssl.git +> **Browse at this commit:** https://github.com/pvarki/docker-rasenmaeher-cfssl/tree/87f2c2b4b22a0d25d7dcc900d0750373b24fec4b + +# cfssl Submodule + +\![Build +Status\]([https://github.com/pvarki/docker-rasenmaeher-cfssl/actions/workflows/build.yml/badge.svg](https://github.com/pvarki/docker-rasenmaeher-cfssl/actions/workflows/build.yml/badge.svg)) + +## Used as git submodule + +This repo is used as submodule in +[https://github.com/pvarki/docker-rasenmaeher-integration](https://github.com/pvarki/docker-rasenmaeher-integration) it is +probably a good idea to handle all development via it because it has +docker composition for bringin up all the other services rasenmaeher-api +depends on + +## Development + +The cfssl itself we can't do much about but the FastAPI thing uses +poetry and in any case use pre-commit: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t ocsprest:devel_shell . + docker create --name ocsprest_devel -v `pwd`/../":/app" -it `echo $DOCKER_SSHAGENT` ocsprest:devel_shell + docker start -i ocsprest_devel + +Or just pwd if working under separate checkout instead of the +integration repo. + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i ocsprest_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i ocsprest_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" ocsprest:devel_shell -c "pre-commit run --all-files" + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target ocsprest -t ocsprest:amd64-latest . + docker run -it --name ocsprest ocsprest:amd64-latest + +There is also a specific target for just dumping the openapi.json: + + docker build --ssh default --target openapi -t ocsprest:amd64-openapi . + docker run --rm -it --name rasenmaeher_openapijson ocsprest:amd64-openapi + diff --git a/docs/dev/integrationrepo/v/1.14.0/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.14.0/submodules/fpintegration.md new file mode 100644 index 00000000..78a6a7da --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/submodules/fpintegration.md @@ -0,0 +1,106 @@ +--- +title: "pvarki/python-rasenmaeher-rmfpapi – README" +--- + +> **Integration tag:** `1.14.0` · **Submodule commit:** `7b24c8155518a4e11dd46b1f8d210b850973004c` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-rmfpapi.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-rmfpapi/tree/7b24c8155518a4e11dd46b1f8d210b850973004c + +# rmfpapi + +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rmfpapi:devel_shell . + docker create --name rmfpapi_devel -p 4625:4625 -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` rmfpapi:devel_shell + docker start -i rmfpapi_devel + +In the shell you can start the uvicorn devel server with (binding to +0.0.0.0 is important!): + + uvicorn "rmfpapi.app:get_app" --factory --host 0.0.0.0 --port 4625 --reload --log-level debug + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmfpapi_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rmfpapi_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rmfpapi:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rmfpapi:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` rmfpapi:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rmfpapi:latest . + docker run -it --name rmfpapi -p 4625:4625 rmfpapi:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.14.0/submodules/keycloak2.md b/docs/dev/integrationrepo/v/1.14.0/submodules/keycloak2.md new file mode 100644 index 00000000..76544609 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/submodules/keycloak2.md @@ -0,0 +1,31 @@ +--- +title: "pvarki/docker-keycloak – README" +--- + +> **Integration tag:** `1.14.0` · **Submodule commit:** `8b2efdf6a4d90fd91d2074617983c086ebe537c5` +> **Repo:** git@github.com:pvarki/docker-keycloak.git +> **Browse at this commit:** https://github.com/pvarki/docker-keycloak/tree/8b2efdf6a4d90fd91d2074617983c086ebe537c5 + +# KeyCloak + +Templates, init container etc to setup keycloak automagically with ENV +variables. + +## Used as git submodule + +This repo is used as submodule in +[https://github.com/pvarki/docker-rasenmaeher-integration](https://github.com/pvarki/docker-rasenmaeher-integration) it is +probably a good idea to handle all development via it because it has +docker composition for bringin up all the other services rasenmaeher-api +depends on + +## pre-commit notes + +Make sure pre-commit is installed: + + pre-commit install --install-hooks + +And it's a good idea to run it regularly before committing: + + pre-commit run --all-files + diff --git a/docs/dev/integrationrepo/v/1.14.0/submodules/kw_product_init.md b/docs/dev/integrationrepo/v/1.14.0/submodules/kw_product_init.md new file mode 100644 index 00000000..888cb761 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/submodules/kw_product_init.md @@ -0,0 +1,78 @@ +--- +title: "pvarki/golang-kraftwerk-init-helper-cli – README" +--- + +> **Integration tag:** `1.14.0` · **Submodule commit:** `0dee1f8f397ba37280f6bf393a3b866d93387e8c` +> **Repo:** git@github.com:pvarki/golang-kraftwerk-init-helper-cli.git +> **Browse at this commit:** https://github.com/pvarki/golang-kraftwerk-init-helper-cli/tree/0dee1f8f397ba37280f6bf393a3b866d93387e8c + +golang-kraftwerk-init-helper-cli =============================== + +Tool for products to create their certificates from +KRAFTWERK manifests. + +# Development + +## Prerequisites + +**Enable** +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/) + +```bash +export DOCKER_BUILDKIT=1 +``` + +**Forward SSH-agent to running instance:** + +**OSX:** + +```bash +export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" +``` + +**Linux:** + +```bash +export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" +``` + +## Create & start development container + +Build the image, create a container and start the container + +```bash +docker build --ssh default --target dev_shell -t kraftwerk_init_helper:dev_shell . +``` + +```bash +docker create --name kraftwerk_init_helper -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` kraftwerk_init_helper:dev_shell +``` + +```bash +docker start -i kraftwerk_init_helper +``` + +## pre-commit initialization + +Once inside the container, run: + +```bash +pre-commit install && pre-commit run --all-files +``` + +That's it, now you have the development environment up & running. + +# Production + +Build the production image: + +```bash +docker build --ssh default --target production -t kraftwerk_init_helper:latest . +``` + +Run the image: + +```bash +docker run --rm -it --name kraftwerk_helper kraftwerk_init_helper:latest +``` + diff --git a/docs/dev/integrationrepo/v/1.14.0/submodules/miniwerk.md b/docs/dev/integrationrepo/v/1.14.0/submodules/miniwerk.md new file mode 100644 index 00000000..45260e41 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/submodules/miniwerk.md @@ -0,0 +1,104 @@ +--- +title: "pvarki/python-rasenmaeher-miniwerk – README" +--- + +> **Integration tag:** `1.14.0` · **Submodule commit:** `073f866ad4b47dc7d1eab0cc06ecaf45c5fc5c4d` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-miniwerk.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-miniwerk/tree/073f866ad4b47dc7d1eab0cc06ecaf45c5fc5c4d + +# miniwerk + +Minimal KRAFTWERK amulation to be able to run a RASENMAEHER+products +deployment on any VM + +There are some required ENV configs check out example_env.sh for them +(.env file is also supported) + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t miniwerk:devel_shell . + docker create --name miniwerk_devel -v `pwd`":/app" -p 80:80 -it `echo $DOCKER_SSHAGENT` miniwerk:devel_shell + docker start -i miniwerk_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i miniwerk_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i miniwerk_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" miniwerk:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t miniwerk:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` miniwerk:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t miniwerk:amd64-latest . + docker run -it --name miniwerk miniwerk:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.14.0/submodules/mtxauthz.md b/docs/dev/integrationrepo/v/1.14.0/submodules/mtxauthz.md new file mode 100644 index 00000000..ca3c7e02 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/submodules/mtxauthz.md @@ -0,0 +1,122 @@ +--- +title: "pvarki/python-mediamtx-rmmtxauthz – README" +--- + +> **Integration tag:** `1.14.0` · **Submodule commit:** `55921125444547f67089706f64432e5e3aef7763` +> **Repo:** git@github.com:pvarki/python-mediamtx-rmmtxauthz.git +> **Browse at this commit:** https://github.com/pvarki/python-mediamtx-rmmtxauthz/tree/55921125444547f67089706f64432e5e3aef7763 + +# rmmtxauthz + +Do HTTP based authz for MediaMTX + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rmmtxauthz:devel_shell . + docker create --name rmmtxauthz_devel -v "$(pwd)/rune/output/rune.json:/opt/templates/mediamtx.json" -v "$(pwd):/app" -it $(echo $DOCKER_SSHAGENT) rmmtxauthz:devel_shell + docker start -i rmmtxauthz_devel + +To rebuild the documentation inside the container run: + + rune rune/src json >/opt/templates/mediamtx.json + +Outside of container use: + + rune rune/src json >rune/output/rune.json + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmmtxauthz_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rmmtxauthz_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v "$(pwd):/app" rmmtxauthz:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rmmtxauthz:tox . + docker run --rm -it -v "$(pwd):/app" $(echo $DOCKER_SSHAGENT) rmmtxauthz:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rmmtxauthz:amd64-latest . + docker run -it --name rmmtxauthz rmmtxauthz:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p $(which python3.11) my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + git add poetry.lock + pre-commit install --install-hooks + pre-commit run --all-files + +If you get weird errors about missing packages from pre-commit try +running it with "poetry run pre-commit". + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + +Running "pre-commit run --all-files" and "py.test -v" regularly during +development and especially before committing will save you some +headache. + +### RUNE instructions compile + +tldr: + + rune rune/src json >rune/output/mediamtx.json + diff --git a/docs/dev/integrationrepo/v/1.14.0/submodules/takintegration.md b/docs/dev/integrationrepo/v/1.14.0/submodules/takintegration.md new file mode 100644 index 00000000..8305decb --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/submodules/takintegration.md @@ -0,0 +1,100 @@ +--- +title: "pvarki/python-tak-rmapi – README" +--- + +> **Integration tag:** `1.14.0` · **Submodule commit:** `0f3021ec2771c0d3846b46a302692f97fa7f2a19` +> **Repo:** git@github.com:pvarki/python-tak-rmapi.git +> **Browse at this commit:** https://github.com/pvarki/python-tak-rmapi/tree/0f3021ec2771c0d3846b46a302692f97fa7f2a19 + +# takrmapi + +RASENMAEHER integration API for TAK server + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t takrmapi:devel_shell . + docker create --name takrmapi_devel -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` takrmapi:devel_shell + docker start -i takrmapi_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i takrmapi_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i takrmapi_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" takrmapi:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t takrmapi:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` takrmapi:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t takrmapi:amd64-latest . + docker run -it --name takrmapi takrmapi:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.14.0/submodules/takserver.md b/docs/dev/integrationrepo/v/1.14.0/submodules/takserver.md new file mode 100644 index 00000000..6f3042ac --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/submodules/takserver.md @@ -0,0 +1,68 @@ +--- +title: "pvarki/docker-atak-server – README" +--- + +> **Integration tag:** `1.14.0` · **Submodule commit:** `7a73302fef52d0e4b76b11bbda92afefaab43732` +> **Repo:** git@github.com:pvarki/docker-atak-server.git +> **Browse at this commit:** https://github.com/pvarki/docker-atak-server/tree/7a73302fef52d0e4b76b11bbda92afefaab43732 + +# Run TAK Java server in container + +tldr: + + cp takserver.env.example takserver.env + # edit the env + # export the variables gomplate uses (see the .tpl files) + docker compose pull --include-deps --ignore-pull-failures + docker compose -p tak up -d + +or use docker compose.local.yml without gomplate for local dev +(rebuilding containers): + + export DOCKER_TAG_EXTRA="-dev" + docker build --no-cache --progress=plain -t takserver:latest${DOCKER_TAG_EXTRA} -t takserver:4.7-RELEASE-32${DOCKER_TAG_EXTRA} -t pvarki/takserver:4.7-RELEASE-32${DOCKER_TAG_EXTRA} . + cp takserver.env.example takserver.env + # edit the env + docker compose -f docker-compose.local.yml -p tak up + +Note, for things that live in the volumes (like TAK certs) you must nuke +the volumes to see changes: + + docker compose -f docker-compose.local.yml -p tak down -v ; docker compose -f docker-compose.local.yml -p tak rm -vf + +## Creating client packages locally + +Using the REST API is probably nicer though + +Create client package: + + docker compose -p tak exec takserver_api /bin/bash -c 'CLIENT_CERT_NAME=replaceme /opt/scripts/make_client_zip.sh' + +Then get /opt/tak/certs/files/clientpkgs/replaceme.zip out of the +container: + + docker compose -p tak exec taktakserver_apiserver /bin/bash -c 'base64 /opt/tak/certs/files/clientpkgs/replaceme.zip' | base64 -id >replaceme.zip + +This approach also works for recovering the admin cert +(/opt/tak/certs/files/admin.p12 unless you changed the ADMIN_CERT_NAME +ENV) + +## Creating new admin users + +Create the user on the takserver container: + + docker compose -p tak exec takserver_api /bin/bash -c 'cd /opt/tak/data/certs/ && CAPASS=$CA_PASS PASS=replaceme_user_cert_pass ./makeCert.sh client replaceme_username && ADMIN_CERT_NAME=replaceme_username /opt/scripts/enable_admin.sh' + +See above about the hard way of getting the cert file, or use the REST +API. + +## Gradle builds + +Build the distribution: + + mkdir outputs + docker build --progress=plain -f Dockerfile_build --target files -t atakbuild:files . + docker run --rm -it -v `pwd`/outputs:/output atakbuild:files + +Now you have the build artefacts in outputs -directory. + diff --git a/docs/dev/integrationrepo/v/1.14.0/submodules/ui.md b/docs/dev/integrationrepo/v/1.14.0/submodules/ui.md new file mode 100644 index 00000000..a5958dff --- /dev/null +++ b/docs/dev/integrationrepo/v/1.14.0/submodules/ui.md @@ -0,0 +1,116 @@ +--- +title: "pvarki/rasenmaeher-ui – README" +--- + +> **Integration tag:** `1.14.0` · **Submodule commit:** `eb2d303774fa7791ba5410080cd55316fe8b32c1` +> **Repo:** git@github.com:pvarki/rasenmaeher-ui.git +> **Browse at this commit:** https://github.com/pvarki/rasenmaeher-ui/tree/eb2d303774fa7791ba5410080cd55316fe8b32c1 + +# UI For RASENMAEHER + +React, TS etc + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rasenmaeher_ui:devel_shell . + docker create --name rmui_devel -v `pwd`":/app" -p 8002:8002 -it `echo $DOCKER_SSHAGENT` rasenmaeher_ui:devel_shell + docker start -i rmui_devel + +Note: due to the volume mapping if you already had run "npm install" on +the host you need to delete the node_modules directory and in any case +you need to run "npm install" inside the container. + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmui_devel /bin/bash -c "pre-commit install" + docker exec -i rmui_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rasenmaeher_ui:devel_shell -c "pre-commit run --all-files" + +### Production docker + +This will just deliver the compiled files to a mounted directory + +> docker build --ssh default --target production -t +> rasenmaeher_ui:amd64-latest . docker run -it --rm -v +> pwd/rmui_files:/deliver +> rasenmaeher_ui:amd64-latest + +Then you need to serve things so that index.html is the default +controller for things, nginx example: + + location / { + index index.html; + root /rmui_files; + try_files $uri $uri/ /index.html =404; + } + +## Development + +TLDR: + +- Create and activate a Node 18 virtualenv: + + FIXME: Insert instructions + +- change to a branch: + + git checkout -b my_branch + +- Install project deps and pre-commit hooks: + + npm install + pre-commit install + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo. + +### Asset set + +Asset set is loaded according to VITE_ASSET_SET enviroment variable. +Changing asset set requires reloading vite. Folder assetSetStore +contains files for for different sets that are copied to src/assets/set/ +folder on startup. Current set is tracked with setName.txt file. + +When you want to use dynamic assets, you want to make sure your dynamic +asset has equivalent for sets (both "fdf" and "neutral", so on). If the +assets are images, you want to then import them to your components from +/assets/set/\* where the env-defined assets are placed by VITE on +startup. If the assets are translation keys, make sure keys are present +in locales under assetSetStore, and then use the "dynamic" namespace for +accessing them. + diff --git a/docs/dev/integrationrepo/v/1.15.0/_category_.json b/docs/dev/integrationrepo/v/1.15.0/_category_.json new file mode 100644 index 00000000..3adfb3a3 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "1.15.0", + "position": 3 +} diff --git a/docs/dev/integrationrepo/v/1.15.0/index.md b/docs/dev/integrationrepo/v/1.15.0/index.md new file mode 100644 index 00000000..2e1abdad --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/index.md @@ -0,0 +1,4 @@ +--- +title: "1.15.0" +--- + diff --git a/docs/dev/integrationrepo/v/1.15.0/integration.md b/docs/dev/integrationrepo/v/1.15.0/integration.md new file mode 100644 index 00000000..d0904018 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/integration.md @@ -0,0 +1,319 @@ +--- +title: "pvarki/docker-rasenmaeher-integration – README" +--- + +> **Integration tag:** `1.15.0` +> **Repo:** https://github.com/pvarki/docker-rasenmaeher-integration + +![Build Status](https://github.com/pvarki/docker-rasenmaeher-integration/actions/workflows/build.yml/badge.svg) + +# Deploy App + +"One Ring to rule them all, One Ring to find them, One Ring to bring +them all and in the darkness bind them." + +Docker compositions, helpers etc to bring it all together into something +resembling grand old ones. + +Codename RASENMAEHER because infantry jokes. + +## WTF is this anyway ? + +This [Disobey24 +talk](https://www.youtube.com/watch?v=m3xd7uygpaY&list=PLLvAhAn5sGfiB9AlEt2KD7H9Dnr6kbd64&index=23) +explains a lot. + +## Running Deploy App in your own docker environment + +### Needed DNS Records + +These need to point to your WAN address. + +> - domain +> - kc.domain +> - tak.domain +> - bl.domain +> - mtx.domain +> - mtls.domain +> - mtls.kc.domain +> - mtls.tak.domain +> - mtls.kc.domain +> - mtls.bl.domain +> - mtls.mtx.domain + +When more products are added to the deployment they will follow the same +naming pattern, you will need subdomains for all products listed in the +composition for miniwerk service variable MW_PRODUCTS and "kc" for +Keycloak. + +### Needed ports open + +And redirected to the server if behind NAT or similar. + +> - 80 +> - 443 +> - 8443 (TAK) +> - 8446 (TAK) +> - 9446 (Keycloak) +> - 4626 (Product integration APIs port) +> - 4666 (Battlelog API/UI port) +> - 1936 RTMPS +> - 8322 RTSPS +> - 8890 SRT +> - 9888 HSL +> - 9889 WebRTC +> - 9996 Playback for recorded streams/videos + +### Downloading and composing Deploy App + +Be mindfull on where you download the repository, you will need to +perform rest of the commands inside the downloaded repository. + +Getting the repository from github (on Windows **first** see "Windows +notes" below): + + git clone --recurse-submodules -j8 git@github.com:pvarki/docker-rasenmaeher-integration.git + +Create `.env` file that defines environmental variables for Deploy App +setup. File must be located inside downloaded repository and file type +must be named literally `.env` (not `something.env`) to work. + +The original example file is: +[https://github.com/pvarki/docker-rasenmaeher-integration/blob/main/example_env.sh](https://github.com/pvarki/docker-rasenmaeher-integration/blob/main/example_env.sh) + +Example .env-file with the minimal information needed: + + KEYCLOAK_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RM_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + POSTGRES_PASSWORD="input-secure-password" # pragma: allowlist secret + LDAP_ADMIN_PASSWORD="input-secure-password" # pragma: allowlist secret + KEYCLOAK_ADMIN_PASSWORD="input-secure-password" # pragma: allowlist secret + TAK_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + SERVER_DOMAIN="input-domain" + CFSSL_CA_NAME="input-ca-name" + MW_LE_EMAIL="input-email-for-lets-encrypt" + MW_LE_TEST="false" + TAKSERVER_CERT_PASS="input-secure-password" # pragma: allowlist secret + TAK_CA_PASS="input-secure-password" # pragma: allowlist secret + VITE_ASSET_SET="${VITE_ASSET_SET:-neutral}" + KEYCLOAK_PROFILEROOT_UUID="input-uuid" + KEYCLOAK_HTTPS_KEY_STORE_PASSWORD="input-secure-password" # pragma: allowlist secret + KEYCLOAK_HTTPS_TRUST_STORE_PASSWORD="input-secure-password" # pragma: allowlist secret + BL_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RMMTX_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RMMTX_API_PASSWORD="input-secure-password" # pragma: allowlist secret + +Replace "intput-secure-password" with a good passphrase that is unique +for each replacment. You can generate an UUID with: + + python -c 'import uuid; print(uuid.uuid4())' + +If you wish to use one deployment for longer than the *design lifetime* +of 1-2 months you can change the following env variables. But do +understand that this is **not recommended** and has **security +implications**. If you do this **you** take **responsibility** to go +through all Dockerfiles and compositions to understand **exactly** how +things are done and how apply security updates into the containers: + + CFSSL_CA_EXPIRY="8800h" # Input time in hours = xxh, 8800h is 366.. days + CFSSL_SIGN_DEFAULT_EXPIRY="8800h" # Input time in hours = xxh, 8800h is 366.. days + +Starting the services: + + docker compose up -d + +Updating the repository from github: + + git pull + git submodule update --init --recursive + docker compose pull + docker compose up -d + +!DO NOT DO! Deleting the services. Deletes the certificates etc you will +need to add all users etc again: + + docker compose down -v + +Getting the admin login invite code for first admin: + + docker compose exec -it rmapi /bin/bash -c "rasenmaeher_api addcode" + +If you get "no such service -d", make sure whatever you copy-pasted the +command from did not render the dash (ASCII 0x2D) as some other +codepoint with a glyph that looks deceptively similar. When in doubt +write the commands yourself instead of copy-pasting. + +### Services + +Deploy App login page: + + https://domain (example.com) + +Deploy App home page: + + https://mtls.domain (mtls.example.com) + +Takserver Admin UI: + + https://tak.domain:8443/ (tak.example.com:8443/) + +Keycloack Admin UI. (Later group management will be withing Deploy App): + + https://kc.domain:9443/admin/RASENMAEHER/console/ (kc.example.com:9443/admin/RASENMAEHER/console/) + +OTA update server inside takserver. Is located in the loaded repository, +location depends on where you downloaded it: + + /home/user/docker-rasenmaeher-integration/takserver/update + +### Using the Deploy App service + +1. Login with first admin code. Create your admin account by typing + your first admin invite code and inputting desired admin callsign. +2. Create invite code for other users. Share the invite code. Go to + Manage Users -\> Add Users -\> Create New Invite. Share link, qr + code or invite code and domain. +3. Approve users in Deploy App. Open approvement link or scan qr code + from users and approve the user. You can also go to Approve Users + -\> Select Waiting User and input the users approvement code. +4. If desired promote some of the added users as admins. Go to Manage + Users -\> Manage Users -\> Select user and select Promote. You can + also Demote Admins or Delete users altogether. + +### Using Deploy App TAK in EUD + +EUD=End User Device + +1. Login to Deploy App. Go to [https://mtls.domain](https://mtls.domain) and select TAK. +2. Download Client Package. Select tak package for desired software + "Android ATAK or Windows WinTAK" or "iOS iTAK". Select Download + Client Package. +3. Go to EUD's TAK Software. Import downloaded package. Device is + connected to server. +4. You should also read Quickstart and Usage Guides. + +## Git submodules + +When cloning for the first time use: + + git clone --recurse-submodules -j8 git@github.com:pvarki/docker-rasenmaeher-integration.git + +When updating or checking out branches use: + + git submodule update --init --recursive + +And if you forgot to --recurse-submodules run the update command above. + +The submodules are repos in their own right, if you plan to make changes +into them change to the directory and create new branch, commit and push +changes as usual under that directory. + +### Directories that are submodules + +> - api [https://github.com/pvarki/python-rasenmaeher-api](https://github.com/pvarki/python-rasenmaeher-api) +> - cfssl [https://github.com/pvarki/docker-rasenmaeher-cfssl](https://github.com/pvarki/docker-rasenmaeher-cfssl) +> - fpintegration [https://github.com/pvarki/python-rasenmaeher-rmfpapi](https://github.com/pvarki/python-rasenmaeher-rmfpapi) +> - keycloak [https://github.com/pvarki/docker-keycloak](https://github.com/pvarki/docker-keycloak) +> - kw_product_init +> [https://github.com/pvarki/golang-kraftwerk-init-helper-cli](https://github.com/pvarki/golang-kraftwerk-init-helper-cli) +> - openldap [https://github.com/pvarki/docker-openldap](https://github.com/pvarki/docker-openldap) +> - miniwerk [https://github.com/pvarki/python-rasenmaeher-miniwerk](https://github.com/pvarki/python-rasenmaeher-miniwerk) +> - ui [https://github.com/pvarki/rasenmaeher-ui](https://github.com/pvarki/rasenmaeher-ui) +> - takserver [https://github.com/pvarki/docker-atak-server](https://github.com/pvarki/docker-atak-server) +> - takintegration [https://github.com/pvarki/python-tak-rmapi](https://github.com/pvarki/python-tak-rmapi) +> - battlelog [https://github.com/pvarki/typescript-liveloki-app](https://github.com/pvarki/typescript-liveloki-app) +> - mtxauthz [https://github.com/pvarki/python-mediamtx-rmmtxauthz](https://github.com/pvarki/python-mediamtx-rmmtxauthz) + +## Autogenerated (mostly API) docs + +> - Module API docs: +> [https://pvarki.github.io/docker-rasenmaeher-integration/docs/](https://pvarki.github.io/docker-rasenmaeher-integration/docs/) +> - Swagger definition for Deploy App API: +> [https://pvarki.github.io/docker-rasenmaeher-integration/](https://pvarki.github.io/docker-rasenmaeher-integration/) + +## Running in local development mode + +### Windows notes + +> 1. Do **NOT** use git-bash, it will cause *weirdest* issues with +> Docker containers +> +> 2. Use WSL, see +> [best_practises](https://github.com/pvarki/markdown-pvarki-best_practises) +> -repo for instructions on how to set it up. +> +> 3. Make sure whatever git client or IDE you use it does not mess with +> line-endings, for CLI client this does the trick: +> +> git config --global core.eol lf +> git config --global core.autocrlf false + +### Compositions + +Generally start with "rmlocal", it corresponds best to a real running +environment. "rmdev" starts a bunch of things in development mode which +does make developing more convenient but also introduces extra +variability to how things work. + +Make sure to always check your changes work correctly in rmlocal mode +where assets are minified and baked in. + +TLDR: + + alias rmlocal="docker compose -p rmlocal -f docker-compose-local.yml" + rmlocal build takinit miniwerk --pull + rmlocal build --pull + rmlocal up + +or: + + alias rmdev="docker compose -p rmdev -f docker-compose-local.yml -f docker-compose-dev.yml" + rmdev build takinit miniwerk --pull + rmdev build --pull + rmdev up + +OpenLDAP and keycloak-init sometimes fail on first start, just run up +again. + +IMPORTANT: Only keep either rmlocal or rmdev created at one time or you +may have weird network issues run "down" for one env before starting the +other. + +Remember to run "down -v" if you want to reset the persistent volumes, +or if you have weird issues when switching between environments. + +The dev version launches all the services and runs rasenmaeher-api in +uvicorn reload mode so any edits you make under /api will soon be +reflected in the running instance. + +If rasenmaeher-ui devel server complains make sure to delete +`ui/node_modules` -directory from host first. The docker NodeJS +distribution probably is not compatible with whatever you have installed +on the host. + +### Gaining first admin access in dev and production mode + +In dev mode: + + docker exec -it rmdev-rmapi-1 /bin/bash -c "source /.venv/bin/activate && rasenmaeher_api addcode" + +Under dev mode, the UI runs at [https://localmaeher.dev.pvarki.fi:4439](https://localmaeher.dev.pvarki.fi:4439). + +In VM production mode: + + docker exec -it rmvm-rmapi-1 /bin/bash -c "rasenmaeher_api addcode" + +## pre-commit notes + +Use "pre-commit run --all-files" liberally (and make sure you have run +"pre-commit install --install-hooks"). If you get complaints about +missing environment variables run "source example_env.sh" + +## Integration tests + +Pytest is used to handle the integration tests, the requirements are in +tests/requirements.txt. NOTE: The tests have side-effects and expect a +clean database to start with so always make sure to run "down -v" for +the composition first, then bring it back up before running integration +tests. + diff --git a/docs/dev/integrationrepo/v/1.15.0/submodules/api.md b/docs/dev/integrationrepo/v/1.15.0/submodules/api.md new file mode 100644 index 00000000..847a44ab --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/submodules/api.md @@ -0,0 +1,240 @@ +--- +title: "pvarki/python-rasenmaeher-api – README" +--- + +> **Integration tag:** `1.15.0` · **Submodule commit:** `0b51a68dda421693c8081505175b1250403bc595` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-api.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-api/tree/0b51a68dda421693c8081505175b1250403bc595 + +# python-rasenmaeher-api + +python-rasenmaeher-api + +## Configuration + +This application can be configured with environment variables. Below is +a example list of available variables. You can find all available +variables here +[https://github.com/pvarki/python-rasenmaeher-api/blob/main/src/rasenmaeher_api/settings.py](https://github.com/pvarki/python-rasenmaeher-api/blob/main/src/rasenmaeher_api/settings.py). + +| ENV VAR | Default value | Info / Usage | +|---------------------------|---------------------------------------------|---------------------------------------------------------------| +| RM_PORT | 8000 | Docker API listen port | +| RM_WORKERS_COUNT | 2 | Number of uvicorn workers | +| RM_RELOAD | False | Reload service on code change | +| RM_ENVIRONMENT | dev | Run dev / prod environment | +| RM_CFSSL_HOST | None | CFSSL host url | +| RM_CFSSL_PORT | None | CFSSL service port | +| RM_KEYCLOAK_SERVER_URL | None | Keycloak server url ([http://1234:8080/auth](http://1234:8080/auth)) | +| RM_KEYCLOAK_CLIENT_ID | None | Keycloak client id | +| RM_KEYCLOAK_REALM_NAME | None | Keycloak realm name | +| RM_KEYCLOAK_CLIENT_SECRET | None | Keycloak secert | +| RM_LDAP_CONN_STRING | None | LDAP conn string | +| RM_LDAP_USERNAME | None | LDAP username | +| RM_LDAP_CLIENT_SECRET | None | LDAP connection secret | +| RM_SQLITE_FILEPATH_PROD | /opt/rasenmaher/persistent/sqlite/rm_db.sql | location for sqlite database file in "prod" | +| RM_SQLITE_FILEPATH_DEV | /tmp/rm_db.sql | location for sqlite database file in "dev", local development | + +Container Variables + +You can create .env file in the root +directory and place all environment variables here. + +All environment variables should start with +RM\_ prefix. + +For example if you see in your "rasenmaeher_api/settings.py" a variable +named like random_parameter, you should +provide the "RM_RANDOM_PARAMETER" variable to configure the value. This +behaviour can be changed by overriding +env_prefix property in +rasenmaeher_api.settings.Settings.Config. + +An example of .env file: +`` `bash RM_RELOAD="True" RM_PORT="8000" RM_ENVIRONMENT="dev" ``\` + +You can read more about BaseSettings class here: +[https://pydantic-docs.helpmanual.io/usage/settings/](https://pydantic-docs.helpmanual.io/usage/settings/) + +## Backend services manifest + +Integration APIs are always https in port 4625. Default location for the +manifest is /pvarki/kraftwerk-rasenmaeher-init.json + +`` `json { "dns": "sleepy-sloth.pvarki.fi", "products": { "tak": "tak.sleepy-sloth.pvarki.fi", "ptt": "ptt.sleepy-sloth.pvarki.fi" } } ``\` + +## Api endpoints and usage + +| State | Method | URI | Request JSON | Response JSON | Api description . | +|---------|--------|------------------------------------|--------------------------------------------------------------|-------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| Dummy | GET | /api/v1/enroll/status/{work_id} | NA | {'status':'None/Processing/Denied/WaitingForAcceptance/ReadyForDelivery/Delivered'} | Check the situation of enrollment process, None = no enrollment started, this work_id is free to use. | +| Dummy | POST | /api/v1/enroll/init | {'work_id':'{work_id}'} | {'work_id':'{work_id}', 'id_hash':{id_string} } | Start service access enrollment for given {work_id} | +| Dummy | GET | /api/v1/enroll/deliver/{id_string} | NA | {'dl_link':"{[http://here.be/zip](http://here.be/zip)}"} | Deliver download link for enrollment zip | +| Dummy | POST | /api/v1/enroll/accept | { 'permit_string':'{permit_string}, 'id_hash':'{id_hash}' '} | { 'access':'granted/denied/None', 'work_id':'{work_id}' } | Accept the enrollment request | +| Testing | POST | /api/v1/product/sign_csr | { 'TO':'DO'} | { 'TO':'DO'} | Accept the enrollment request | +| Testing | POST | /api/v1/enroll/accept | { 'permit_string':'{permit_string}, 'id_hash':'{id_hash}' '} | { 'access':'granted/denied/None', 'work_id':'{work_id}' } | Accept the enrollment request | + +API vimpain + +## Example usage + +\# REQUEST A NEW CERTIFICATE USING CSR (requires cfssl backend for the +api container) + +> `` `curl -L -H "Content-Type: application/json" -d '{"csr": "-----BEGIN CERTIFICATE REQUEST-----\nMIIBUjCB+QIBADBqMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLZXhhbXBsZS5jb20x\nFjAUBgNVBAcTDVNhbiBGcmFuY2lzY28xEzARBgNVBAgTCkNhbGlmb3JuaWExGDAW\nBgNVBAMTD3d3dy5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA\nBK/CtZaQ4VliKE+DLIVGLwtSxJgtUKRzGvN1EwI3HRgKDQ3l3urBIzHtUcdMq6HZ\nb8jX0O9fXYUOf4XWggrLk1agLTArBgkqhkiG9w0BCQ4xHjAcMBoGA1UdEQQTMBGC\nD3d3dy5leGFtcGxlLmNvbTAKBggqhkjOPQQDAgNIADBFAiAcvfhXnsLtzep2sKSa\n36W7G9PRbHh8zVGlw3Hph8jR1QIhAKfrgplKwXcUctU5grjQ8KXkJV8RxQUo5KKs\ngFnXYtkb\n-----END CERTIFICATE REQUEST-----\n"}' 127.0.0.1:8000/api/v1/takreg | jq ``\` + +\# REQUEST A NEW CERTIFICATE WITHOUT CSR (requires cfssl backend for the +api container) + +> `` `curl -L -H "Content-Type: application/json" -d '{ "request": {"hosts":["harjoitus1.pvarki.fi"], "names":[{"C":"FI", "ST":"Jyvaskyla", "L":"KeskiSuomi", "O":"harjoitus1.pvarki.fi"}], "CN": "harjoitus1.pvarki.fi"}, "bundle":true, "profile":"client"}' 127.0.0.1:8000/takreg | jq ``\` + +\# LIST CFSSL CRL LIST + +> `` `curl -L -H "Content-Type: application/json" -d '{ "request": {"hosts":["harjoitus1.pvarki.fi"], "names":[{"C":"FI", "ST":"Jyvaskyla", "L":"KeskiSuomi", "O":"harjoitus1.pvarki.fi"}], "CN": "harjoitus1.pvarki.fi"}, "bundle":true, "profile":"client"}' 127.0.0.1:8000/takreg | jq ``\` + +The cfssl used behind API listens this kind of stuff +[https://github.com/cloudflare/cfssl/blob/master/doc/api/endpoint_newcert.txt](https://github.com/cloudflare/cfssl/blob/master/doc/api/endpoint_newcert.txt) + +\# Enrollment - Enroll a new work_id + +> `` `curl -H "Content-Type: application/json" -d '{"work_id":"porakoira666"}' http://127.0.0.1:8000/api/v1/enrollment/init ``\` + +\# Enrollment - Check the status and availability of work_id + +> `` `curl http://127.0.0.1:8000/api/v1/enrollment/status/koira ``\` + +\# Enrollment - Request the download link using the provided work_id_hash +`` `curl http://127.0.0.1:8000/api/v1/enrollment/deliver/zxzxzxzxzxzxzxxzzx ``\` + +\# Enrollment - Accept enrollment using permit_str +`` `curl -H "Content-Type: application/json" -d '{"enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/accept ``\` + +\# Enrollment - Set download link for enrollment +`` `curl -H "Content-Type: application/json" -d '{"download_link":"https://kuvaton.com","enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/set-dl-link ``\` + +\# Enrollment - Set state for enrollment +`` `curl -H "Content-Type: application/json" -d '{"state":"ReadyForDelivery","enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/set-state ``\` + +\# Enrollment - Add new permit_str +`` `curl -H "Content-Type: application/json" -d '{"permissions_str":"all", "new_permit_hash":"too_short","permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/add-manager ``\` + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rasenmaeher_api:devel_shell . + docker create --name rasenmaeher_api_devel -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` rasenmaeher_api:devel_shell + docker start -i rasenmaeher_api_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rasenmaeher_api_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rasenmaeher_api_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rasenmaeher_api:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rasenmaeher_api:tox . + docker run --rm -it --network host -v /var/run/docker.sock:/var/run/docker.sock -v `pwd`":/app" `echo $DOCKER_SSHAGENT` rasenmaeher_api:tox + +NOTE: This will not work from the git submodule directory in the +integration repo, docker tests must be run from the original repo. + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rasenmaeher_api:amd64-latest . + docker run --rm -it --name rasenmaeher_openapijson rasenmaeher_api:amd64-latest rasenmaeher_api openapi + docker run -it --name rasenmaeher_api rasenmaeher_api:amd64-latest + +There is also a specific target for just dumping the openapi.json: + + docker build --ssh default --target openapi -t rasenmaeher_api:amd64-openapi . + docker run --rm -it --name rasenmaeher_openapijson rasenmaeher_api:amd64-openapi + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.15.0/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.15.0/submodules/battlelog.md new file mode 100644 index 00000000..e63d9426 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/submodules/battlelog.md @@ -0,0 +1,53 @@ +--- +title: "pvarki/typescript-liveloki-app – README" +--- + +> **Integration tag:** `1.15.0` · **Submodule commit:** `197f6b74507fb81fc442fa1522088f0cb7ed3939` +> **Repo:** git@github.com:pvarki/typescript-liveloki-app.git +> **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/197f6b74507fb81fc442fa1522088f0cb7ed3939 + +# BattleLog + +## Install + +1. Install docker + compose +2. Rename .env_example --> .env and modify as you like. +3. Run docker compose build --no-cache +4. Run docker compose up -d +5. Navigate to localhost:3000 + +## Frontend development + +The frontend is bundled with [Vite](https://vitejs.dev/). + +Once the backend is running (on port 3000), you can navigate to `frontend/`, +run `npm i` and `npm run dev` to run the Vite development server that has +hot reload and all that jazz. + +## Info + +Database is currently preseeded with test data from preseed/preseed.csv + +## Migrations + +To add new migration, locally run: `./node_modules/.bin/node-pg-migrate create ` and modify created file in `/migrations` directory. +Migrations are run (if needed) when docker container starts. `wait-for-it.sh` will ensure that psql container is up and accepting connections before running migrations. + +## JWT testing + +Remove comments from "jwt-test-network" for local jwt testing, to allow joining containers from 2 different compose runs to same docker network. + +## OpenAPI Specification + +https://pvarki.github.io/typescript-liveloki-app/ + +## Running tests + +In directory `tests`, use + +```shell +npm run test:integration +``` + +with the server running in `localhost:3000` as it does by default after `docker compose up -d`. + diff --git a/docs/dev/integrationrepo/v/1.15.0/submodules/cfssl.md b/docs/dev/integrationrepo/v/1.15.0/submodules/cfssl.md new file mode 100644 index 00000000..3a041b2e --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/submodules/cfssl.md @@ -0,0 +1,91 @@ +--- +title: "pvarki/docker-rasenmaeher-cfssl – README" +--- + +> **Integration tag:** `1.15.0` · **Submodule commit:** `87f2c2b4b22a0d25d7dcc900d0750373b24fec4b` +> **Repo:** git@github.com:pvarki/docker-rasenmaeher-cfssl.git +> **Browse at this commit:** https://github.com/pvarki/docker-rasenmaeher-cfssl/tree/87f2c2b4b22a0d25d7dcc900d0750373b24fec4b + +# cfssl Submodule + +\![Build +Status\]([https://github.com/pvarki/docker-rasenmaeher-cfssl/actions/workflows/build.yml/badge.svg](https://github.com/pvarki/docker-rasenmaeher-cfssl/actions/workflows/build.yml/badge.svg)) + +## Used as git submodule + +This repo is used as submodule in +[https://github.com/pvarki/docker-rasenmaeher-integration](https://github.com/pvarki/docker-rasenmaeher-integration) it is +probably a good idea to handle all development via it because it has +docker composition for bringin up all the other services rasenmaeher-api +depends on + +## Development + +The cfssl itself we can't do much about but the FastAPI thing uses +poetry and in any case use pre-commit: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t ocsprest:devel_shell . + docker create --name ocsprest_devel -v `pwd`/../":/app" -it `echo $DOCKER_SSHAGENT` ocsprest:devel_shell + docker start -i ocsprest_devel + +Or just pwd if working under separate checkout instead of the +integration repo. + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i ocsprest_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i ocsprest_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" ocsprest:devel_shell -c "pre-commit run --all-files" + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target ocsprest -t ocsprest:amd64-latest . + docker run -it --name ocsprest ocsprest:amd64-latest + +There is also a specific target for just dumping the openapi.json: + + docker build --ssh default --target openapi -t ocsprest:amd64-openapi . + docker run --rm -it --name rasenmaeher_openapijson ocsprest:amd64-openapi + diff --git a/docs/dev/integrationrepo/v/1.15.0/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.15.0/submodules/fpintegration.md new file mode 100644 index 00000000..06ed1d01 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/submodules/fpintegration.md @@ -0,0 +1,106 @@ +--- +title: "pvarki/python-rasenmaeher-rmfpapi – README" +--- + +> **Integration tag:** `1.15.0` · **Submodule commit:** `7b24c8155518a4e11dd46b1f8d210b850973004c` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-rmfpapi.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-rmfpapi/tree/7b24c8155518a4e11dd46b1f8d210b850973004c + +# rmfpapi + +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rmfpapi:devel_shell . + docker create --name rmfpapi_devel -p 4625:4625 -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` rmfpapi:devel_shell + docker start -i rmfpapi_devel + +In the shell you can start the uvicorn devel server with (binding to +0.0.0.0 is important!): + + uvicorn "rmfpapi.app:get_app" --factory --host 0.0.0.0 --port 4625 --reload --log-level debug + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmfpapi_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rmfpapi_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rmfpapi:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rmfpapi:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` rmfpapi:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rmfpapi:latest . + docker run -it --name rmfpapi -p 4625:4625 rmfpapi:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.15.0/submodules/keycloak2.md b/docs/dev/integrationrepo/v/1.15.0/submodules/keycloak2.md new file mode 100644 index 00000000..0077893f --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/submodules/keycloak2.md @@ -0,0 +1,31 @@ +--- +title: "pvarki/docker-keycloak – README" +--- + +> **Integration tag:** `1.15.0` · **Submodule commit:** `8b2efdf6a4d90fd91d2074617983c086ebe537c5` +> **Repo:** git@github.com:pvarki/docker-keycloak.git +> **Browse at this commit:** https://github.com/pvarki/docker-keycloak/tree/8b2efdf6a4d90fd91d2074617983c086ebe537c5 + +# KeyCloak + +Templates, init container etc to setup keycloak automagically with ENV +variables. + +## Used as git submodule + +This repo is used as submodule in +[https://github.com/pvarki/docker-rasenmaeher-integration](https://github.com/pvarki/docker-rasenmaeher-integration) it is +probably a good idea to handle all development via it because it has +docker composition for bringin up all the other services rasenmaeher-api +depends on + +## pre-commit notes + +Make sure pre-commit is installed: + + pre-commit install --install-hooks + +And it's a good idea to run it regularly before committing: + + pre-commit run --all-files + diff --git a/docs/dev/integrationrepo/v/1.15.0/submodules/kw_product_init.md b/docs/dev/integrationrepo/v/1.15.0/submodules/kw_product_init.md new file mode 100644 index 00000000..64ec607d --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/submodules/kw_product_init.md @@ -0,0 +1,78 @@ +--- +title: "pvarki/golang-kraftwerk-init-helper-cli – README" +--- + +> **Integration tag:** `1.15.0` · **Submodule commit:** `0dee1f8f397ba37280f6bf393a3b866d93387e8c` +> **Repo:** git@github.com:pvarki/golang-kraftwerk-init-helper-cli.git +> **Browse at this commit:** https://github.com/pvarki/golang-kraftwerk-init-helper-cli/tree/0dee1f8f397ba37280f6bf393a3b866d93387e8c + +golang-kraftwerk-init-helper-cli =============================== + +Tool for products to create their certificates from +KRAFTWERK manifests. + +# Development + +## Prerequisites + +**Enable** +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/) + +```bash +export DOCKER_BUILDKIT=1 +``` + +**Forward SSH-agent to running instance:** + +**OSX:** + +```bash +export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" +``` + +**Linux:** + +```bash +export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" +``` + +## Create & start development container + +Build the image, create a container and start the container + +```bash +docker build --ssh default --target dev_shell -t kraftwerk_init_helper:dev_shell . +``` + +```bash +docker create --name kraftwerk_init_helper -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` kraftwerk_init_helper:dev_shell +``` + +```bash +docker start -i kraftwerk_init_helper +``` + +## pre-commit initialization + +Once inside the container, run: + +```bash +pre-commit install && pre-commit run --all-files +``` + +That's it, now you have the development environment up & running. + +# Production + +Build the production image: + +```bash +docker build --ssh default --target production -t kraftwerk_init_helper:latest . +``` + +Run the image: + +```bash +docker run --rm -it --name kraftwerk_helper kraftwerk_init_helper:latest +``` + diff --git a/docs/dev/integrationrepo/v/1.15.0/submodules/miniwerk.md b/docs/dev/integrationrepo/v/1.15.0/submodules/miniwerk.md new file mode 100644 index 00000000..4729609c --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/submodules/miniwerk.md @@ -0,0 +1,104 @@ +--- +title: "pvarki/python-rasenmaeher-miniwerk – README" +--- + +> **Integration tag:** `1.15.0` · **Submodule commit:** `073f866ad4b47dc7d1eab0cc06ecaf45c5fc5c4d` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-miniwerk.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-miniwerk/tree/073f866ad4b47dc7d1eab0cc06ecaf45c5fc5c4d + +# miniwerk + +Minimal KRAFTWERK amulation to be able to run a RASENMAEHER+products +deployment on any VM + +There are some required ENV configs check out example_env.sh for them +(.env file is also supported) + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t miniwerk:devel_shell . + docker create --name miniwerk_devel -v `pwd`":/app" -p 80:80 -it `echo $DOCKER_SSHAGENT` miniwerk:devel_shell + docker start -i miniwerk_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i miniwerk_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i miniwerk_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" miniwerk:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t miniwerk:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` miniwerk:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t miniwerk:amd64-latest . + docker run -it --name miniwerk miniwerk:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.15.0/submodules/mtxauthz.md b/docs/dev/integrationrepo/v/1.15.0/submodules/mtxauthz.md new file mode 100644 index 00000000..066c0640 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/submodules/mtxauthz.md @@ -0,0 +1,122 @@ +--- +title: "pvarki/python-mediamtx-rmmtxauthz – README" +--- + +> **Integration tag:** `1.15.0` · **Submodule commit:** `55921125444547f67089706f64432e5e3aef7763` +> **Repo:** git@github.com:pvarki/python-mediamtx-rmmtxauthz.git +> **Browse at this commit:** https://github.com/pvarki/python-mediamtx-rmmtxauthz/tree/55921125444547f67089706f64432e5e3aef7763 + +# rmmtxauthz + +Do HTTP based authz for MediaMTX + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rmmtxauthz:devel_shell . + docker create --name rmmtxauthz_devel -v "$(pwd)/rune/output/rune.json:/opt/templates/mediamtx.json" -v "$(pwd):/app" -it $(echo $DOCKER_SSHAGENT) rmmtxauthz:devel_shell + docker start -i rmmtxauthz_devel + +To rebuild the documentation inside the container run: + + rune rune/src json >/opt/templates/mediamtx.json + +Outside of container use: + + rune rune/src json >rune/output/rune.json + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmmtxauthz_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rmmtxauthz_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v "$(pwd):/app" rmmtxauthz:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rmmtxauthz:tox . + docker run --rm -it -v "$(pwd):/app" $(echo $DOCKER_SSHAGENT) rmmtxauthz:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rmmtxauthz:amd64-latest . + docker run -it --name rmmtxauthz rmmtxauthz:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p $(which python3.11) my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + git add poetry.lock + pre-commit install --install-hooks + pre-commit run --all-files + +If you get weird errors about missing packages from pre-commit try +running it with "poetry run pre-commit". + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + +Running "pre-commit run --all-files" and "py.test -v" regularly during +development and especially before committing will save you some +headache. + +### RUNE instructions compile + +tldr: + + rune rune/src json >rune/output/mediamtx.json + diff --git a/docs/dev/integrationrepo/v/1.15.0/submodules/takintegration.md b/docs/dev/integrationrepo/v/1.15.0/submodules/takintegration.md new file mode 100644 index 00000000..546c2aeb --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/submodules/takintegration.md @@ -0,0 +1,100 @@ +--- +title: "pvarki/python-tak-rmapi – README" +--- + +> **Integration tag:** `1.15.0` · **Submodule commit:** `0f3021ec2771c0d3846b46a302692f97fa7f2a19` +> **Repo:** git@github.com:pvarki/python-tak-rmapi.git +> **Browse at this commit:** https://github.com/pvarki/python-tak-rmapi/tree/0f3021ec2771c0d3846b46a302692f97fa7f2a19 + +# takrmapi + +RASENMAEHER integration API for TAK server + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t takrmapi:devel_shell . + docker create --name takrmapi_devel -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` takrmapi:devel_shell + docker start -i takrmapi_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i takrmapi_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i takrmapi_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" takrmapi:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t takrmapi:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` takrmapi:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t takrmapi:amd64-latest . + docker run -it --name takrmapi takrmapi:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.15.0/submodules/takserver.md b/docs/dev/integrationrepo/v/1.15.0/submodules/takserver.md new file mode 100644 index 00000000..8d3b11c5 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/submodules/takserver.md @@ -0,0 +1,68 @@ +--- +title: "pvarki/docker-atak-server – README" +--- + +> **Integration tag:** `1.15.0` · **Submodule commit:** `7a73302fef52d0e4b76b11bbda92afefaab43732` +> **Repo:** git@github.com:pvarki/docker-atak-server.git +> **Browse at this commit:** https://github.com/pvarki/docker-atak-server/tree/7a73302fef52d0e4b76b11bbda92afefaab43732 + +# Run TAK Java server in container + +tldr: + + cp takserver.env.example takserver.env + # edit the env + # export the variables gomplate uses (see the .tpl files) + docker compose pull --include-deps --ignore-pull-failures + docker compose -p tak up -d + +or use docker compose.local.yml without gomplate for local dev +(rebuilding containers): + + export DOCKER_TAG_EXTRA="-dev" + docker build --no-cache --progress=plain -t takserver:latest${DOCKER_TAG_EXTRA} -t takserver:4.7-RELEASE-32${DOCKER_TAG_EXTRA} -t pvarki/takserver:4.7-RELEASE-32${DOCKER_TAG_EXTRA} . + cp takserver.env.example takserver.env + # edit the env + docker compose -f docker-compose.local.yml -p tak up + +Note, for things that live in the volumes (like TAK certs) you must nuke +the volumes to see changes: + + docker compose -f docker-compose.local.yml -p tak down -v ; docker compose -f docker-compose.local.yml -p tak rm -vf + +## Creating client packages locally + +Using the REST API is probably nicer though + +Create client package: + + docker compose -p tak exec takserver_api /bin/bash -c 'CLIENT_CERT_NAME=replaceme /opt/scripts/make_client_zip.sh' + +Then get /opt/tak/certs/files/clientpkgs/replaceme.zip out of the +container: + + docker compose -p tak exec taktakserver_apiserver /bin/bash -c 'base64 /opt/tak/certs/files/clientpkgs/replaceme.zip' | base64 -id >replaceme.zip + +This approach also works for recovering the admin cert +(/opt/tak/certs/files/admin.p12 unless you changed the ADMIN_CERT_NAME +ENV) + +## Creating new admin users + +Create the user on the takserver container: + + docker compose -p tak exec takserver_api /bin/bash -c 'cd /opt/tak/data/certs/ && CAPASS=$CA_PASS PASS=replaceme_user_cert_pass ./makeCert.sh client replaceme_username && ADMIN_CERT_NAME=replaceme_username /opt/scripts/enable_admin.sh' + +See above about the hard way of getting the cert file, or use the REST +API. + +## Gradle builds + +Build the distribution: + + mkdir outputs + docker build --progress=plain -f Dockerfile_build --target files -t atakbuild:files . + docker run --rm -it -v `pwd`/outputs:/output atakbuild:files + +Now you have the build artefacts in outputs -directory. + diff --git a/docs/dev/integrationrepo/v/1.15.0/submodules/ui.md b/docs/dev/integrationrepo/v/1.15.0/submodules/ui.md new file mode 100644 index 00000000..b3f32a55 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.0/submodules/ui.md @@ -0,0 +1,116 @@ +--- +title: "pvarki/rasenmaeher-ui – README" +--- + +> **Integration tag:** `1.15.0` · **Submodule commit:** `eb2d303774fa7791ba5410080cd55316fe8b32c1` +> **Repo:** git@github.com:pvarki/rasenmaeher-ui.git +> **Browse at this commit:** https://github.com/pvarki/rasenmaeher-ui/tree/eb2d303774fa7791ba5410080cd55316fe8b32c1 + +# UI For RASENMAEHER + +React, TS etc + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rasenmaeher_ui:devel_shell . + docker create --name rmui_devel -v `pwd`":/app" -p 8002:8002 -it `echo $DOCKER_SSHAGENT` rasenmaeher_ui:devel_shell + docker start -i rmui_devel + +Note: due to the volume mapping if you already had run "npm install" on +the host you need to delete the node_modules directory and in any case +you need to run "npm install" inside the container. + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmui_devel /bin/bash -c "pre-commit install" + docker exec -i rmui_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rasenmaeher_ui:devel_shell -c "pre-commit run --all-files" + +### Production docker + +This will just deliver the compiled files to a mounted directory + +> docker build --ssh default --target production -t +> rasenmaeher_ui:amd64-latest . docker run -it --rm -v +> pwd/rmui_files:/deliver +> rasenmaeher_ui:amd64-latest + +Then you need to serve things so that index.html is the default +controller for things, nginx example: + + location / { + index index.html; + root /rmui_files; + try_files $uri $uri/ /index.html =404; + } + +## Development + +TLDR: + +- Create and activate a Node 18 virtualenv: + + FIXME: Insert instructions + +- change to a branch: + + git checkout -b my_branch + +- Install project deps and pre-commit hooks: + + npm install + pre-commit install + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo. + +### Asset set + +Asset set is loaded according to VITE_ASSET_SET enviroment variable. +Changing asset set requires reloading vite. Folder assetSetStore +contains files for for different sets that are copied to src/assets/set/ +folder on startup. Current set is tracked with setName.txt file. + +When you want to use dynamic assets, you want to make sure your dynamic +asset has equivalent for sets (both "fdf" and "neutral", so on). If the +assets are images, you want to then import them to your components from +/assets/set/\* where the env-defined assets are placed by VITE on +startup. If the assets are translation keys, make sure keys are present +in locales under assetSetStore, and then use the "dynamic" namespace for +accessing them. + diff --git a/docs/dev/integrationrepo/v/1.15.1/_category_.json b/docs/dev/integrationrepo/v/1.15.1/_category_.json new file mode 100644 index 00000000..554a389b --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "1.15.1", + "position": 2 +} diff --git a/docs/dev/integrationrepo/v/1.15.1/index.md b/docs/dev/integrationrepo/v/1.15.1/index.md new file mode 100644 index 00000000..968942ae --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/index.md @@ -0,0 +1,4 @@ +--- +title: "1.15.1" +--- + diff --git a/docs/dev/integrationrepo/v/1.15.1/integration.md b/docs/dev/integrationrepo/v/1.15.1/integration.md new file mode 100644 index 00000000..fac8d67e --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/integration.md @@ -0,0 +1,319 @@ +--- +title: "pvarki/docker-rasenmaeher-integration – README" +--- + +> **Integration tag:** `1.15.1` +> **Repo:** https://github.com/pvarki/docker-rasenmaeher-integration + +![Build Status](https://github.com/pvarki/docker-rasenmaeher-integration/actions/workflows/build.yml/badge.svg) + +# Deploy App + +"One Ring to rule them all, One Ring to find them, One Ring to bring +them all and in the darkness bind them." + +Docker compositions, helpers etc to bring it all together into something +resembling grand old ones. + +Codename RASENMAEHER because infantry jokes. + +## WTF is this anyway ? + +This [Disobey24 +talk](https://www.youtube.com/watch?v=m3xd7uygpaY&list=PLLvAhAn5sGfiB9AlEt2KD7H9Dnr6kbd64&index=23) +explains a lot. + +## Running Deploy App in your own docker environment + +### Needed DNS Records + +These need to point to your WAN address. + +> - domain +> - kc.domain +> - tak.domain +> - bl.domain +> - mtx.domain +> - mtls.domain +> - mtls.kc.domain +> - mtls.tak.domain +> - mtls.kc.domain +> - mtls.bl.domain +> - mtls.mtx.domain + +When more products are added to the deployment they will follow the same +naming pattern, you will need subdomains for all products listed in the +composition for miniwerk service variable MW_PRODUCTS and "kc" for +Keycloak. + +### Needed ports open + +And redirected to the server if behind NAT or similar. + +> - 80 +> - 443 +> - 8443 (TAK) +> - 8446 (TAK) +> - 9446 (Keycloak) +> - 4626 (Product integration APIs port) +> - 4666 (Battlelog API/UI port) +> - 1936 RTMPS +> - 8322 RTSPS +> - 8890 SRT +> - 9888 HSL +> - 9889 WebRTC +> - 9996 Playback for recorded streams/videos + +### Downloading and composing Deploy App + +Be mindfull on where you download the repository, you will need to +perform rest of the commands inside the downloaded repository. + +Getting the repository from github (on Windows **first** see "Windows +notes" below): + + git clone --recurse-submodules -j8 git@github.com:pvarki/docker-rasenmaeher-integration.git + +Create `.env` file that defines environmental variables for Deploy App +setup. File must be located inside downloaded repository and file type +must be named literally `.env` (not `something.env`) to work. + +The original example file is: +[https://github.com/pvarki/docker-rasenmaeher-integration/blob/main/example_env.sh](https://github.com/pvarki/docker-rasenmaeher-integration/blob/main/example_env.sh) + +Example .env-file with the minimal information needed: + + KEYCLOAK_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RM_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + POSTGRES_PASSWORD="input-secure-password" # pragma: allowlist secret + LDAP_ADMIN_PASSWORD="input-secure-password" # pragma: allowlist secret + KEYCLOAK_ADMIN_PASSWORD="input-secure-password" # pragma: allowlist secret + TAK_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + SERVER_DOMAIN="input-domain" + CFSSL_CA_NAME="input-ca-name" + MW_LE_EMAIL="input-email-for-lets-encrypt" + MW_LE_TEST="false" + TAKSERVER_CERT_PASS="input-secure-password" # pragma: allowlist secret + TAK_CA_PASS="input-secure-password" # pragma: allowlist secret + VITE_ASSET_SET="${VITE_ASSET_SET:-neutral}" + KEYCLOAK_PROFILEROOT_UUID="input-uuid" + KEYCLOAK_HTTPS_KEY_STORE_PASSWORD="input-secure-password" # pragma: allowlist secret + KEYCLOAK_HTTPS_TRUST_STORE_PASSWORD="input-secure-password" # pragma: allowlist secret + BL_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RMMTX_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RMMTX_API_PASSWORD="input-secure-password" # pragma: allowlist secret + +Replace "intput-secure-password" with a good passphrase that is unique +for each replacment. You can generate an UUID with: + + python -c 'import uuid; print(uuid.uuid4())' + +If you wish to use one deployment for longer than the *design lifetime* +of 1-2 months you can change the following env variables. But do +understand that this is **not recommended** and has **security +implications**. If you do this **you** take **responsibility** to go +through all Dockerfiles and compositions to understand **exactly** how +things are done and how apply security updates into the containers: + + CFSSL_CA_EXPIRY="8800h" # Input time in hours = xxh, 8800h is 366.. days + CFSSL_SIGN_DEFAULT_EXPIRY="8800h" # Input time in hours = xxh, 8800h is 366.. days + +Starting the services: + + docker compose up -d + +Updating the repository from github: + + git pull + git submodule update --init --recursive + docker compose pull + docker compose up -d + +!DO NOT DO! Deleting the services. Deletes the certificates etc you will +need to add all users etc again: + + docker compose down -v + +Getting the admin login invite code for first admin: + + docker compose exec -it rmapi /bin/bash -c "rasenmaeher_api addcode" + +If you get "no such service -d", make sure whatever you copy-pasted the +command from did not render the dash (ASCII 0x2D) as some other +codepoint with a glyph that looks deceptively similar. When in doubt +write the commands yourself instead of copy-pasting. + +### Services + +Deploy App login page: + + https://domain (example.com) + +Deploy App home page: + + https://mtls.domain (mtls.example.com) + +Takserver Admin UI: + + https://tak.domain:8443/ (tak.example.com:8443/) + +Keycloack Admin UI. (Later group management will be withing Deploy App): + + https://kc.domain:9443/admin/RASENMAEHER/console/ (kc.example.com:9443/admin/RASENMAEHER/console/) + +OTA update server inside takserver. Is located in the loaded repository, +location depends on where you downloaded it: + + /home/user/docker-rasenmaeher-integration/takserver/update + +### Using the Deploy App service + +1. Login with first admin code. Create your admin account by typing + your first admin invite code and inputting desired admin callsign. +2. Create invite code for other users. Share the invite code. Go to + Manage Users -\> Add Users -\> Create New Invite. Share link, qr + code or invite code and domain. +3. Approve users in Deploy App. Open approvement link or scan qr code + from users and approve the user. You can also go to Approve Users + -\> Select Waiting User and input the users approvement code. +4. If desired promote some of the added users as admins. Go to Manage + Users -\> Manage Users -\> Select user and select Promote. You can + also Demote Admins or Delete users altogether. + +### Using Deploy App TAK in EUD + +EUD=End User Device + +1. Login to Deploy App. Go to [https://mtls.domain](https://mtls.domain) and select TAK. +2. Download Client Package. Select tak package for desired software + "Android ATAK or Windows WinTAK" or "iOS iTAK". Select Download + Client Package. +3. Go to EUD's TAK Software. Import downloaded package. Device is + connected to server. +4. You should also read Quickstart and Usage Guides. + +## Git submodules + +When cloning for the first time use: + + git clone --recurse-submodules -j8 git@github.com:pvarki/docker-rasenmaeher-integration.git + +When updating or checking out branches use: + + git submodule update --init --recursive + +And if you forgot to --recurse-submodules run the update command above. + +The submodules are repos in their own right, if you plan to make changes +into them change to the directory and create new branch, commit and push +changes as usual under that directory. + +### Directories that are submodules + +> - api [https://github.com/pvarki/python-rasenmaeher-api](https://github.com/pvarki/python-rasenmaeher-api) +> - cfssl [https://github.com/pvarki/docker-rasenmaeher-cfssl](https://github.com/pvarki/docker-rasenmaeher-cfssl) +> - fpintegration [https://github.com/pvarki/python-rasenmaeher-rmfpapi](https://github.com/pvarki/python-rasenmaeher-rmfpapi) +> - keycloak [https://github.com/pvarki/docker-keycloak](https://github.com/pvarki/docker-keycloak) +> - kw_product_init +> [https://github.com/pvarki/golang-kraftwerk-init-helper-cli](https://github.com/pvarki/golang-kraftwerk-init-helper-cli) +> - openldap [https://github.com/pvarki/docker-openldap](https://github.com/pvarki/docker-openldap) +> - miniwerk [https://github.com/pvarki/python-rasenmaeher-miniwerk](https://github.com/pvarki/python-rasenmaeher-miniwerk) +> - ui [https://github.com/pvarki/rasenmaeher-ui](https://github.com/pvarki/rasenmaeher-ui) +> - takserver [https://github.com/pvarki/docker-atak-server](https://github.com/pvarki/docker-atak-server) +> - takintegration [https://github.com/pvarki/python-tak-rmapi](https://github.com/pvarki/python-tak-rmapi) +> - battlelog [https://github.com/pvarki/typescript-liveloki-app](https://github.com/pvarki/typescript-liveloki-app) +> - mtxauthz [https://github.com/pvarki/python-mediamtx-rmmtxauthz](https://github.com/pvarki/python-mediamtx-rmmtxauthz) + +## Autogenerated (mostly API) docs + +> - Module API docs: +> [https://pvarki.github.io/docker-rasenmaeher-integration/docs/](https://pvarki.github.io/docker-rasenmaeher-integration/docs/) +> - Swagger definition for Deploy App API: +> [https://pvarki.github.io/docker-rasenmaeher-integration/](https://pvarki.github.io/docker-rasenmaeher-integration/) + +## Running in local development mode + +### Windows notes + +> 1. Do **NOT** use git-bash, it will cause *weirdest* issues with +> Docker containers +> +> 2. Use WSL, see +> [best_practises](https://github.com/pvarki/markdown-pvarki-best_practises) +> -repo for instructions on how to set it up. +> +> 3. Make sure whatever git client or IDE you use it does not mess with +> line-endings, for CLI client this does the trick: +> +> git config --global core.eol lf +> git config --global core.autocrlf false + +### Compositions + +Generally start with "rmlocal", it corresponds best to a real running +environment. "rmdev" starts a bunch of things in development mode which +does make developing more convenient but also introduces extra +variability to how things work. + +Make sure to always check your changes work correctly in rmlocal mode +where assets are minified and baked in. + +TLDR: + + alias rmlocal="docker compose -p rmlocal -f docker-compose-local.yml" + rmlocal build takinit miniwerk --pull + rmlocal build --pull + rmlocal up + +or: + + alias rmdev="docker compose -p rmdev -f docker-compose-local.yml -f docker-compose-dev.yml" + rmdev build takinit miniwerk --pull + rmdev build --pull + rmdev up + +OpenLDAP and keycloak-init sometimes fail on first start, just run up +again. + +IMPORTANT: Only keep either rmlocal or rmdev created at one time or you +may have weird network issues run "down" for one env before starting the +other. + +Remember to run "down -v" if you want to reset the persistent volumes, +or if you have weird issues when switching between environments. + +The dev version launches all the services and runs rasenmaeher-api in +uvicorn reload mode so any edits you make under /api will soon be +reflected in the running instance. + +If rasenmaeher-ui devel server complains make sure to delete +`ui/node_modules` -directory from host first. The docker NodeJS +distribution probably is not compatible with whatever you have installed +on the host. + +### Gaining first admin access in dev and production mode + +In dev mode: + + docker exec -it rmdev-rmapi-1 /bin/bash -c "source /.venv/bin/activate && rasenmaeher_api addcode" + +Under dev mode, the UI runs at [https://localmaeher.dev.pvarki.fi:4439](https://localmaeher.dev.pvarki.fi:4439). + +In VM production mode: + + docker exec -it rmvm-rmapi-1 /bin/bash -c "rasenmaeher_api addcode" + +## pre-commit notes + +Use "pre-commit run --all-files" liberally (and make sure you have run +"pre-commit install --install-hooks"). If you get complaints about +missing environment variables run "source example_env.sh" + +## Integration tests + +Pytest is used to handle the integration tests, the requirements are in +tests/requirements.txt. NOTE: The tests have side-effects and expect a +clean database to start with so always make sure to run "down -v" for +the composition first, then bring it back up before running integration +tests. + diff --git a/docs/dev/integrationrepo/v/1.15.1/submodules/api.md b/docs/dev/integrationrepo/v/1.15.1/submodules/api.md new file mode 100644 index 00000000..bdd28afe --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/submodules/api.md @@ -0,0 +1,240 @@ +--- +title: "pvarki/python-rasenmaeher-api – README" +--- + +> **Integration tag:** `1.15.1` · **Submodule commit:** `0b51a68dda421693c8081505175b1250403bc595` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-api.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-api/tree/0b51a68dda421693c8081505175b1250403bc595 + +# python-rasenmaeher-api + +python-rasenmaeher-api + +## Configuration + +This application can be configured with environment variables. Below is +a example list of available variables. You can find all available +variables here +[https://github.com/pvarki/python-rasenmaeher-api/blob/main/src/rasenmaeher_api/settings.py](https://github.com/pvarki/python-rasenmaeher-api/blob/main/src/rasenmaeher_api/settings.py). + +| ENV VAR | Default value | Info / Usage | +|---------------------------|---------------------------------------------|---------------------------------------------------------------| +| RM_PORT | 8000 | Docker API listen port | +| RM_WORKERS_COUNT | 2 | Number of uvicorn workers | +| RM_RELOAD | False | Reload service on code change | +| RM_ENVIRONMENT | dev | Run dev / prod environment | +| RM_CFSSL_HOST | None | CFSSL host url | +| RM_CFSSL_PORT | None | CFSSL service port | +| RM_KEYCLOAK_SERVER_URL | None | Keycloak server url ([http://1234:8080/auth](http://1234:8080/auth)) | +| RM_KEYCLOAK_CLIENT_ID | None | Keycloak client id | +| RM_KEYCLOAK_REALM_NAME | None | Keycloak realm name | +| RM_KEYCLOAK_CLIENT_SECRET | None | Keycloak secert | +| RM_LDAP_CONN_STRING | None | LDAP conn string | +| RM_LDAP_USERNAME | None | LDAP username | +| RM_LDAP_CLIENT_SECRET | None | LDAP connection secret | +| RM_SQLITE_FILEPATH_PROD | /opt/rasenmaher/persistent/sqlite/rm_db.sql | location for sqlite database file in "prod" | +| RM_SQLITE_FILEPATH_DEV | /tmp/rm_db.sql | location for sqlite database file in "dev", local development | + +Container Variables + +You can create .env file in the root +directory and place all environment variables here. + +All environment variables should start with +RM\_ prefix. + +For example if you see in your "rasenmaeher_api/settings.py" a variable +named like random_parameter, you should +provide the "RM_RANDOM_PARAMETER" variable to configure the value. This +behaviour can be changed by overriding +env_prefix property in +rasenmaeher_api.settings.Settings.Config. + +An example of .env file: +`` `bash RM_RELOAD="True" RM_PORT="8000" RM_ENVIRONMENT="dev" ``\` + +You can read more about BaseSettings class here: +[https://pydantic-docs.helpmanual.io/usage/settings/](https://pydantic-docs.helpmanual.io/usage/settings/) + +## Backend services manifest + +Integration APIs are always https in port 4625. Default location for the +manifest is /pvarki/kraftwerk-rasenmaeher-init.json + +`` `json { "dns": "sleepy-sloth.pvarki.fi", "products": { "tak": "tak.sleepy-sloth.pvarki.fi", "ptt": "ptt.sleepy-sloth.pvarki.fi" } } ``\` + +## Api endpoints and usage + +| State | Method | URI | Request JSON | Response JSON | Api description . | +|---------|--------|------------------------------------|--------------------------------------------------------------|-------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| Dummy | GET | /api/v1/enroll/status/{work_id} | NA | {'status':'None/Processing/Denied/WaitingForAcceptance/ReadyForDelivery/Delivered'} | Check the situation of enrollment process, None = no enrollment started, this work_id is free to use. | +| Dummy | POST | /api/v1/enroll/init | {'work_id':'{work_id}'} | {'work_id':'{work_id}', 'id_hash':{id_string} } | Start service access enrollment for given {work_id} | +| Dummy | GET | /api/v1/enroll/deliver/{id_string} | NA | {'dl_link':"{[http://here.be/zip](http://here.be/zip)}"} | Deliver download link for enrollment zip | +| Dummy | POST | /api/v1/enroll/accept | { 'permit_string':'{permit_string}, 'id_hash':'{id_hash}' '} | { 'access':'granted/denied/None', 'work_id':'{work_id}' } | Accept the enrollment request | +| Testing | POST | /api/v1/product/sign_csr | { 'TO':'DO'} | { 'TO':'DO'} | Accept the enrollment request | +| Testing | POST | /api/v1/enroll/accept | { 'permit_string':'{permit_string}, 'id_hash':'{id_hash}' '} | { 'access':'granted/denied/None', 'work_id':'{work_id}' } | Accept the enrollment request | + +API vimpain + +## Example usage + +\# REQUEST A NEW CERTIFICATE USING CSR (requires cfssl backend for the +api container) + +> `` `curl -L -H "Content-Type: application/json" -d '{"csr": "-----BEGIN CERTIFICATE REQUEST-----\nMIIBUjCB+QIBADBqMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLZXhhbXBsZS5jb20x\nFjAUBgNVBAcTDVNhbiBGcmFuY2lzY28xEzARBgNVBAgTCkNhbGlmb3JuaWExGDAW\nBgNVBAMTD3d3dy5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA\nBK/CtZaQ4VliKE+DLIVGLwtSxJgtUKRzGvN1EwI3HRgKDQ3l3urBIzHtUcdMq6HZ\nb8jX0O9fXYUOf4XWggrLk1agLTArBgkqhkiG9w0BCQ4xHjAcMBoGA1UdEQQTMBGC\nD3d3dy5leGFtcGxlLmNvbTAKBggqhkjOPQQDAgNIADBFAiAcvfhXnsLtzep2sKSa\n36W7G9PRbHh8zVGlw3Hph8jR1QIhAKfrgplKwXcUctU5grjQ8KXkJV8RxQUo5KKs\ngFnXYtkb\n-----END CERTIFICATE REQUEST-----\n"}' 127.0.0.1:8000/api/v1/takreg | jq ``\` + +\# REQUEST A NEW CERTIFICATE WITHOUT CSR (requires cfssl backend for the +api container) + +> `` `curl -L -H "Content-Type: application/json" -d '{ "request": {"hosts":["harjoitus1.pvarki.fi"], "names":[{"C":"FI", "ST":"Jyvaskyla", "L":"KeskiSuomi", "O":"harjoitus1.pvarki.fi"}], "CN": "harjoitus1.pvarki.fi"}, "bundle":true, "profile":"client"}' 127.0.0.1:8000/takreg | jq ``\` + +\# LIST CFSSL CRL LIST + +> `` `curl -L -H "Content-Type: application/json" -d '{ "request": {"hosts":["harjoitus1.pvarki.fi"], "names":[{"C":"FI", "ST":"Jyvaskyla", "L":"KeskiSuomi", "O":"harjoitus1.pvarki.fi"}], "CN": "harjoitus1.pvarki.fi"}, "bundle":true, "profile":"client"}' 127.0.0.1:8000/takreg | jq ``\` + +The cfssl used behind API listens this kind of stuff +[https://github.com/cloudflare/cfssl/blob/master/doc/api/endpoint_newcert.txt](https://github.com/cloudflare/cfssl/blob/master/doc/api/endpoint_newcert.txt) + +\# Enrollment - Enroll a new work_id + +> `` `curl -H "Content-Type: application/json" -d '{"work_id":"porakoira666"}' http://127.0.0.1:8000/api/v1/enrollment/init ``\` + +\# Enrollment - Check the status and availability of work_id + +> `` `curl http://127.0.0.1:8000/api/v1/enrollment/status/koira ``\` + +\# Enrollment - Request the download link using the provided work_id_hash +`` `curl http://127.0.0.1:8000/api/v1/enrollment/deliver/zxzxzxzxzxzxzxxzzx ``\` + +\# Enrollment - Accept enrollment using permit_str +`` `curl -H "Content-Type: application/json" -d '{"enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/accept ``\` + +\# Enrollment - Set download link for enrollment +`` `curl -H "Content-Type: application/json" -d '{"download_link":"https://kuvaton.com","enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/set-dl-link ``\` + +\# Enrollment - Set state for enrollment +`` `curl -H "Content-Type: application/json" -d '{"state":"ReadyForDelivery","enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/set-state ``\` + +\# Enrollment - Add new permit_str +`` `curl -H "Content-Type: application/json" -d '{"permissions_str":"all", "new_permit_hash":"too_short","permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/add-manager ``\` + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rasenmaeher_api:devel_shell . + docker create --name rasenmaeher_api_devel -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` rasenmaeher_api:devel_shell + docker start -i rasenmaeher_api_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rasenmaeher_api_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rasenmaeher_api_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rasenmaeher_api:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rasenmaeher_api:tox . + docker run --rm -it --network host -v /var/run/docker.sock:/var/run/docker.sock -v `pwd`":/app" `echo $DOCKER_SSHAGENT` rasenmaeher_api:tox + +NOTE: This will not work from the git submodule directory in the +integration repo, docker tests must be run from the original repo. + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rasenmaeher_api:amd64-latest . + docker run --rm -it --name rasenmaeher_openapijson rasenmaeher_api:amd64-latest rasenmaeher_api openapi + docker run -it --name rasenmaeher_api rasenmaeher_api:amd64-latest + +There is also a specific target for just dumping the openapi.json: + + docker build --ssh default --target openapi -t rasenmaeher_api:amd64-openapi . + docker run --rm -it --name rasenmaeher_openapijson rasenmaeher_api:amd64-openapi + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.15.1/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.15.1/submodules/battlelog.md new file mode 100644 index 00000000..aa1243d9 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/submodules/battlelog.md @@ -0,0 +1,53 @@ +--- +title: "pvarki/typescript-liveloki-app – README" +--- + +> **Integration tag:** `1.15.1` · **Submodule commit:** `197f6b74507fb81fc442fa1522088f0cb7ed3939` +> **Repo:** git@github.com:pvarki/typescript-liveloki-app.git +> **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/197f6b74507fb81fc442fa1522088f0cb7ed3939 + +# BattleLog + +## Install + +1. Install docker + compose +2. Rename .env_example --> .env and modify as you like. +3. Run docker compose build --no-cache +4. Run docker compose up -d +5. Navigate to localhost:3000 + +## Frontend development + +The frontend is bundled with [Vite](https://vitejs.dev/). + +Once the backend is running (on port 3000), you can navigate to `frontend/`, +run `npm i` and `npm run dev` to run the Vite development server that has +hot reload and all that jazz. + +## Info + +Database is currently preseeded with test data from preseed/preseed.csv + +## Migrations + +To add new migration, locally run: `./node_modules/.bin/node-pg-migrate create ` and modify created file in `/migrations` directory. +Migrations are run (if needed) when docker container starts. `wait-for-it.sh` will ensure that psql container is up and accepting connections before running migrations. + +## JWT testing + +Remove comments from "jwt-test-network" for local jwt testing, to allow joining containers from 2 different compose runs to same docker network. + +## OpenAPI Specification + +https://pvarki.github.io/typescript-liveloki-app/ + +## Running tests + +In directory `tests`, use + +```shell +npm run test:integration +``` + +with the server running in `localhost:3000` as it does by default after `docker compose up -d`. + diff --git a/docs/dev/integrationrepo/v/1.15.1/submodules/cfssl.md b/docs/dev/integrationrepo/v/1.15.1/submodules/cfssl.md new file mode 100644 index 00000000..55bb40d1 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/submodules/cfssl.md @@ -0,0 +1,91 @@ +--- +title: "pvarki/docker-rasenmaeher-cfssl – README" +--- + +> **Integration tag:** `1.15.1` · **Submodule commit:** `87f2c2b4b22a0d25d7dcc900d0750373b24fec4b` +> **Repo:** git@github.com:pvarki/docker-rasenmaeher-cfssl.git +> **Browse at this commit:** https://github.com/pvarki/docker-rasenmaeher-cfssl/tree/87f2c2b4b22a0d25d7dcc900d0750373b24fec4b + +# cfssl Submodule + +\![Build +Status\]([https://github.com/pvarki/docker-rasenmaeher-cfssl/actions/workflows/build.yml/badge.svg](https://github.com/pvarki/docker-rasenmaeher-cfssl/actions/workflows/build.yml/badge.svg)) + +## Used as git submodule + +This repo is used as submodule in +[https://github.com/pvarki/docker-rasenmaeher-integration](https://github.com/pvarki/docker-rasenmaeher-integration) it is +probably a good idea to handle all development via it because it has +docker composition for bringin up all the other services rasenmaeher-api +depends on + +## Development + +The cfssl itself we can't do much about but the FastAPI thing uses +poetry and in any case use pre-commit: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t ocsprest:devel_shell . + docker create --name ocsprest_devel -v `pwd`/../":/app" -it `echo $DOCKER_SSHAGENT` ocsprest:devel_shell + docker start -i ocsprest_devel + +Or just pwd if working under separate checkout instead of the +integration repo. + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i ocsprest_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i ocsprest_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" ocsprest:devel_shell -c "pre-commit run --all-files" + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target ocsprest -t ocsprest:amd64-latest . + docker run -it --name ocsprest ocsprest:amd64-latest + +There is also a specific target for just dumping the openapi.json: + + docker build --ssh default --target openapi -t ocsprest:amd64-openapi . + docker run --rm -it --name rasenmaeher_openapijson ocsprest:amd64-openapi + diff --git a/docs/dev/integrationrepo/v/1.15.1/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.15.1/submodules/fpintegration.md new file mode 100644 index 00000000..709d5d66 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/submodules/fpintegration.md @@ -0,0 +1,106 @@ +--- +title: "pvarki/python-rasenmaeher-rmfpapi – README" +--- + +> **Integration tag:** `1.15.1` · **Submodule commit:** `7b24c8155518a4e11dd46b1f8d210b850973004c` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-rmfpapi.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-rmfpapi/tree/7b24c8155518a4e11dd46b1f8d210b850973004c + +# rmfpapi + +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rmfpapi:devel_shell . + docker create --name rmfpapi_devel -p 4625:4625 -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` rmfpapi:devel_shell + docker start -i rmfpapi_devel + +In the shell you can start the uvicorn devel server with (binding to +0.0.0.0 is important!): + + uvicorn "rmfpapi.app:get_app" --factory --host 0.0.0.0 --port 4625 --reload --log-level debug + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmfpapi_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rmfpapi_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rmfpapi:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rmfpapi:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` rmfpapi:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rmfpapi:latest . + docker run -it --name rmfpapi -p 4625:4625 rmfpapi:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.15.1/submodules/keycloak2.md b/docs/dev/integrationrepo/v/1.15.1/submodules/keycloak2.md new file mode 100644 index 00000000..1082b4cf --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/submodules/keycloak2.md @@ -0,0 +1,31 @@ +--- +title: "pvarki/docker-keycloak – README" +--- + +> **Integration tag:** `1.15.1` · **Submodule commit:** `8b2efdf6a4d90fd91d2074617983c086ebe537c5` +> **Repo:** git@github.com:pvarki/docker-keycloak.git +> **Browse at this commit:** https://github.com/pvarki/docker-keycloak/tree/8b2efdf6a4d90fd91d2074617983c086ebe537c5 + +# KeyCloak + +Templates, init container etc to setup keycloak automagically with ENV +variables. + +## Used as git submodule + +This repo is used as submodule in +[https://github.com/pvarki/docker-rasenmaeher-integration](https://github.com/pvarki/docker-rasenmaeher-integration) it is +probably a good idea to handle all development via it because it has +docker composition for bringin up all the other services rasenmaeher-api +depends on + +## pre-commit notes + +Make sure pre-commit is installed: + + pre-commit install --install-hooks + +And it's a good idea to run it regularly before committing: + + pre-commit run --all-files + diff --git a/docs/dev/integrationrepo/v/1.15.1/submodules/kw_product_init.md b/docs/dev/integrationrepo/v/1.15.1/submodules/kw_product_init.md new file mode 100644 index 00000000..aa8931f5 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/submodules/kw_product_init.md @@ -0,0 +1,78 @@ +--- +title: "pvarki/golang-kraftwerk-init-helper-cli – README" +--- + +> **Integration tag:** `1.15.1` · **Submodule commit:** `0dee1f8f397ba37280f6bf393a3b866d93387e8c` +> **Repo:** git@github.com:pvarki/golang-kraftwerk-init-helper-cli.git +> **Browse at this commit:** https://github.com/pvarki/golang-kraftwerk-init-helper-cli/tree/0dee1f8f397ba37280f6bf393a3b866d93387e8c + +golang-kraftwerk-init-helper-cli =============================== + +Tool for products to create their certificates from +KRAFTWERK manifests. + +# Development + +## Prerequisites + +**Enable** +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/) + +```bash +export DOCKER_BUILDKIT=1 +``` + +**Forward SSH-agent to running instance:** + +**OSX:** + +```bash +export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" +``` + +**Linux:** + +```bash +export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" +``` + +## Create & start development container + +Build the image, create a container and start the container + +```bash +docker build --ssh default --target dev_shell -t kraftwerk_init_helper:dev_shell . +``` + +```bash +docker create --name kraftwerk_init_helper -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` kraftwerk_init_helper:dev_shell +``` + +```bash +docker start -i kraftwerk_init_helper +``` + +## pre-commit initialization + +Once inside the container, run: + +```bash +pre-commit install && pre-commit run --all-files +``` + +That's it, now you have the development environment up & running. + +# Production + +Build the production image: + +```bash +docker build --ssh default --target production -t kraftwerk_init_helper:latest . +``` + +Run the image: + +```bash +docker run --rm -it --name kraftwerk_helper kraftwerk_init_helper:latest +``` + diff --git a/docs/dev/integrationrepo/v/1.15.1/submodules/miniwerk.md b/docs/dev/integrationrepo/v/1.15.1/submodules/miniwerk.md new file mode 100644 index 00000000..45406f30 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/submodules/miniwerk.md @@ -0,0 +1,104 @@ +--- +title: "pvarki/python-rasenmaeher-miniwerk – README" +--- + +> **Integration tag:** `1.15.1` · **Submodule commit:** `073f866ad4b47dc7d1eab0cc06ecaf45c5fc5c4d` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-miniwerk.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-miniwerk/tree/073f866ad4b47dc7d1eab0cc06ecaf45c5fc5c4d + +# miniwerk + +Minimal KRAFTWERK amulation to be able to run a RASENMAEHER+products +deployment on any VM + +There are some required ENV configs check out example_env.sh for them +(.env file is also supported) + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t miniwerk:devel_shell . + docker create --name miniwerk_devel -v `pwd`":/app" -p 80:80 -it `echo $DOCKER_SSHAGENT` miniwerk:devel_shell + docker start -i miniwerk_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i miniwerk_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i miniwerk_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" miniwerk:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t miniwerk:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` miniwerk:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t miniwerk:amd64-latest . + docker run -it --name miniwerk miniwerk:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.15.1/submodules/mtxauthz.md b/docs/dev/integrationrepo/v/1.15.1/submodules/mtxauthz.md new file mode 100644 index 00000000..52c54b50 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/submodules/mtxauthz.md @@ -0,0 +1,122 @@ +--- +title: "pvarki/python-mediamtx-rmmtxauthz – README" +--- + +> **Integration tag:** `1.15.1` · **Submodule commit:** `55921125444547f67089706f64432e5e3aef7763` +> **Repo:** git@github.com:pvarki/python-mediamtx-rmmtxauthz.git +> **Browse at this commit:** https://github.com/pvarki/python-mediamtx-rmmtxauthz/tree/55921125444547f67089706f64432e5e3aef7763 + +# rmmtxauthz + +Do HTTP based authz for MediaMTX + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rmmtxauthz:devel_shell . + docker create --name rmmtxauthz_devel -v "$(pwd)/rune/output/rune.json:/opt/templates/mediamtx.json" -v "$(pwd):/app" -it $(echo $DOCKER_SSHAGENT) rmmtxauthz:devel_shell + docker start -i rmmtxauthz_devel + +To rebuild the documentation inside the container run: + + rune rune/src json >/opt/templates/mediamtx.json + +Outside of container use: + + rune rune/src json >rune/output/rune.json + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmmtxauthz_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rmmtxauthz_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v "$(pwd):/app" rmmtxauthz:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rmmtxauthz:tox . + docker run --rm -it -v "$(pwd):/app" $(echo $DOCKER_SSHAGENT) rmmtxauthz:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rmmtxauthz:amd64-latest . + docker run -it --name rmmtxauthz rmmtxauthz:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p $(which python3.11) my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + git add poetry.lock + pre-commit install --install-hooks + pre-commit run --all-files + +If you get weird errors about missing packages from pre-commit try +running it with "poetry run pre-commit". + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + +Running "pre-commit run --all-files" and "py.test -v" regularly during +development and especially before committing will save you some +headache. + +### RUNE instructions compile + +tldr: + + rune rune/src json >rune/output/mediamtx.json + diff --git a/docs/dev/integrationrepo/v/1.15.1/submodules/takintegration.md b/docs/dev/integrationrepo/v/1.15.1/submodules/takintegration.md new file mode 100644 index 00000000..a1dc094a --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/submodules/takintegration.md @@ -0,0 +1,100 @@ +--- +title: "pvarki/python-tak-rmapi – README" +--- + +> **Integration tag:** `1.15.1` · **Submodule commit:** `0f3021ec2771c0d3846b46a302692f97fa7f2a19` +> **Repo:** git@github.com:pvarki/python-tak-rmapi.git +> **Browse at this commit:** https://github.com/pvarki/python-tak-rmapi/tree/0f3021ec2771c0d3846b46a302692f97fa7f2a19 + +# takrmapi + +RASENMAEHER integration API for TAK server + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t takrmapi:devel_shell . + docker create --name takrmapi_devel -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` takrmapi:devel_shell + docker start -i takrmapi_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i takrmapi_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i takrmapi_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" takrmapi:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t takrmapi:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` takrmapi:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t takrmapi:amd64-latest . + docker run -it --name takrmapi takrmapi:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.15.1/submodules/takserver.md b/docs/dev/integrationrepo/v/1.15.1/submodules/takserver.md new file mode 100644 index 00000000..3aa111f4 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/submodules/takserver.md @@ -0,0 +1,68 @@ +--- +title: "pvarki/docker-atak-server – README" +--- + +> **Integration tag:** `1.15.1` · **Submodule commit:** `7a73302fef52d0e4b76b11bbda92afefaab43732` +> **Repo:** git@github.com:pvarki/docker-atak-server.git +> **Browse at this commit:** https://github.com/pvarki/docker-atak-server/tree/7a73302fef52d0e4b76b11bbda92afefaab43732 + +# Run TAK Java server in container + +tldr: + + cp takserver.env.example takserver.env + # edit the env + # export the variables gomplate uses (see the .tpl files) + docker compose pull --include-deps --ignore-pull-failures + docker compose -p tak up -d + +or use docker compose.local.yml without gomplate for local dev +(rebuilding containers): + + export DOCKER_TAG_EXTRA="-dev" + docker build --no-cache --progress=plain -t takserver:latest${DOCKER_TAG_EXTRA} -t takserver:4.7-RELEASE-32${DOCKER_TAG_EXTRA} -t pvarki/takserver:4.7-RELEASE-32${DOCKER_TAG_EXTRA} . + cp takserver.env.example takserver.env + # edit the env + docker compose -f docker-compose.local.yml -p tak up + +Note, for things that live in the volumes (like TAK certs) you must nuke +the volumes to see changes: + + docker compose -f docker-compose.local.yml -p tak down -v ; docker compose -f docker-compose.local.yml -p tak rm -vf + +## Creating client packages locally + +Using the REST API is probably nicer though + +Create client package: + + docker compose -p tak exec takserver_api /bin/bash -c 'CLIENT_CERT_NAME=replaceme /opt/scripts/make_client_zip.sh' + +Then get /opt/tak/certs/files/clientpkgs/replaceme.zip out of the +container: + + docker compose -p tak exec taktakserver_apiserver /bin/bash -c 'base64 /opt/tak/certs/files/clientpkgs/replaceme.zip' | base64 -id >replaceme.zip + +This approach also works for recovering the admin cert +(/opt/tak/certs/files/admin.p12 unless you changed the ADMIN_CERT_NAME +ENV) + +## Creating new admin users + +Create the user on the takserver container: + + docker compose -p tak exec takserver_api /bin/bash -c 'cd /opt/tak/data/certs/ && CAPASS=$CA_PASS PASS=replaceme_user_cert_pass ./makeCert.sh client replaceme_username && ADMIN_CERT_NAME=replaceme_username /opt/scripts/enable_admin.sh' + +See above about the hard way of getting the cert file, or use the REST +API. + +## Gradle builds + +Build the distribution: + + mkdir outputs + docker build --progress=plain -f Dockerfile_build --target files -t atakbuild:files . + docker run --rm -it -v `pwd`/outputs:/output atakbuild:files + +Now you have the build artefacts in outputs -directory. + diff --git a/docs/dev/integrationrepo/v/1.15.1/submodules/ui.md b/docs/dev/integrationrepo/v/1.15.1/submodules/ui.md new file mode 100644 index 00000000..271f908f --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.1/submodules/ui.md @@ -0,0 +1,116 @@ +--- +title: "pvarki/rasenmaeher-ui – README" +--- + +> **Integration tag:** `1.15.1` · **Submodule commit:** `eb2d303774fa7791ba5410080cd55316fe8b32c1` +> **Repo:** git@github.com:pvarki/rasenmaeher-ui.git +> **Browse at this commit:** https://github.com/pvarki/rasenmaeher-ui/tree/eb2d303774fa7791ba5410080cd55316fe8b32c1 + +# UI For RASENMAEHER + +React, TS etc + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rasenmaeher_ui:devel_shell . + docker create --name rmui_devel -v `pwd`":/app" -p 8002:8002 -it `echo $DOCKER_SSHAGENT` rasenmaeher_ui:devel_shell + docker start -i rmui_devel + +Note: due to the volume mapping if you already had run "npm install" on +the host you need to delete the node_modules directory and in any case +you need to run "npm install" inside the container. + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmui_devel /bin/bash -c "pre-commit install" + docker exec -i rmui_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rasenmaeher_ui:devel_shell -c "pre-commit run --all-files" + +### Production docker + +This will just deliver the compiled files to a mounted directory + +> docker build --ssh default --target production -t +> rasenmaeher_ui:amd64-latest . docker run -it --rm -v +> pwd/rmui_files:/deliver +> rasenmaeher_ui:amd64-latest + +Then you need to serve things so that index.html is the default +controller for things, nginx example: + + location / { + index index.html; + root /rmui_files; + try_files $uri $uri/ /index.html =404; + } + +## Development + +TLDR: + +- Create and activate a Node 18 virtualenv: + + FIXME: Insert instructions + +- change to a branch: + + git checkout -b my_branch + +- Install project deps and pre-commit hooks: + + npm install + pre-commit install + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo. + +### Asset set + +Asset set is loaded according to VITE_ASSET_SET enviroment variable. +Changing asset set requires reloading vite. Folder assetSetStore +contains files for for different sets that are copied to src/assets/set/ +folder on startup. Current set is tracked with setName.txt file. + +When you want to use dynamic assets, you want to make sure your dynamic +asset has equivalent for sets (both "fdf" and "neutral", so on). If the +assets are images, you want to then import them to your components from +/assets/set/\* where the env-defined assets are placed by VITE on +startup. If the assets are translation keys, make sure keys are present +in locales under assetSetStore, and then use the "dynamic" namespace for +accessing them. + diff --git a/docs/dev/integrationrepo/v/1.15.2/_category_.json b/docs/dev/integrationrepo/v/1.15.2/_category_.json new file mode 100644 index 00000000..a14909ff --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "1.15.2", + "position": 1 +} diff --git a/docs/dev/integrationrepo/v/1.15.2/index.md b/docs/dev/integrationrepo/v/1.15.2/index.md new file mode 100644 index 00000000..7fbfe6a4 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/index.md @@ -0,0 +1,4 @@ +--- +title: "1.15.2" +--- + diff --git a/docs/dev/integrationrepo/v/1.15.2/integration.md b/docs/dev/integrationrepo/v/1.15.2/integration.md new file mode 100644 index 00000000..62a2e8dc --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/integration.md @@ -0,0 +1,319 @@ +--- +title: "pvarki/docker-rasenmaeher-integration – README" +--- + +> **Integration tag:** `1.15.2` +> **Repo:** https://github.com/pvarki/docker-rasenmaeher-integration + +![Build Status](https://github.com/pvarki/docker-rasenmaeher-integration/actions/workflows/build.yml/badge.svg) + +# Deploy App + +"One Ring to rule them all, One Ring to find them, One Ring to bring +them all and in the darkness bind them." + +Docker compositions, helpers etc to bring it all together into something +resembling grand old ones. + +Codename RASENMAEHER because infantry jokes. + +## WTF is this anyway ? + +This [Disobey24 +talk](https://www.youtube.com/watch?v=m3xd7uygpaY&list=PLLvAhAn5sGfiB9AlEt2KD7H9Dnr6kbd64&index=23) +explains a lot. + +## Running Deploy App in your own docker environment + +### Needed DNS Records + +These need to point to your WAN address. + +> - domain +> - kc.domain +> - tak.domain +> - bl.domain +> - mtx.domain +> - mtls.domain +> - mtls.kc.domain +> - mtls.tak.domain +> - mtls.kc.domain +> - mtls.bl.domain +> - mtls.mtx.domain + +When more products are added to the deployment they will follow the same +naming pattern, you will need subdomains for all products listed in the +composition for miniwerk service variable MW_PRODUCTS and "kc" for +Keycloak. + +### Needed ports open + +And redirected to the server if behind NAT or similar. + +> - 80 +> - 443 +> - 8443 (TAK) +> - 8446 (TAK) +> - 9446 (Keycloak) +> - 4626 (Product integration APIs port) +> - 4666 (Battlelog API/UI port) +> - 1936 RTMPS +> - 8322 RTSPS +> - 8890 SRT +> - 9888 HSL +> - 9889 WebRTC +> - 9996 Playback for recorded streams/videos + +### Downloading and composing Deploy App + +Be mindfull on where you download the repository, you will need to +perform rest of the commands inside the downloaded repository. + +Getting the repository from github (on Windows **first** see "Windows +notes" below): + + git clone --recurse-submodules -j8 git@github.com:pvarki/docker-rasenmaeher-integration.git + +Create `.env` file that defines environmental variables for Deploy App +setup. File must be located inside downloaded repository and file type +must be named literally `.env` (not `something.env`) to work. + +The original example file is: +[https://github.com/pvarki/docker-rasenmaeher-integration/blob/main/example_env.sh](https://github.com/pvarki/docker-rasenmaeher-integration/blob/main/example_env.sh) + +Example .env-file with the minimal information needed: + + KEYCLOAK_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RM_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + POSTGRES_PASSWORD="input-secure-password" # pragma: allowlist secret + LDAP_ADMIN_PASSWORD="input-secure-password" # pragma: allowlist secret + KEYCLOAK_ADMIN_PASSWORD="input-secure-password" # pragma: allowlist secret + TAK_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + SERVER_DOMAIN="input-domain" + CFSSL_CA_NAME="input-ca-name" + MW_LE_EMAIL="input-email-for-lets-encrypt" + MW_LE_TEST="false" + TAKSERVER_CERT_PASS="input-secure-password" # pragma: allowlist secret + TAK_CA_PASS="input-secure-password" # pragma: allowlist secret + VITE_ASSET_SET="${VITE_ASSET_SET:-neutral}" + KEYCLOAK_PROFILEROOT_UUID="input-uuid" + KEYCLOAK_HTTPS_KEY_STORE_PASSWORD="input-secure-password" # pragma: allowlist secret + KEYCLOAK_HTTPS_TRUST_STORE_PASSWORD="input-secure-password" # pragma: allowlist secret + BL_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RMMTX_DATABASE_PASSWORD="input-secure-password" # pragma: allowlist secret + RMMTX_API_PASSWORD="input-secure-password" # pragma: allowlist secret + +Replace "intput-secure-password" with a good passphrase that is unique +for each replacment. You can generate an UUID with: + + python -c 'import uuid; print(uuid.uuid4())' + +If you wish to use one deployment for longer than the *design lifetime* +of 1-2 months you can change the following env variables. But do +understand that this is **not recommended** and has **security +implications**. If you do this **you** take **responsibility** to go +through all Dockerfiles and compositions to understand **exactly** how +things are done and how apply security updates into the containers: + + CFSSL_CA_EXPIRY="8800h" # Input time in hours = xxh, 8800h is 366.. days + CFSSL_SIGN_DEFAULT_EXPIRY="8800h" # Input time in hours = xxh, 8800h is 366.. days + +Starting the services: + + docker compose up -d + +Updating the repository from github: + + git pull + git submodule update --init --recursive + docker compose pull + docker compose up -d + +!DO NOT DO! Deleting the services. Deletes the certificates etc you will +need to add all users etc again: + + docker compose down -v + +Getting the admin login invite code for first admin: + + docker compose exec -it rmapi /bin/bash -c "rasenmaeher_api addcode" + +If you get "no such service -d", make sure whatever you copy-pasted the +command from did not render the dash (ASCII 0x2D) as some other +codepoint with a glyph that looks deceptively similar. When in doubt +write the commands yourself instead of copy-pasting. + +### Services + +Deploy App login page: + + https://domain (example.com) + +Deploy App home page: + + https://mtls.domain (mtls.example.com) + +Takserver Admin UI: + + https://tak.domain:8443/ (tak.example.com:8443/) + +Keycloack Admin UI. (Later group management will be withing Deploy App): + + https://kc.domain:9443/admin/RASENMAEHER/console/ (kc.example.com:9443/admin/RASENMAEHER/console/) + +OTA update server inside takserver. Is located in the loaded repository, +location depends on where you downloaded it: + + /home/user/docker-rasenmaeher-integration/takserver/update + +### Using the Deploy App service + +1. Login with first admin code. Create your admin account by typing + your first admin invite code and inputting desired admin callsign. +2. Create invite code for other users. Share the invite code. Go to + Manage Users -\> Add Users -\> Create New Invite. Share link, qr + code or invite code and domain. +3. Approve users in Deploy App. Open approvement link or scan qr code + from users and approve the user. You can also go to Approve Users + -\> Select Waiting User and input the users approvement code. +4. If desired promote some of the added users as admins. Go to Manage + Users -\> Manage Users -\> Select user and select Promote. You can + also Demote Admins or Delete users altogether. + +### Using Deploy App TAK in EUD + +EUD=End User Device + +1. Login to Deploy App. Go to [https://mtls.domain](https://mtls.domain) and select TAK. +2. Download Client Package. Select tak package for desired software + "Android ATAK or Windows WinTAK" or "iOS iTAK". Select Download + Client Package. +3. Go to EUD's TAK Software. Import downloaded package. Device is + connected to server. +4. You should also read Quickstart and Usage Guides. + +## Git submodules + +When cloning for the first time use: + + git clone --recurse-submodules -j8 git@github.com:pvarki/docker-rasenmaeher-integration.git + +When updating or checking out branches use: + + git submodule update --init --recursive + +And if you forgot to --recurse-submodules run the update command above. + +The submodules are repos in their own right, if you plan to make changes +into them change to the directory and create new branch, commit and push +changes as usual under that directory. + +### Directories that are submodules + +> - api [https://github.com/pvarki/python-rasenmaeher-api](https://github.com/pvarki/python-rasenmaeher-api) +> - cfssl [https://github.com/pvarki/docker-rasenmaeher-cfssl](https://github.com/pvarki/docker-rasenmaeher-cfssl) +> - fpintegration [https://github.com/pvarki/python-rasenmaeher-rmfpapi](https://github.com/pvarki/python-rasenmaeher-rmfpapi) +> - keycloak [https://github.com/pvarki/docker-keycloak](https://github.com/pvarki/docker-keycloak) +> - kw_product_init +> [https://github.com/pvarki/golang-kraftwerk-init-helper-cli](https://github.com/pvarki/golang-kraftwerk-init-helper-cli) +> - openldap [https://github.com/pvarki/docker-openldap](https://github.com/pvarki/docker-openldap) +> - miniwerk [https://github.com/pvarki/python-rasenmaeher-miniwerk](https://github.com/pvarki/python-rasenmaeher-miniwerk) +> - ui [https://github.com/pvarki/rasenmaeher-ui](https://github.com/pvarki/rasenmaeher-ui) +> - takserver [https://github.com/pvarki/docker-atak-server](https://github.com/pvarki/docker-atak-server) +> - takintegration [https://github.com/pvarki/python-tak-rmapi](https://github.com/pvarki/python-tak-rmapi) +> - battlelog [https://github.com/pvarki/typescript-liveloki-app](https://github.com/pvarki/typescript-liveloki-app) +> - mtxauthz [https://github.com/pvarki/python-mediamtx-rmmtxauthz](https://github.com/pvarki/python-mediamtx-rmmtxauthz) + +## Autogenerated (mostly API) docs + +> - Module API docs: +> [https://pvarki.github.io/docker-rasenmaeher-integration/docs/](https://pvarki.github.io/docker-rasenmaeher-integration/docs/) +> - Swagger definition for Deploy App API: +> [https://pvarki.github.io/docker-rasenmaeher-integration/](https://pvarki.github.io/docker-rasenmaeher-integration/) + +## Running in local development mode + +### Windows notes + +> 1. Do **NOT** use git-bash, it will cause *weirdest* issues with +> Docker containers +> +> 2. Use WSL, see +> [best_practises](https://github.com/pvarki/markdown-pvarki-best_practises) +> -repo for instructions on how to set it up. +> +> 3. Make sure whatever git client or IDE you use it does not mess with +> line-endings, for CLI client this does the trick: +> +> git config --global core.eol lf +> git config --global core.autocrlf false + +### Compositions + +Generally start with "rmlocal", it corresponds best to a real running +environment. "rmdev" starts a bunch of things in development mode which +does make developing more convenient but also introduces extra +variability to how things work. + +Make sure to always check your changes work correctly in rmlocal mode +where assets are minified and baked in. + +TLDR: + + alias rmlocal="docker compose -p rmlocal -f docker-compose-local.yml" + rmlocal build takinit miniwerk --pull + rmlocal build --pull + rmlocal up + +or: + + alias rmdev="docker compose -p rmdev -f docker-compose-local.yml -f docker-compose-dev.yml" + rmdev build takinit miniwerk --pull + rmdev build --pull + rmdev up + +OpenLDAP and keycloak-init sometimes fail on first start, just run up +again. + +IMPORTANT: Only keep either rmlocal or rmdev created at one time or you +may have weird network issues run "down" for one env before starting the +other. + +Remember to run "down -v" if you want to reset the persistent volumes, +or if you have weird issues when switching between environments. + +The dev version launches all the services and runs rasenmaeher-api in +uvicorn reload mode so any edits you make under /api will soon be +reflected in the running instance. + +If rasenmaeher-ui devel server complains make sure to delete +`ui/node_modules` -directory from host first. The docker NodeJS +distribution probably is not compatible with whatever you have installed +on the host. + +### Gaining first admin access in dev and production mode + +In dev mode: + + docker exec -it rmdev-rmapi-1 /bin/bash -c "source /.venv/bin/activate && rasenmaeher_api addcode" + +Under dev mode, the UI runs at [https://localmaeher.dev.pvarki.fi:4439](https://localmaeher.dev.pvarki.fi:4439). + +In VM production mode: + + docker exec -it rmvm-rmapi-1 /bin/bash -c "rasenmaeher_api addcode" + +## pre-commit notes + +Use "pre-commit run --all-files" liberally (and make sure you have run +"pre-commit install --install-hooks"). If you get complaints about +missing environment variables run "source example_env.sh" + +## Integration tests + +Pytest is used to handle the integration tests, the requirements are in +tests/requirements.txt. NOTE: The tests have side-effects and expect a +clean database to start with so always make sure to run "down -v" for +the composition first, then bring it back up before running integration +tests. + diff --git a/docs/dev/integrationrepo/v/1.15.2/submodules/api.md b/docs/dev/integrationrepo/v/1.15.2/submodules/api.md new file mode 100644 index 00000000..a2cc0486 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/submodules/api.md @@ -0,0 +1,240 @@ +--- +title: "pvarki/python-rasenmaeher-api – README" +--- + +> **Integration tag:** `1.15.2` · **Submodule commit:** `0b51a68dda421693c8081505175b1250403bc595` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-api.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-api/tree/0b51a68dda421693c8081505175b1250403bc595 + +# python-rasenmaeher-api + +python-rasenmaeher-api + +## Configuration + +This application can be configured with environment variables. Below is +a example list of available variables. You can find all available +variables here +[https://github.com/pvarki/python-rasenmaeher-api/blob/main/src/rasenmaeher_api/settings.py](https://github.com/pvarki/python-rasenmaeher-api/blob/main/src/rasenmaeher_api/settings.py). + +| ENV VAR | Default value | Info / Usage | +|---------------------------|---------------------------------------------|---------------------------------------------------------------| +| RM_PORT | 8000 | Docker API listen port | +| RM_WORKERS_COUNT | 2 | Number of uvicorn workers | +| RM_RELOAD | False | Reload service on code change | +| RM_ENVIRONMENT | dev | Run dev / prod environment | +| RM_CFSSL_HOST | None | CFSSL host url | +| RM_CFSSL_PORT | None | CFSSL service port | +| RM_KEYCLOAK_SERVER_URL | None | Keycloak server url ([http://1234:8080/auth](http://1234:8080/auth)) | +| RM_KEYCLOAK_CLIENT_ID | None | Keycloak client id | +| RM_KEYCLOAK_REALM_NAME | None | Keycloak realm name | +| RM_KEYCLOAK_CLIENT_SECRET | None | Keycloak secert | +| RM_LDAP_CONN_STRING | None | LDAP conn string | +| RM_LDAP_USERNAME | None | LDAP username | +| RM_LDAP_CLIENT_SECRET | None | LDAP connection secret | +| RM_SQLITE_FILEPATH_PROD | /opt/rasenmaher/persistent/sqlite/rm_db.sql | location for sqlite database file in "prod" | +| RM_SQLITE_FILEPATH_DEV | /tmp/rm_db.sql | location for sqlite database file in "dev", local development | + +Container Variables + +You can create .env file in the root +directory and place all environment variables here. + +All environment variables should start with +RM\_ prefix. + +For example if you see in your "rasenmaeher_api/settings.py" a variable +named like random_parameter, you should +provide the "RM_RANDOM_PARAMETER" variable to configure the value. This +behaviour can be changed by overriding +env_prefix property in +rasenmaeher_api.settings.Settings.Config. + +An example of .env file: +`` `bash RM_RELOAD="True" RM_PORT="8000" RM_ENVIRONMENT="dev" ``\` + +You can read more about BaseSettings class here: +[https://pydantic-docs.helpmanual.io/usage/settings/](https://pydantic-docs.helpmanual.io/usage/settings/) + +## Backend services manifest + +Integration APIs are always https in port 4625. Default location for the +manifest is /pvarki/kraftwerk-rasenmaeher-init.json + +`` `json { "dns": "sleepy-sloth.pvarki.fi", "products": { "tak": "tak.sleepy-sloth.pvarki.fi", "ptt": "ptt.sleepy-sloth.pvarki.fi" } } ``\` + +## Api endpoints and usage + +| State | Method | URI | Request JSON | Response JSON | Api description . | +|---------|--------|------------------------------------|--------------------------------------------------------------|-------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| Dummy | GET | /api/v1/enroll/status/{work_id} | NA | {'status':'None/Processing/Denied/WaitingForAcceptance/ReadyForDelivery/Delivered'} | Check the situation of enrollment process, None = no enrollment started, this work_id is free to use. | +| Dummy | POST | /api/v1/enroll/init | {'work_id':'{work_id}'} | {'work_id':'{work_id}', 'id_hash':{id_string} } | Start service access enrollment for given {work_id} | +| Dummy | GET | /api/v1/enroll/deliver/{id_string} | NA | {'dl_link':"{[http://here.be/zip](http://here.be/zip)}"} | Deliver download link for enrollment zip | +| Dummy | POST | /api/v1/enroll/accept | { 'permit_string':'{permit_string}, 'id_hash':'{id_hash}' '} | { 'access':'granted/denied/None', 'work_id':'{work_id}' } | Accept the enrollment request | +| Testing | POST | /api/v1/product/sign_csr | { 'TO':'DO'} | { 'TO':'DO'} | Accept the enrollment request | +| Testing | POST | /api/v1/enroll/accept | { 'permit_string':'{permit_string}, 'id_hash':'{id_hash}' '} | { 'access':'granted/denied/None', 'work_id':'{work_id}' } | Accept the enrollment request | + +API vimpain + +## Example usage + +\# REQUEST A NEW CERTIFICATE USING CSR (requires cfssl backend for the +api container) + +> `` `curl -L -H "Content-Type: application/json" -d '{"csr": "-----BEGIN CERTIFICATE REQUEST-----\nMIIBUjCB+QIBADBqMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLZXhhbXBsZS5jb20x\nFjAUBgNVBAcTDVNhbiBGcmFuY2lzY28xEzARBgNVBAgTCkNhbGlmb3JuaWExGDAW\nBgNVBAMTD3d3dy5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA\nBK/CtZaQ4VliKE+DLIVGLwtSxJgtUKRzGvN1EwI3HRgKDQ3l3urBIzHtUcdMq6HZ\nb8jX0O9fXYUOf4XWggrLk1agLTArBgkqhkiG9w0BCQ4xHjAcMBoGA1UdEQQTMBGC\nD3d3dy5leGFtcGxlLmNvbTAKBggqhkjOPQQDAgNIADBFAiAcvfhXnsLtzep2sKSa\n36W7G9PRbHh8zVGlw3Hph8jR1QIhAKfrgplKwXcUctU5grjQ8KXkJV8RxQUo5KKs\ngFnXYtkb\n-----END CERTIFICATE REQUEST-----\n"}' 127.0.0.1:8000/api/v1/takreg | jq ``\` + +\# REQUEST A NEW CERTIFICATE WITHOUT CSR (requires cfssl backend for the +api container) + +> `` `curl -L -H "Content-Type: application/json" -d '{ "request": {"hosts":["harjoitus1.pvarki.fi"], "names":[{"C":"FI", "ST":"Jyvaskyla", "L":"KeskiSuomi", "O":"harjoitus1.pvarki.fi"}], "CN": "harjoitus1.pvarki.fi"}, "bundle":true, "profile":"client"}' 127.0.0.1:8000/takreg | jq ``\` + +\# LIST CFSSL CRL LIST + +> `` `curl -L -H "Content-Type: application/json" -d '{ "request": {"hosts":["harjoitus1.pvarki.fi"], "names":[{"C":"FI", "ST":"Jyvaskyla", "L":"KeskiSuomi", "O":"harjoitus1.pvarki.fi"}], "CN": "harjoitus1.pvarki.fi"}, "bundle":true, "profile":"client"}' 127.0.0.1:8000/takreg | jq ``\` + +The cfssl used behind API listens this kind of stuff +[https://github.com/cloudflare/cfssl/blob/master/doc/api/endpoint_newcert.txt](https://github.com/cloudflare/cfssl/blob/master/doc/api/endpoint_newcert.txt) + +\# Enrollment - Enroll a new work_id + +> `` `curl -H "Content-Type: application/json" -d '{"work_id":"porakoira666"}' http://127.0.0.1:8000/api/v1/enrollment/init ``\` + +\# Enrollment - Check the status and availability of work_id + +> `` `curl http://127.0.0.1:8000/api/v1/enrollment/status/koira ``\` + +\# Enrollment - Request the download link using the provided work_id_hash +`` `curl http://127.0.0.1:8000/api/v1/enrollment/deliver/zxzxzxzxzxzxzxxzzx ``\` + +\# Enrollment - Accept enrollment using permit_str +`` `curl -H "Content-Type: application/json" -d '{"enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/accept ``\` + +\# Enrollment - Set download link for enrollment +`` `curl -H "Content-Type: application/json" -d '{"download_link":"https://kuvaton.com","enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/set-dl-link ``\` + +\# Enrollment - Set state for enrollment +`` `curl -H "Content-Type: application/json" -d '{"state":"ReadyForDelivery","enroll_str":"zxzxzxzxzxzxzxxzzx", "permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/set-state ``\` + +\# Enrollment - Add new permit_str +`` `curl -H "Content-Type: application/json" -d '{"permissions_str":"all", "new_permit_hash":"too_short","permit_str":"PaulinTaikaKaulinOnKaunis_PaulisMagicPinIsBuuutiful!11!1"}' http://127.0.0.1:8000/api/v1/enrollment/config/add-manager ``\` + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rasenmaeher_api:devel_shell . + docker create --name rasenmaeher_api_devel -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` rasenmaeher_api:devel_shell + docker start -i rasenmaeher_api_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rasenmaeher_api_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rasenmaeher_api_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rasenmaeher_api:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rasenmaeher_api:tox . + docker run --rm -it --network host -v /var/run/docker.sock:/var/run/docker.sock -v `pwd`":/app" `echo $DOCKER_SSHAGENT` rasenmaeher_api:tox + +NOTE: This will not work from the git submodule directory in the +integration repo, docker tests must be run from the original repo. + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rasenmaeher_api:amd64-latest . + docker run --rm -it --name rasenmaeher_openapijson rasenmaeher_api:amd64-latest rasenmaeher_api openapi + docker run -it --name rasenmaeher_api rasenmaeher_api:amd64-latest + +There is also a specific target for just dumping the openapi.json: + + docker build --ssh default --target openapi -t rasenmaeher_api:amd64-openapi . + docker run --rm -it --name rasenmaeher_openapijson rasenmaeher_api:amd64-openapi + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.15.2/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.15.2/submodules/battlelog.md new file mode 100644 index 00000000..6957bc9e --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/submodules/battlelog.md @@ -0,0 +1,53 @@ +--- +title: "pvarki/typescript-liveloki-app – README" +--- + +> **Integration tag:** `1.15.2` · **Submodule commit:** `197f6b74507fb81fc442fa1522088f0cb7ed3939` +> **Repo:** git@github.com:pvarki/typescript-liveloki-app.git +> **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/197f6b74507fb81fc442fa1522088f0cb7ed3939 + +# BattleLog + +## Install + +1. Install docker + compose +2. Rename .env_example --> .env and modify as you like. +3. Run docker compose build --no-cache +4. Run docker compose up -d +5. Navigate to localhost:3000 + +## Frontend development + +The frontend is bundled with [Vite](https://vitejs.dev/). + +Once the backend is running (on port 3000), you can navigate to `frontend/`, +run `npm i` and `npm run dev` to run the Vite development server that has +hot reload and all that jazz. + +## Info + +Database is currently preseeded with test data from preseed/preseed.csv + +## Migrations + +To add new migration, locally run: `./node_modules/.bin/node-pg-migrate create ` and modify created file in `/migrations` directory. +Migrations are run (if needed) when docker container starts. `wait-for-it.sh` will ensure that psql container is up and accepting connections before running migrations. + +## JWT testing + +Remove comments from "jwt-test-network" for local jwt testing, to allow joining containers from 2 different compose runs to same docker network. + +## OpenAPI Specification + +https://pvarki.github.io/typescript-liveloki-app/ + +## Running tests + +In directory `tests`, use + +```shell +npm run test:integration +``` + +with the server running in `localhost:3000` as it does by default after `docker compose up -d`. + diff --git a/docs/dev/integrationrepo/v/1.15.2/submodules/cfssl.md b/docs/dev/integrationrepo/v/1.15.2/submodules/cfssl.md new file mode 100644 index 00000000..8612a80b --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/submodules/cfssl.md @@ -0,0 +1,91 @@ +--- +title: "pvarki/docker-rasenmaeher-cfssl – README" +--- + +> **Integration tag:** `1.15.2` · **Submodule commit:** `87f2c2b4b22a0d25d7dcc900d0750373b24fec4b` +> **Repo:** git@github.com:pvarki/docker-rasenmaeher-cfssl.git +> **Browse at this commit:** https://github.com/pvarki/docker-rasenmaeher-cfssl/tree/87f2c2b4b22a0d25d7dcc900d0750373b24fec4b + +# cfssl Submodule + +\![Build +Status\]([https://github.com/pvarki/docker-rasenmaeher-cfssl/actions/workflows/build.yml/badge.svg](https://github.com/pvarki/docker-rasenmaeher-cfssl/actions/workflows/build.yml/badge.svg)) + +## Used as git submodule + +This repo is used as submodule in +[https://github.com/pvarki/docker-rasenmaeher-integration](https://github.com/pvarki/docker-rasenmaeher-integration) it is +probably a good idea to handle all development via it because it has +docker composition for bringin up all the other services rasenmaeher-api +depends on + +## Development + +The cfssl itself we can't do much about but the FastAPI thing uses +poetry and in any case use pre-commit: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t ocsprest:devel_shell . + docker create --name ocsprest_devel -v `pwd`/../":/app" -it `echo $DOCKER_SSHAGENT` ocsprest:devel_shell + docker start -i ocsprest_devel + +Or just pwd if working under separate checkout instead of the +integration repo. + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i ocsprest_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i ocsprest_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" ocsprest:devel_shell -c "pre-commit run --all-files" + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target ocsprest -t ocsprest:amd64-latest . + docker run -it --name ocsprest ocsprest:amd64-latest + +There is also a specific target for just dumping the openapi.json: + + docker build --ssh default --target openapi -t ocsprest:amd64-openapi . + docker run --rm -it --name rasenmaeher_openapijson ocsprest:amd64-openapi + diff --git a/docs/dev/integrationrepo/v/1.15.2/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.15.2/submodules/fpintegration.md new file mode 100644 index 00000000..b3ae5774 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/submodules/fpintegration.md @@ -0,0 +1,106 @@ +--- +title: "pvarki/python-rasenmaeher-rmfpapi – README" +--- + +> **Integration tag:** `1.15.2` · **Submodule commit:** `7b24c8155518a4e11dd46b1f8d210b850973004c` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-rmfpapi.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-rmfpapi/tree/7b24c8155518a4e11dd46b1f8d210b850973004c + +# rmfpapi + +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rmfpapi:devel_shell . + docker create --name rmfpapi_devel -p 4625:4625 -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` rmfpapi:devel_shell + docker start -i rmfpapi_devel + +In the shell you can start the uvicorn devel server with (binding to +0.0.0.0 is important!): + + uvicorn "rmfpapi.app:get_app" --factory --host 0.0.0.0 --port 4625 --reload --log-level debug + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmfpapi_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rmfpapi_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rmfpapi:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rmfpapi:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` rmfpapi:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rmfpapi:latest . + docker run -it --name rmfpapi -p 4625:4625 rmfpapi:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.15.2/submodules/keycloak2.md b/docs/dev/integrationrepo/v/1.15.2/submodules/keycloak2.md new file mode 100644 index 00000000..af7051ac --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/submodules/keycloak2.md @@ -0,0 +1,31 @@ +--- +title: "pvarki/docker-keycloak – README" +--- + +> **Integration tag:** `1.15.2` · **Submodule commit:** `8b2efdf6a4d90fd91d2074617983c086ebe537c5` +> **Repo:** git@github.com:pvarki/docker-keycloak.git +> **Browse at this commit:** https://github.com/pvarki/docker-keycloak/tree/8b2efdf6a4d90fd91d2074617983c086ebe537c5 + +# KeyCloak + +Templates, init container etc to setup keycloak automagically with ENV +variables. + +## Used as git submodule + +This repo is used as submodule in +[https://github.com/pvarki/docker-rasenmaeher-integration](https://github.com/pvarki/docker-rasenmaeher-integration) it is +probably a good idea to handle all development via it because it has +docker composition for bringin up all the other services rasenmaeher-api +depends on + +## pre-commit notes + +Make sure pre-commit is installed: + + pre-commit install --install-hooks + +And it's a good idea to run it regularly before committing: + + pre-commit run --all-files + diff --git a/docs/dev/integrationrepo/v/1.15.2/submodules/kw_product_init.md b/docs/dev/integrationrepo/v/1.15.2/submodules/kw_product_init.md new file mode 100644 index 00000000..185908ab --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/submodules/kw_product_init.md @@ -0,0 +1,78 @@ +--- +title: "pvarki/golang-kraftwerk-init-helper-cli – README" +--- + +> **Integration tag:** `1.15.2` · **Submodule commit:** `0dee1f8f397ba37280f6bf393a3b866d93387e8c` +> **Repo:** git@github.com:pvarki/golang-kraftwerk-init-helper-cli.git +> **Browse at this commit:** https://github.com/pvarki/golang-kraftwerk-init-helper-cli/tree/0dee1f8f397ba37280f6bf393a3b866d93387e8c + +golang-kraftwerk-init-helper-cli =============================== + +Tool for products to create their certificates from +KRAFTWERK manifests. + +# Development + +## Prerequisites + +**Enable** +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/) + +```bash +export DOCKER_BUILDKIT=1 +``` + +**Forward SSH-agent to running instance:** + +**OSX:** + +```bash +export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" +``` + +**Linux:** + +```bash +export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" +``` + +## Create & start development container + +Build the image, create a container and start the container + +```bash +docker build --ssh default --target dev_shell -t kraftwerk_init_helper:dev_shell . +``` + +```bash +docker create --name kraftwerk_init_helper -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` kraftwerk_init_helper:dev_shell +``` + +```bash +docker start -i kraftwerk_init_helper +``` + +## pre-commit initialization + +Once inside the container, run: + +```bash +pre-commit install && pre-commit run --all-files +``` + +That's it, now you have the development environment up & running. + +# Production + +Build the production image: + +```bash +docker build --ssh default --target production -t kraftwerk_init_helper:latest . +``` + +Run the image: + +```bash +docker run --rm -it --name kraftwerk_helper kraftwerk_init_helper:latest +``` + diff --git a/docs/dev/integrationrepo/v/1.15.2/submodules/miniwerk.md b/docs/dev/integrationrepo/v/1.15.2/submodules/miniwerk.md new file mode 100644 index 00000000..53f138db --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/submodules/miniwerk.md @@ -0,0 +1,104 @@ +--- +title: "pvarki/python-rasenmaeher-miniwerk – README" +--- + +> **Integration tag:** `1.15.2` · **Submodule commit:** `073f866ad4b47dc7d1eab0cc06ecaf45c5fc5c4d` +> **Repo:** git@github.com:pvarki/python-rasenmaeher-miniwerk.git +> **Browse at this commit:** https://github.com/pvarki/python-rasenmaeher-miniwerk/tree/073f866ad4b47dc7d1eab0cc06ecaf45c5fc5c4d + +# miniwerk + +Minimal KRAFTWERK amulation to be able to run a RASENMAEHER+products +deployment on any VM + +There are some required ENV configs check out example_env.sh for them +(.env file is also supported) + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t miniwerk:devel_shell . + docker create --name miniwerk_devel -v `pwd`":/app" -p 80:80 -it `echo $DOCKER_SSHAGENT` miniwerk:devel_shell + docker start -i miniwerk_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i miniwerk_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i miniwerk_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" miniwerk:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t miniwerk:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` miniwerk:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t miniwerk:amd64-latest . + docker run -it --name miniwerk miniwerk:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.15.2/submodules/mtxauthz.md b/docs/dev/integrationrepo/v/1.15.2/submodules/mtxauthz.md new file mode 100644 index 00000000..3ba4de6b --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/submodules/mtxauthz.md @@ -0,0 +1,122 @@ +--- +title: "pvarki/python-mediamtx-rmmtxauthz – README" +--- + +> **Integration tag:** `1.15.2` · **Submodule commit:** `55921125444547f67089706f64432e5e3aef7763` +> **Repo:** git@github.com:pvarki/python-mediamtx-rmmtxauthz.git +> **Browse at this commit:** https://github.com/pvarki/python-mediamtx-rmmtxauthz/tree/55921125444547f67089706f64432e5e3aef7763 + +# rmmtxauthz + +Do HTTP based authz for MediaMTX + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rmmtxauthz:devel_shell . + docker create --name rmmtxauthz_devel -v "$(pwd)/rune/output/rune.json:/opt/templates/mediamtx.json" -v "$(pwd):/app" -it $(echo $DOCKER_SSHAGENT) rmmtxauthz:devel_shell + docker start -i rmmtxauthz_devel + +To rebuild the documentation inside the container run: + + rune rune/src json >/opt/templates/mediamtx.json + +Outside of container use: + + rune rune/src json >rune/output/rune.json + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmmtxauthz_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i rmmtxauthz_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v "$(pwd):/app" rmmtxauthz:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t rmmtxauthz:tox . + docker run --rm -it -v "$(pwd):/app" $(echo $DOCKER_SSHAGENT) rmmtxauthz:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t rmmtxauthz:amd64-latest . + docker run -it --name rmmtxauthz rmmtxauthz:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p $(which python3.11) my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + git add poetry.lock + pre-commit install --install-hooks + pre-commit run --all-files + +If you get weird errors about missing packages from pre-commit try +running it with "poetry run pre-commit". + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + +Running "pre-commit run --all-files" and "py.test -v" regularly during +development and especially before committing will save you some +headache. + +### RUNE instructions compile + +tldr: + + rune rune/src json >rune/output/mediamtx.json + diff --git a/docs/dev/integrationrepo/v/1.15.2/submodules/takintegration.md b/docs/dev/integrationrepo/v/1.15.2/submodules/takintegration.md new file mode 100644 index 00000000..a9a43ac1 --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/submodules/takintegration.md @@ -0,0 +1,100 @@ +--- +title: "pvarki/python-tak-rmapi – README" +--- + +> **Integration tag:** `1.15.2` · **Submodule commit:** `0f3021ec2771c0d3846b46a302692f97fa7f2a19` +> **Repo:** git@github.com:pvarki/python-tak-rmapi.git +> **Browse at this commit:** https://github.com/pvarki/python-tak-rmapi/tree/0f3021ec2771c0d3846b46a302692f97fa7f2a19 + +# takrmapi + +RASENMAEHER integration API for TAK server + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t takrmapi:devel_shell . + docker create --name takrmapi_devel -v `pwd`":/app" -it `echo $DOCKER_SSHAGENT` takrmapi:devel_shell + docker start -i takrmapi_devel + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i takrmapi_devel /bin/bash -c "pre-commit install --install-hooks" + docker exec -i takrmapi_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" takrmapi:devel_shell -c "pre-commit run --all-files" + +### Test suite + +You can use the devel shell to run py.test when doing development, for +CI use the "tox" target in the Dockerfile: + + docker build --ssh default --target tox -t takrmapi:tox . + docker run --rm -it -v `pwd`":/app" `echo $DOCKER_SSHAGENT` takrmapi:tox + +### Production docker + +There's a "production" target as well for running the application, +remember to change that architecture tag to arm64 if building on ARM: + + docker build --ssh default --target production -t takrmapi:amd64-latest . + docker run -it --name takrmapi takrmapi:amd64-latest + +## Development + +TLDR: + +- Create and activate a Python 3.11 virtualenv (assuming + virtualenvwrapper): + + mkvirtualenv -p `which python3.11` my_virtualenv + +- change to a branch: + + git checkout -b my_branch + +- install Poetry: [https://python-poetry.org/docs/#installation](https://python-poetry.org/docs/#installation) + +- Install project deps and pre-commit hooks: + + poetry install + pre-commit install --install-hooks + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo, this +is needed because pylint and mypy pre-commit hooks use the "system" +python for now (because reasons). + diff --git a/docs/dev/integrationrepo/v/1.15.2/submodules/takserver.md b/docs/dev/integrationrepo/v/1.15.2/submodules/takserver.md new file mode 100644 index 00000000..e015069c --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/submodules/takserver.md @@ -0,0 +1,68 @@ +--- +title: "pvarki/docker-atak-server – README" +--- + +> **Integration tag:** `1.15.2` · **Submodule commit:** `7a73302fef52d0e4b76b11bbda92afefaab43732` +> **Repo:** git@github.com:pvarki/docker-atak-server.git +> **Browse at this commit:** https://github.com/pvarki/docker-atak-server/tree/7a73302fef52d0e4b76b11bbda92afefaab43732 + +# Run TAK Java server in container + +tldr: + + cp takserver.env.example takserver.env + # edit the env + # export the variables gomplate uses (see the .tpl files) + docker compose pull --include-deps --ignore-pull-failures + docker compose -p tak up -d + +or use docker compose.local.yml without gomplate for local dev +(rebuilding containers): + + export DOCKER_TAG_EXTRA="-dev" + docker build --no-cache --progress=plain -t takserver:latest${DOCKER_TAG_EXTRA} -t takserver:4.7-RELEASE-32${DOCKER_TAG_EXTRA} -t pvarki/takserver:4.7-RELEASE-32${DOCKER_TAG_EXTRA} . + cp takserver.env.example takserver.env + # edit the env + docker compose -f docker-compose.local.yml -p tak up + +Note, for things that live in the volumes (like TAK certs) you must nuke +the volumes to see changes: + + docker compose -f docker-compose.local.yml -p tak down -v ; docker compose -f docker-compose.local.yml -p tak rm -vf + +## Creating client packages locally + +Using the REST API is probably nicer though + +Create client package: + + docker compose -p tak exec takserver_api /bin/bash -c 'CLIENT_CERT_NAME=replaceme /opt/scripts/make_client_zip.sh' + +Then get /opt/tak/certs/files/clientpkgs/replaceme.zip out of the +container: + + docker compose -p tak exec taktakserver_apiserver /bin/bash -c 'base64 /opt/tak/certs/files/clientpkgs/replaceme.zip' | base64 -id >replaceme.zip + +This approach also works for recovering the admin cert +(/opt/tak/certs/files/admin.p12 unless you changed the ADMIN_CERT_NAME +ENV) + +## Creating new admin users + +Create the user on the takserver container: + + docker compose -p tak exec takserver_api /bin/bash -c 'cd /opt/tak/data/certs/ && CAPASS=$CA_PASS PASS=replaceme_user_cert_pass ./makeCert.sh client replaceme_username && ADMIN_CERT_NAME=replaceme_username /opt/scripts/enable_admin.sh' + +See above about the hard way of getting the cert file, or use the REST +API. + +## Gradle builds + +Build the distribution: + + mkdir outputs + docker build --progress=plain -f Dockerfile_build --target files -t atakbuild:files . + docker run --rm -it -v `pwd`/outputs:/output atakbuild:files + +Now you have the build artefacts in outputs -directory. + diff --git a/docs/dev/integrationrepo/v/1.15.2/submodules/ui.md b/docs/dev/integrationrepo/v/1.15.2/submodules/ui.md new file mode 100644 index 00000000..0582f81e --- /dev/null +++ b/docs/dev/integrationrepo/v/1.15.2/submodules/ui.md @@ -0,0 +1,116 @@ +--- +title: "pvarki/rasenmaeher-ui – README" +--- + +> **Integration tag:** `1.15.2` · **Submodule commit:** `eb2d303774fa7791ba5410080cd55316fe8b32c1` +> **Repo:** git@github.com:pvarki/rasenmaeher-ui.git +> **Browse at this commit:** https://github.com/pvarki/rasenmaeher-ui/tree/eb2d303774fa7791ba5410080cd55316fe8b32c1 + +# UI For RASENMAEHER + +React, TS etc + +## Docker + +For more controlled deployments and to get rid of "works on my computer" +-syndrome, we always make sure our software works under docker. + +It's also a quick way to get started with a standard development +environment. + +### SSH agent forwarding + +We need +[buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/): + + export DOCKER_BUILDKIT=1 + +And also the exact way for forwarding agent to running instance is +different on OSX: + + export DOCKER_SSHAGENT="-v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock" + +and Linux: + + export DOCKER_SSHAGENT="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + +### Creating a development container + +Build image, create container and start it: + + docker build --ssh default --target devel_shell -t rasenmaeher_ui:devel_shell . + docker create --name rmui_devel -v `pwd`":/app" -p 8002:8002 -it `echo $DOCKER_SSHAGENT` rasenmaeher_ui:devel_shell + docker start -i rmui_devel + +Note: due to the volume mapping if you already had run "npm install" on +the host you need to delete the node_modules directory and in any case +you need to run "npm install" inside the container. + +### pre-commit considerations + +If working in Docker instead of native env you need to run the +pre-commit checks in docker too: + + docker exec -i rmui_devel /bin/bash -c "pre-commit install" + docker exec -i rmui_devel /bin/bash -c "pre-commit run --all-files" + +You need to have the container running, see above. Or alternatively use +the docker run syntax but using the running container is faster: + + docker run --rm -it -v `pwd`":/app" rasenmaeher_ui:devel_shell -c "pre-commit run --all-files" + +### Production docker + +This will just deliver the compiled files to a mounted directory + +> docker build --ssh default --target production -t +> rasenmaeher_ui:amd64-latest . docker run -it --rm -v +> pwd/rmui_files:/deliver +> rasenmaeher_ui:amd64-latest + +Then you need to serve things so that index.html is the default +controller for things, nginx example: + + location / { + index index.html; + root /rmui_files; + try_files $uri $uri/ /index.html =404; + } + +## Development + +TLDR: + +- Create and activate a Node 18 virtualenv: + + FIXME: Insert instructions + +- change to a branch: + + git checkout -b my_branch + +- Install project deps and pre-commit hooks: + + npm install + pre-commit install + pre-commit run --all-files + +- Ready to go. + +Remember to activate your virtualenv whenever working on the repo. + +### Asset set + +Asset set is loaded according to VITE_ASSET_SET enviroment variable. +Changing asset set requires reloading vite. Folder assetSetStore +contains files for for different sets that are copied to src/assets/set/ +folder on startup. Current set is tracked with setName.txt file. + +When you want to use dynamic assets, you want to make sure your dynamic +asset has equivalent for sets (both "fdf" and "neutral", so on). If the +assets are images, you want to then import them to your components from +/assets/set/\* where the env-defined assets are placed by VITE on +startup. If the assets are translation keys, make sure keys are present +in locales under assetSetStore, and then use the "dynamic" namespace for +accessing them. + diff --git a/docs/dev/integrationrepo/v/1.5.0/_category_.json b/docs/dev/integrationrepo/v/1.5.0/_category_.json index 8090d441..ba22ed80 100644 --- a/docs/dev/integrationrepo/v/1.5.0/_category_.json +++ b/docs/dev/integrationrepo/v/1.5.0/_category_.json @@ -1,4 +1,4 @@ { "label": "1.5.0", - "position": 14 + "position": 20 } diff --git a/docs/dev/integrationrepo/v/1.5.0/submodules/api.md b/docs/dev/integrationrepo/v/1.5.0/submodules/api.md index 038dd890..1e3eaef3 100644 --- a/docs/dev/integrationrepo/v/1.5.0/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.5.0/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.5.0/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.5.0/submodules/fpintegration.md index d7330a91..42d0fc29 100644 --- a/docs/dev/integrationrepo/v/1.5.0/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.5.0/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.5.1/_category_.json b/docs/dev/integrationrepo/v/1.5.1/_category_.json index f6f426ce..19e8e9de 100644 --- a/docs/dev/integrationrepo/v/1.5.1/_category_.json +++ b/docs/dev/integrationrepo/v/1.5.1/_category_.json @@ -1,4 +1,4 @@ { "label": "1.5.1", - "position": 13 + "position": 19 } diff --git a/docs/dev/integrationrepo/v/1.5.1/submodules/api.md b/docs/dev/integrationrepo/v/1.5.1/submodules/api.md index 8abe3d68..810920cd 100644 --- a/docs/dev/integrationrepo/v/1.5.1/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.5.1/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.5.1/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.5.1/submodules/fpintegration.md index 287f7791..6e99df19 100644 --- a/docs/dev/integrationrepo/v/1.5.1/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.5.1/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.6.0/_category_.json b/docs/dev/integrationrepo/v/1.6.0/_category_.json index a20e6114..aa8cb1ff 100644 --- a/docs/dev/integrationrepo/v/1.6.0/_category_.json +++ b/docs/dev/integrationrepo/v/1.6.0/_category_.json @@ -1,4 +1,4 @@ { "label": "1.6.0", - "position": 12 + "position": 18 } diff --git a/docs/dev/integrationrepo/v/1.6.0/submodules/api.md b/docs/dev/integrationrepo/v/1.6.0/submodules/api.md index fbb732d5..e3b8dbfe 100644 --- a/docs/dev/integrationrepo/v/1.6.0/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.6.0/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.6.0/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.6.0/submodules/fpintegration.md index 49a196e5..4441e16e 100644 --- a/docs/dev/integrationrepo/v/1.6.0/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.6.0/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.7.0/_category_.json b/docs/dev/integrationrepo/v/1.7.0/_category_.json index fbbb8e03..b0aa4d90 100644 --- a/docs/dev/integrationrepo/v/1.7.0/_category_.json +++ b/docs/dev/integrationrepo/v/1.7.0/_category_.json @@ -1,4 +1,4 @@ { "label": "1.7.0", - "position": 11 + "position": 17 } diff --git a/docs/dev/integrationrepo/v/1.7.0/submodules/api.md b/docs/dev/integrationrepo/v/1.7.0/submodules/api.md index 2926caac..6b07c23e 100644 --- a/docs/dev/integrationrepo/v/1.7.0/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.7.0/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.7.0/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.7.0/submodules/fpintegration.md index 24d81240..e07eb2d8 100644 --- a/docs/dev/integrationrepo/v/1.7.0/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.7.0/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.7.1/_category_.json b/docs/dev/integrationrepo/v/1.7.1/_category_.json index e78cd59e..d0d74f8f 100644 --- a/docs/dev/integrationrepo/v/1.7.1/_category_.json +++ b/docs/dev/integrationrepo/v/1.7.1/_category_.json @@ -1,4 +1,4 @@ { "label": "1.7.1", - "position": 10 + "position": 16 } diff --git a/docs/dev/integrationrepo/v/1.7.1/submodules/api.md b/docs/dev/integrationrepo/v/1.7.1/submodules/api.md index 5bae57b8..3d380e3f 100644 --- a/docs/dev/integrationrepo/v/1.7.1/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.7.1/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.7.1/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.7.1/submodules/fpintegration.md index a312275f..32f4aa52 100644 --- a/docs/dev/integrationrepo/v/1.7.1/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.7.1/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.7.2/_category_.json b/docs/dev/integrationrepo/v/1.7.2/_category_.json index 699334d4..e96ce61f 100644 --- a/docs/dev/integrationrepo/v/1.7.2/_category_.json +++ b/docs/dev/integrationrepo/v/1.7.2/_category_.json @@ -1,4 +1,4 @@ { "label": "1.7.2", - "position": 9 + "position": 15 } diff --git a/docs/dev/integrationrepo/v/1.7.2/submodules/api.md b/docs/dev/integrationrepo/v/1.7.2/submodules/api.md index f297845d..625ae6ab 100644 --- a/docs/dev/integrationrepo/v/1.7.2/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.7.2/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.7.2/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.7.2/submodules/fpintegration.md index 0e3fb185..77cf3abb 100644 --- a/docs/dev/integrationrepo/v/1.7.2/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.7.2/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.7.3/_category_.json b/docs/dev/integrationrepo/v/1.7.3/_category_.json index dcddfcf0..e980a15f 100644 --- a/docs/dev/integrationrepo/v/1.7.3/_category_.json +++ b/docs/dev/integrationrepo/v/1.7.3/_category_.json @@ -1,4 +1,4 @@ { "label": "1.7.3", - "position": 8 + "position": 14 } diff --git a/docs/dev/integrationrepo/v/1.7.3/submodules/api.md b/docs/dev/integrationrepo/v/1.7.3/submodules/api.md index 8df13412..abe5b25c 100644 --- a/docs/dev/integrationrepo/v/1.7.3/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.7.3/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.7.3/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.7.3/submodules/fpintegration.md index a2871f89..63c9fd77 100644 --- a/docs/dev/integrationrepo/v/1.7.3/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.7.3/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.8.0/_category_.json b/docs/dev/integrationrepo/v/1.8.0/_category_.json index daa99bfd..052a9214 100644 --- a/docs/dev/integrationrepo/v/1.8.0/_category_.json +++ b/docs/dev/integrationrepo/v/1.8.0/_category_.json @@ -1,4 +1,4 @@ { "label": "1.8.0", - "position": 7 + "position": 13 } diff --git a/docs/dev/integrationrepo/v/1.8.0/submodules/api.md b/docs/dev/integrationrepo/v/1.8.0/submodules/api.md index f5850157..1c7ddd9f 100644 --- a/docs/dev/integrationrepo/v/1.8.0/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.8.0/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.8.0/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.8.0/submodules/battlelog.md index e2cc680c..ff2d3f86 100644 --- a/docs/dev/integrationrepo/v/1.8.0/submodules/battlelog.md +++ b/docs/dev/integrationrepo/v/1.8.0/submodules/battlelog.md @@ -6,7 +6,7 @@ title: "pvarki/typescript-liveloki-app – README" > **Repo:** git@github.com:pvarki/typescript-liveloki-app.git > **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/0c198ea663a37553df88fe08b80b008f5dba95e2 -# livelogi +# BattleLog ## Install diff --git a/docs/dev/integrationrepo/v/1.8.0/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.8.0/submodules/fpintegration.md index 57d6e197..0e68a543 100644 --- a/docs/dev/integrationrepo/v/1.8.0/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.8.0/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/integrationrepo/v/1.8.1/_category_.json b/docs/dev/integrationrepo/v/1.8.1/_category_.json index d9c23d86..500f42d7 100644 --- a/docs/dev/integrationrepo/v/1.8.1/_category_.json +++ b/docs/dev/integrationrepo/v/1.8.1/_category_.json @@ -1,4 +1,4 @@ { "label": "1.8.1", - "position": 6 + "position": 12 } diff --git a/docs/dev/integrationrepo/v/1.8.1/submodules/api.md b/docs/dev/integrationrepo/v/1.8.1/submodules/api.md index 279b603b..55fb4a35 100644 --- a/docs/dev/integrationrepo/v/1.8.1/submodules/api.md +++ b/docs/dev/integrationrepo/v/1.8.1/submodules/api.md @@ -215,3 +215,26 @@ Remember to activate your virtualenv whenever working on the repo, this is needed because pylint and mypy pre-commit hooks use the "system" python for now (because reasons). +## Pipeline public-key for JWT verification + +Pipeline can pass a public-key whose private key is used to sign JWTs to +access RM API with environment variable +\$EXTERNAL_JWT_PUBKEY_B64. The public key +should be base64-encoded to avoid issues with newlines and such when the +key is passed through the system. + +Export a previously generated key to the container: + + export EXTERNAL_JWT_PUBKEY_B64=$(cat ~/p/jwt-test/private_key.key | base64) + rmlocal up --build + +Generate a token and set it in the environment +TOKEN. The JWT should have the claim +anon_admin_session set to +true. Use the token in the API call: + + curl -X POST --insecure https://localmaeher.dev.pvarki.fi:4439/\ + api/v1/token/code/generate -H 'Content-type: application/json' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"claims": {"anon_admin_session": true}}' + diff --git a/docs/dev/integrationrepo/v/1.8.1/submodules/battlelog.md b/docs/dev/integrationrepo/v/1.8.1/submodules/battlelog.md index befb5f5d..ae8c032d 100644 --- a/docs/dev/integrationrepo/v/1.8.1/submodules/battlelog.md +++ b/docs/dev/integrationrepo/v/1.8.1/submodules/battlelog.md @@ -6,7 +6,7 @@ title: "pvarki/typescript-liveloki-app – README" > **Repo:** git@github.com:pvarki/typescript-liveloki-app.git > **Browse at this commit:** https://github.com/pvarki/typescript-liveloki-app/tree/7674ef0da0ab295837fea723d9f9d0a8a02637f0 -# livelogi +# BattleLog ## Install diff --git a/docs/dev/integrationrepo/v/1.8.1/submodules/fpintegration.md b/docs/dev/integrationrepo/v/1.8.1/submodules/fpintegration.md index fb19cf10..75e39f15 100644 --- a/docs/dev/integrationrepo/v/1.8.1/submodules/fpintegration.md +++ b/docs/dev/integrationrepo/v/1.8.1/submodules/fpintegration.md @@ -8,7 +8,8 @@ title: "pvarki/python-rasenmaeher-rmfpapi – README" # rmfpapi -Fake product RASENMAEHER integration API service +Fake product RASENMAEHER integration API service. Serves as a reference +implementation for a new integration into the deploy app ecosystem. ## Docker @@ -80,10 +81,10 @@ remember to change that architecture tag to arm64 if building on ARM: TLDR: -- Create and activate a Python 3.8 virtualenv (assuming +- Create and activate a Python 3.11 virtualenv (assuming virtualenvwrapper): - mkvirtualenv -p `which python3.8` my_virtualenv + mkvirtualenv -p `which python3.11` my_virtualenv - change to a branch: diff --git a/docs/dev/roadmap/03-tak.md b/docs/dev/roadmap/03-tak.md index 7d9aca1a..f1b47625 100644 --- a/docs/dev/roadmap/03-tak.md +++ b/docs/dev/roadmap/03-tak.md @@ -7,11 +7,11 @@ title: "03-TAK" ### Core features - [ ] **Docs: Write TAK Basic User Guides to the New Docs platform** - Get rid of user guides written into Deploy App UI, instead do them in Docs that is shipped as static site. This simplifies developing Guides a lot, so that you don't have to ship them via Integration API and you can show them in the Docs Platform easily. -- [ ] **Docs: Community Wiki:** Simple to use community wiki to write TAK usage documentation with as low bar as possible. We can polish docs later in order to deploy them to the New Docs platform, that shouldn't block actually writing them at first place. +- [x] **Docs: Community Wiki:** Simple to use community wiki to write TAK usage documentation with as low bar as possible. We can polish docs later in order to deploy them to the New Docs platform, that shouldn't block actually writing them at first place. ### New Features -- [ ] **Docs: TAK Usage Model guides** - While we don't think there is one correct way to use TAK, we want to develop a good basic usage model so that user groups could, upon their will, employ that without having their users extensively trained (as they can read our stellar Usage Model docs). +- [x] **Docs: TAK Usage Model guides** - While we don't think there is one correct way to use TAK, we want to develop a good basic usage model so that user groups could, upon their will, employ that without having their users extensively trained (as they can read our stellar Usage Model docs). - [ ] **Docs: GeoChat usage guide -** While lacking, with a good guide GeoChat pretty easy to use and works for basic use cases. - [ ] **UAS Tool: Docs** & Integration: As soon as Deploy App supports MediaMTX well, write usage docs & ensure smooth integration with the TAK UAS Tool. This will enable us to maximum the value off drones as their video & location can be easily shared within TAK. diff --git a/docs/dev/contributing/04-deploy-app-release-process/1-submodule-development-process.md b/docs/dev/specs/01-deploy-app/deploy-app-release-process/1-submodule-development-process.md similarity index 100% rename from docs/dev/contributing/04-deploy-app-release-process/1-submodule-development-process.md rename to docs/dev/specs/01-deploy-app/deploy-app-release-process/1-submodule-development-process.md diff --git a/docs/dev/contributing/04-deploy-app-release-process/2-submodule-merge-process.md b/docs/dev/specs/01-deploy-app/deploy-app-release-process/2-submodule-merge-process.md similarity index 100% rename from docs/dev/contributing/04-deploy-app-release-process/2-submodule-merge-process.md rename to docs/dev/specs/01-deploy-app/deploy-app-release-process/2-submodule-merge-process.md diff --git a/docs/dev/contributing/04-deploy-app-release-process/3-integration-repo-merge-process.md b/docs/dev/specs/01-deploy-app/deploy-app-release-process/3-integration-repo-merge-process.md similarity index 100% rename from docs/dev/contributing/04-deploy-app-release-process/3-integration-repo-merge-process.md rename to docs/dev/specs/01-deploy-app/deploy-app-release-process/3-integration-repo-merge-process.md diff --git a/docs/dev/specs/01-deploy-app/deploy-app-release-process/_category_.json b/docs/dev/specs/01-deploy-app/deploy-app-release-process/_category_.json new file mode 100644 index 00000000..bd933adb --- /dev/null +++ b/docs/dev/specs/01-deploy-app/deploy-app-release-process/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Deploy App Release Process", + "collapsed": true +} diff --git a/docs/dev/contributing/04-deploy-app-release-process/index.md b/docs/dev/specs/01-deploy-app/deploy-app-release-process/index.md similarity index 89% rename from docs/dev/contributing/04-deploy-app-release-process/index.md rename to docs/dev/specs/01-deploy-app/deploy-app-release-process/index.md index 500a7069..1e3e84a1 100644 --- a/docs/dev/contributing/04-deploy-app-release-process/index.md +++ b/docs/dev/specs/01-deploy-app/deploy-app-release-process/index.md @@ -1,5 +1,5 @@ --- -title: "04-Deploy App Release Process" +title: "Deploy App Release Process" --- # Version Release Guide