Skip to content

Commit e87badf

Browse files
MashBMashB
authored
Merge pull request #25 from TaskarCenterAtUW/stage
Stage to Prod upmerge
2 parents ce0b5a8 + d9e2fd1 commit e87badf

39 files changed

+1935
-2
lines changed

‎.github/workflows/dev_workflow.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
---
2+
######### Dev Workflow ########
3+
on:
4+
pull_request:
5+
branches: [dev]
6+
types:
7+
- closed
8+
workflow_dispatch:
9+
permissions:
10+
id-token: write
11+
contents: read
12+
13+
jobs:
14+
Build:
15+
environment: Development
16+
runs-on: ubuntu-latest
17+
if: github.event.pull_request.merged == true
18+
steps:
19+
- uses: actions/checkout@v2
20+
- uses: azure/docker-login@v1
21+
with:
22+
login-server: ${{ secrets.REGISTRY_DOMAIN }}
23+
username: ${{ secrets.REGISTRY_USERNAME }}
24+
password: ${{ secrets.REGISTRY_PASSWORD }}
25+
- name: Publish image to Azure Registry
26+
run: |
27+
docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'master' && 'prod' || github.ref_name }}${{ github.ref_name != 'master' && '-latest' || 'latest' }} .
28+
docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags
29+
Deploy:
30+
needs: Build
31+
environment:
32+
name: Development
33+
runs-on: ubuntu-latest
34+
steps:
35+
- name: Login to Azure
36+
uses: azure/login@v2.0.0
37+
with:
38+
creds: ${{secrets.TDEI_CORE_AZURE_CREDS}}
39+
- name: Deploy to Dev
40+
uses: azure/webapps-deploy@v2
41+
with:
42+
app-name: ${{ secrets.AZURE_WEBAPP_NAME }}
43+
images: ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}

‎.github/workflows/prod_workflow.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
---
2+
######### Prod Workflow ########
3+
on:
4+
pull_request:
5+
branches: [main]
6+
types:
7+
- closed
8+
workflow_dispatch:
9+
permissions:
10+
id-token: write
11+
contents: read
12+
13+
jobs:
14+
Build:
15+
environment: Production
16+
runs-on: ubuntu-latest
17+
if: github.event.pull_request.merged == true
18+
steps:
19+
- uses: actions/checkout@v2
20+
- uses: azure/docker-login@v1
21+
with:
22+
login-server: ${{ secrets.REGISTRY_DOMAIN }}
23+
username: ${{ secrets.REGISTRY_USERNAME }}
24+
password: ${{ secrets.REGISTRY_PASSWORD }}
25+
- name: Publish image to Azure Registry
26+
run: |
27+
docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'master' && 'prod' || github.ref_name }}${{ github.ref_name != 'master' && '-latest' || 'latest' }} .
28+
docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags
29+
Deploy:
30+
needs: Build
31+
environment:
32+
name: Production
33+
runs-on: ubuntu-latest
34+
steps:
35+
- name: Login to Azure
36+
uses: azure/login@v2.0.0
37+
with:
38+
creds: ${{secrets.TDEI_CORE_AZURE_CREDS}}
39+
- name: Deploy to Production
40+
uses: azure/webapps-deploy@v2
41+
with:
42+
app-name: ${{ secrets.AZURE_WEBAPP_NAME }}
43+
images: ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}.

‎.github/workflows/stage_workflow.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
---
2+
######### Stage Workflow ########
3+
on:
4+
pull_request:
5+
branches: [stage]
6+
types:
7+
- closed
8+
workflow_dispatch:
9+
permissions:
10+
id-token: write
11+
contents: read
12+
13+
jobs:
14+
Build:
15+
environment: Stage
16+
runs-on: ubuntu-latest
17+
if: github.event.pull_request.merged == true
18+
steps:
19+
- uses: actions/checkout@v2
20+
- uses: azure/docker-login@v1
21+
with:
22+
login-server: ${{ secrets.REGISTRY_DOMAIN }}
23+
username: ${{ secrets.REGISTRY_USERNAME }}
24+
password: ${{ secrets.REGISTRY_PASSWORD }}
25+
- name: Publish image to Azure Registry
26+
run: |
27+
docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'master' && 'prod' || github.ref_name }}${{ github.ref_name != 'master' && '-latest' || 'latest' }} .
28+
docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags
29+
Deploy:
30+
needs: Build
31+
environment:
32+
name: Stage
33+
runs-on: ubuntu-latest
34+
steps:
35+
- name: Login to Azure
36+
uses: azure/login@v2.0.0
37+
with:
38+
creds: ${{secrets.TDEI_CORE_AZURE_CREDS}}
39+
- name: Deploy to Stage
40+
uses: azure/webapps-deploy@v2
41+
with:
42+
app-name: ${{ secrets.AZURE_WEBAPP_NAME }}
43+
images: ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}.

‎.github/workflows/unit_tests.yaml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
---
2+
name: Unit Tests
3+
4+
#############################
5+
# Start the job on all push #
6+
#############################
7+
on:
8+
push:
9+
branches-ignore:
10+
- '**'
11+
# Remove the line above to run when pushing to master
12+
pull_request:
13+
branches: [master, dev, stage]
14+
15+
###############
16+
# Set the Job #
17+
###############
18+
jobs:
19+
UnitTest:
20+
name: Unit Test Cases
21+
# Set the agent to run on
22+
runs-on: ubuntu-latest
23+
steps:
24+
- name: Checkout code
25+
uses: actions/checkout@v3
26+
27+
- name: Set up Python
28+
uses: actions/setup-python@v4
29+
with:
30+
python-version: "3.10" # Use the appropriate Python version
31+
32+
- name: Install dependencies
33+
run: |
34+
pip install -r requirements.txt
35+
36+
- name: Run unit tests
37+
run: |
38+
python test_report.py
39+
coverage run --source=src -m unittest discover -s tests/
40+
coverage report -m
41+
exit_status=$?
42+
43+
# Set the exit status as an output for later use
44+
echo "::set-output name=exit_status::$exit_status"
45+
46+
- name: Archive Coverage Report
47+
if: ${{ always() }} # Upload the coverage report even if tests fail
48+
uses: actions/upload-artifact@v2
49+
with:
50+
name: htmlcov
51+
path: htmlcov

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,4 +157,5 @@ cython_debug/
157157
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158158
# and can be added to the global gitignore or merged into this file. For a more nuclear
159159
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
160-
#.idea/
160+
.idea/
161+
reports/

‎Dockerfile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM python:3.10
2+
WORKDIR /code
3+
COPY ./requirements.txt /code/requirements.txt
4+
RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
5+
COPY ./src /code/src
6+
EXPOSE 8080
7+
CMD ["uvicorn", "src.main:app", "--reload", "--host", "0.0.0.0", "--port", "8080"]

‎README.md‎

Lines changed: 150 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,151 @@
11
# TDEI-python-osw-validation
2-
Python service to Validate the OSW file that is uploaded.
2+
3+
## Introduction
4+
Service to Validate the OSW files that is uploaded. At the moment, the service does the following:
5+
- Listens to the topic which is mentioned in `.env` file for any new message (that is triggered when a file is uploaded), example `UPLOAD_TOPIC=osw-upload`
6+
- Consumes the message and perform following checks -
7+
- Download the file locally
8+
- File location is in the message `data.meta.file_upload_path`
9+
- Uses `python-osw-validation` to validate the file
10+
- Adds the `isValid` and `validationMessage` keys to the original message
11+
- Publishes the result to the topic mentioned in `.env` file, example `VALIDATION_TOPIC=osw-validation`
12+
13+
## Getting Started
14+
The project is built on Python with FastAPI framework. All the regular nuances for a Python project are valid for this.
15+
16+
### System requirements
17+
| Software | Version |
18+
|------------|---------|
19+
| Python | 3.10.x |
20+
21+
22+
### Connectivity to cloud
23+
- Connecting this to cloud will need the following in the `.env` file
24+
25+
```bash
26+
QUEUECONNECTION=xxxx
27+
STORAGECONNECTION=xxxx
28+
VALIDATION_REQ_TOPIC=xxxx
29+
VALIDATION_REQ_SUB=xxxx
30+
VALIDATION_RES_TOPIC=xxxx
31+
CONTAINER_NAME=xxxx
32+
AUTH_PERMISSION_URL=xxx
33+
34+
```
35+
36+
The application connect with the `STORAGECONNECTION` string provided in `.env` file and validates downloaded zipfile using `python-osw-validation` package.
37+
`QUEUECONNECTION` is used to send out the messages and listen to messages.
38+
39+
40+
### How to Set up and Build
41+
Follow the steps to install the python packages required for both building and running the application
42+
43+
1. Setup virtual environment
44+
```
45+
python3.10 -m venv .venv
46+
source .venv/bin/activate
47+
```
48+
49+
2. Install the dependencies. Run the following command in terminal on the same directory as `requirements.txt`
50+
```
51+
# Installing requirements
52+
pip install -r requirements.txt
53+
```
54+
### How to Run the Server/APIs
55+
56+
1. The http server by default starts with `8000` port
57+
2. Run server
58+
```
59+
uvicorn src.main:app --reload
60+
```
61+
3. By default `get` call on `localhost:8000/health` gives a sample response
62+
4. Other routes include a `ping` with get and post. Make `get` or `post` request to `http://localhost:8000/health/ping`
63+
5. Once the server starts, it will start to listening the subscriber(`VALIDATION_REQ_SUB` should be in env file)
64+
65+
66+
#### Request Format
67+
68+
```json
69+
{
70+
"messageId": "tdei_record_id",
71+
"messageType": "workflow_identifier",
72+
"data": {
73+
"file_upload_path": "file_upload_path",
74+
"user_id": "user_id",
75+
"tdei_project_group_id": "tdei_project_group_id"
76+
}
77+
}
78+
```
79+
80+
#### Response Format
81+
82+
```json
83+
{
84+
"messageId": "tdei_record_id",
85+
"messageType": "workflow_identifier",
86+
"data": {
87+
"file_upload_path": "file_upload_path",
88+
"user_id": "user_id",
89+
"tdei_project_group_id": "tdei_project_group_id",
90+
"success": true/false,
91+
"message": "message" // if false the error string else empty string
92+
},
93+
"publishedDate": "published date"
94+
}
95+
```
96+
97+
98+
### How to Set up and run the Tests
99+
100+
Make sure you have set up the project properly before running the tests, see above for `How to Setup and Build`.
101+
102+
#### How to run test harness
103+
1. Add the new set of test inside `tests/test_harness/tests.json` file like -
104+
```
105+
{
106+
"Name": "Test Name",
107+
"Input_file": "test_files/osw_test_case1.json", // Input file path which you want to provide to the test
108+
"Result": true/false // Defining the test output
109+
}
110+
```
111+
2. Test Harness would require a valid `.env` file.
112+
3. To run the test harness `python tests/test_harness/run_tests.py`
113+
#### How to run unit test cases
114+
1. `.env` file is not required for Unit test cases.
115+
2. To run the unit test cases
116+
1. `python test_report.py`
117+
2. Above command will run all test cases and generate the html report, in `reports` folder at the root level.
118+
3. To run the coverage
119+
1. `python -m coverage run --source=src -m unittest discover -s tests/unit_tests`
120+
2. Above command will run all the unit test cases.
121+
3. To generate the coverage report in console
122+
1. `coverage report`
123+
2. Above command will generate the code coverage report in terminal.
124+
4. To generate the coverage report in html.
125+
1. `coverage html`
126+
2. Above command will generate the html report, and generated html would be in `htmlcov` directory at the root level.
127+
5. _NOTE :_ To run the `html` or `report` coverage, 3.i) command is mandatory
128+
129+
#### How to run integration test cases
130+
1. `.env` file is required for Unit test cases.
131+
2. To run the integration test cases, run the below command
132+
1. `python test_integration.py`
133+
2. Above command will run all integration test cases and generate the html report, in `reports` folder at the root level.
134+
135+
136+
### Messaging
137+
138+
This microservice deals with two topics/queues.
139+
- upload queue from osw-upload
140+
- validation queue from osw-validation
141+
142+
143+
#### Incoming
144+
The incoming messages will be from the upload queue `osw-upload`.
145+
The format is mentioned in [osw-upload.json](./src/assets/osw-upload.json)
146+
147+
#### Outgoing
148+
The outgoing messages will be to the `osw-validation` topic.
149+
The format of the message is at [osw-validation.json](./src/assets/osw-validation.json)
150+
151+

‎requirements.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
psutil==5.9.5
2+
fastapi==0.88.0
3+
python-dotenv==0.21.0
4+
pydantic==1.10.4
5+
python-ms-core==0.0.18
6+
uvicorn==0.20.0
7+
coverage==7.2.7
8+
html_testRunner==1.2.1
9+
httpx==0.24.1
10+
python-osw-validation==0.2.3

‎src/assets/osw-upload.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"messageId": "c8c76e89f30944d2b2abd2491bd95337",
3+
"messageType": "workflow_identifier",
4+
"data": {
5+
"file_upload_path": "https://tdeisamplestorage.blob.core.windows.net/osw/test_upload/valid.zip",
6+
"user_id": "c59d29b6-a063-4249-943f-d320d15ac9ab",
7+
"tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb"
8+
}
9+
}

‎src/assets/osw-validation.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"messageId": "c8c76e89f30944d2b2abd2491bd95337",
3+
"messageType": "workflow_identifier",
4+
"data": {
5+
"file_upload_path": "https://tdeisamplestorage.blob.core.windows.net/osw/test_upload/valid.zip",
6+
"user_id": "c59d29b6-a063-4249-943f-d320d15ac9ab",
7+
"tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb",
8+
"success": true,
9+
"message": ""
10+
}
11+
}

0 commit comments

Comments
 (0)