From 622bf9e84cc9cb42b5661f057ec8c93a286374dd Mon Sep 17 00:00:00 2001 From: gckang <76260739+gckang@users.noreply.github.com> Date: Mon, 17 Feb 2025 17:34:22 -0500 Subject: [PATCH 01/32] add deploy using aws into cicd --- .github/workflows/continuous-integration.yml | 57 ++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 925390dca..d4780ccbb 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -30,6 +30,18 @@ env: REVIEW_DRIVE_ID: ${{ secrets.REVIEW_DRIVE_ID }} REVIEW_DRIVE_EMAIL: ${{ secrets.REVIEW_DRIVE_EMAIL }} REVIEW_DRIVE_PASSWORD: ${{ secrets.REVIEW_DRIVE_PASSWORD }} + # aws ecs image + IMAGE_NAME: latest + AWS_REGION: us-east-1f + ECS_SERVICE: barrett-fogle-love-v1 + ECS_CLUSTER: tcf-fargate-cluster + ECS_TASK_DEFINITION: tcf-prod-task:14 + CONTAINER_IMAGE: tcf/thecourseforum2:latest # so idk if this is right but it should be the url to the docker image, used the repo name but the image in there is called "latest"? + CONTAINER_NAME: latest + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} jobs: pylint: @@ -108,3 +120,48 @@ jobs: - name: Run ESLint run: npx eslint -c .config/.eslintrc.yml tcf_website/static/ + + deploy: + name: Deploy + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ secrets.DOCKER_HUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + - name: Fill in the new image ID in the Amazon ECS task definition + id: task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: ${{ env.ECS_TASK_DEFINITION }} + container-name: ${{ env.CONTAINER_NAME }} + image: ${{ env.CONTAINER_IMAGE }} + - name: Deploy Amazon ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + with: + task-definition: ${{ steps.task-def.outputs.task-definition }} + service: ${{ env.ECS_SERVICE }} + cluster: ${{ env.ECS_CLUSTER }} + wait-for-service-stability: true + From 2adbbefc2a018359e9c93a197b82fa8bc0ce7358 Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Sun, 19 Jan 2025 14:28:13 -0500 Subject: [PATCH 02/32] feat(settings): load balancer configs --- tcf_core/settings/base.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tcf_core/settings/base.py b/tcf_core/settings/base.py index 753d3c134..f6c875996 100644 --- a/tcf_core/settings/base.py +++ b/tcf_core/settings/base.py @@ -23,7 +23,14 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = env.bool("DEBUG") # default value set on the top -ALLOWED_HOSTS = ["localhost", ".ngrok.io", "127.0.0.1"] +ALLOWED_HOSTS = [ + "localhost", + ".ngrok.io", + "127.0.0.1", + "tcf-load-balancer-1374896025.us-east-1.elb.amazonaws.com", +] + +CORS_ALLOWED_ORIGINS = ["https://thecourseforum.com", "https://thecourseforumtest.com"] # Application definition @@ -158,7 +165,9 @@ WHITELISTED_DOMAINS = ["virginia.edu"] -SOCIAL_AUTH_GOOGLE_OAUTH2_LOGIN_URL = reverse_lazy("social:begin", args=["google-oauth2"]) +SOCIAL_AUTH_GOOGLE_OAUTH2_LOGIN_URL = reverse_lazy( + "social:begin", args=["google-oauth2"] +) SOCIAL_AUTH_RAISE_EXCEPTIONS = False SOCIAL_AUTH_PIPELINE = ( "tcf_core.auth_pipeline.password_validation", From 2c1edab2d62698053a7e15c0da3ace30066ae179 Mon Sep 17 00:00:00 2001 From: gckang <76260739+gckang@users.noreply.github.com> Date: Sun, 2 Mar 2025 15:15:13 -0500 Subject: [PATCH 03/32] remove dockerhub stuff from cicd --- .github/workflows/continuous-integration.yml | 39 +++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index d4780ccbb..9c9e148fd 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -36,12 +36,11 @@ env: ECS_SERVICE: barrett-fogle-love-v1 ECS_CLUSTER: tcf-fargate-cluster ECS_TASK_DEFINITION: tcf-prod-task:14 - CONTAINER_IMAGE: tcf/thecourseforum2:latest # so idk if this is right but it should be the url to the docker image, used the repo name but the image in there is called "latest"? + CONTAINER_IMAGE: tcf/thecourseforum2:latest CONTAINER_NAME: latest - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} jobs: pylint: @@ -130,33 +129,37 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Build and push uses: docker/build-push-action@v4 with: context: . file: ./Dockerfile push: true - tags: ${{ secrets.DOCKER_HUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} - - name: Fill in the new image ID in the Amazon ECS task definition + tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.CONTAINER_IMAGE }}:$(github.sha) + + - name: Add image in ECS task definition id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 with: - task-definition: ${{ env.ECS_TASK_DEFINITION }} - container-name: ${{ env.CONTAINER_NAME }} - image: ${{ env.CONTAINER_IMAGE }} + task-definition: ${{ env.ECS_TASK_DEFINITION }} + container-name: ${{ env.CONTAINER_NAME }} + image: ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.CONTAINER_IMAGE }}$(github.sha) + - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v1 with: From 70d3efcf2535ea7fa7488d4217fc1139352d06f3 Mon Sep 17 00:00:00 2001 From: gckang <76260739+gckang@users.noreply.github.com> Date: Sun, 2 Mar 2025 15:25:16 -0500 Subject: [PATCH 04/32] move aws deploy cicd to master-to-app-engine.yml --- .github/workflows/continuous-integration.yml | 59 -------------------- .github/workflows/master-to-app-engine.yml | 57 +++++++++++++++++-- 2 files changed, 52 insertions(+), 64 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 9c9e148fd..02a121130 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -30,17 +30,6 @@ env: REVIEW_DRIVE_ID: ${{ secrets.REVIEW_DRIVE_ID }} REVIEW_DRIVE_EMAIL: ${{ secrets.REVIEW_DRIVE_EMAIL }} REVIEW_DRIVE_PASSWORD: ${{ secrets.REVIEW_DRIVE_PASSWORD }} - # aws ecs image - IMAGE_NAME: latest - AWS_REGION: us-east-1f - ECS_SERVICE: barrett-fogle-love-v1 - ECS_CLUSTER: tcf-fargate-cluster - ECS_TASK_DEFINITION: tcf-prod-task:14 - CONTAINER_IMAGE: tcf/thecourseforum2:latest - CONTAINER_NAME: latest - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} jobs: pylint: @@ -119,52 +108,4 @@ jobs: - name: Run ESLint run: npx eslint -c .config/.eslintrc.yml tcf_website/static/ - - deploy: - name: Deploy - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: true - tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.CONTAINER_IMAGE }}:$(github.sha) - - - name: Add image in ECS task definition - id: task-def - uses: aws-actions/amazon-ecs-render-task-definition@v1 - with: - task-definition: ${{ env.ECS_TASK_DEFINITION }} - container-name: ${{ env.CONTAINER_NAME }} - image: ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.CONTAINER_IMAGE }}$(github.sha) - - - name: Deploy Amazon ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 - with: - task-definition: ${{ steps.task-def.outputs.task-definition }} - service: ${{ env.ECS_SERVICE }} - cluster: ${{ env.ECS_CLUSTER }} - wait-for-service-stability: true diff --git a/.github/workflows/master-to-app-engine.yml b/.github/workflows/master-to-app-engine.yml index cde3bf674..e6989414a 100644 --- a/.github/workflows/master-to-app-engine.yml +++ b/.github/workflows/master-to-app-engine.yml @@ -31,6 +31,18 @@ env: REVIEW_DRIVE_ID: ${{ secrets.REVIEW_DRIVE_ID }} REVIEW_DRIVE_EMAIL: ${{ secrets.REVIEW_DRIVE_EMAIL }} REVIEW_DRIVE_PASSWORD: ${{ secrets.REVIEW_DRIVE_PASSWORD }} + # aws ecs image + IMAGE_NAME: latest + AWS_REGION: us-east-1f + ECS_SERVICE: barrett-fogle-love-v1 + ECS_CLUSTER: tcf-fargate-cluster + ECS_TASK_DEFINITION: tcf-prod-task:14 + CONTAINER_IMAGE: tcf/thecourseforum2:latest + CONTAINER_NAME: latest + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + S3_BUCKET_NAME: tcf-static-files jobs: deploy: @@ -69,10 +81,45 @@ jobs: - name: Collect static files run: python3 manage.py collectstatic --noinput --clear - - name: Substitute environment variables - run: envsubst < .config/app.yaml.template > app.yaml + - name: Upload static files to S3 + run: | + aws s3 sync ./static/ s3://${{ env.S3_BUCKET_NAME }}/ --delete + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - - name: Deploy to Google App Engine - uses: google-github-actions/deploy-appengine@main + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.CONTAINER_IMAGE }}:$(github.sha) + + - name: Add image in ECS task definition + id: task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: ${{ env.ECS_TASK_DEFINITION }} + container-name: ${{ env.CONTAINER_NAME }} + image: ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.CONTAINER_IMAGE }}$(github.sha) + + - name: Deploy Amazon ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v1 with: - project_id: ${{ secrets.GCP_PROJECT_ID }} + task-definition: ${{ steps.task-def.outputs.task-definition }} + service: ${{ env.ECS_SERVICE }} + cluster: ${{ env.ECS_CLUSTER }} + wait-for-service-stability: true + From 04d0d3c443b5b66ba4e49a90a6834e26d1686c5b Mon Sep 17 00:00:00 2001 From: Lucas Kohler Date: Sun, 2 Mar 2025 15:22:52 -0500 Subject: [PATCH 05/32] s3 - by ldkohler (GOAT) --- tcf_core/settings/base.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tcf_core/settings/base.py b/tcf_core/settings/base.py index f6c875996..8cf792042 100644 --- a/tcf_core/settings/base.py +++ b/tcf_core/settings/base.py @@ -43,11 +43,26 @@ "django.contrib.staticfiles", "social_django", "cachalot", # TODO: add Redis? + "storages", "rest_framework", "django_filters", "tcf_website", ] +AWS_ACCESS_KEY_ID = env.str("AWS_ACCESS_KEY_ID") +AWS_SECRET_ACCESS_KEY = env.str("AWS_SECRET_ACCESS_KEY") +AWS_STORAGE_BUCKET_NAME = env.str("AWS_STORAGE_BUCKET_NAME") +AWS_S3_REGION_NAME = env.str("AWS_S3_REGION_NAME", default="us-east-1") +AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" +AWS_DEFAULT_ACL = "public-read" +AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"} + +STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/static/" +STATICFILES_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" + +MEDIA_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/media/" +DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" + MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", @@ -122,8 +137,8 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.0/howto/static-files/ -STATIC_URL = "/static/" -STATIC_ROOT = os.path.join(BASE_DIR, "static/") +STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/static/" +STATICFILES_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" # Database From 011f47f346e202486d137a84dc79a7688783a060 Mon Sep 17 00:00:00 2001 From: gckang <76260739+gckang@users.noreply.github.com> Date: Sun, 2 Mar 2025 15:27:51 -0500 Subject: [PATCH 06/32] update region aws cicd --- .github/workflows/master-to-app-engine.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master-to-app-engine.yml b/.github/workflows/master-to-app-engine.yml index e6989414a..6348faf87 100644 --- a/.github/workflows/master-to-app-engine.yml +++ b/.github/workflows/master-to-app-engine.yml @@ -33,7 +33,7 @@ env: REVIEW_DRIVE_PASSWORD: ${{ secrets.REVIEW_DRIVE_PASSWORD }} # aws ecs image IMAGE_NAME: latest - AWS_REGION: us-east-1f + AWS_REGION: us-east-1 ECS_SERVICE: barrett-fogle-love-v1 ECS_CLUSTER: tcf-fargate-cluster ECS_TASK_DEFINITION: tcf-prod-task:14 From 95ca2a17d2601ed15c03690b324a4028c6c24f9a Mon Sep 17 00:00:00 2001 From: gckang <76260739+gckang@users.noreply.github.com> Date: Sun, 2 Mar 2025 15:37:51 -0500 Subject: [PATCH 07/32] dynamic task defintion update and ecs image name --- .github/workflows/master-to-app-engine.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/master-to-app-engine.yml b/.github/workflows/master-to-app-engine.yml index 6348faf87..d4658dc9a 100644 --- a/.github/workflows/master-to-app-engine.yml +++ b/.github/workflows/master-to-app-engine.yml @@ -32,16 +32,14 @@ env: REVIEW_DRIVE_EMAIL: ${{ secrets.REVIEW_DRIVE_EMAIL }} REVIEW_DRIVE_PASSWORD: ${{ secrets.REVIEW_DRIVE_PASSWORD }} # aws ecs image - IMAGE_NAME: latest AWS_REGION: us-east-1 ECS_SERVICE: barrett-fogle-love-v1 - ECS_CLUSTER: tcf-fargate-cluster - ECS_TASK_DEFINITION: tcf-prod-task:14 - CONTAINER_IMAGE: tcf/thecourseforum2:latest - CONTAINER_NAME: latest + ECS_CLUSTER: tcf-fargate-cluster + ECR_REPO: tcf/thecourseforum2 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + # aws s3 bucket S3_BUCKET_NAME: tcf-static-files jobs: @@ -105,15 +103,15 @@ jobs: context: . file: ./Dockerfile push: true - tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.CONTAINER_IMAGE }}:$(github.sha) + tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPO }}:${{github.sha}} - name: Add image in ECS task definition id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 with: - task-definition: ${{ env.ECS_TASK_DEFINITION }} - container-name: ${{ env.CONTAINER_NAME }} - image: ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.CONTAINER_IMAGE }}$(github.sha) + task-definition: $(aws ecs list-task-definitions --family-prefix tcf-prod-task --sort DESC --max-items 1 --region=us-east-1 --profile tcf | jq -r '.taskDefinitionArns[0] | split("/")[-1]') + container-name: ${{ env.ECR_REPO }} + image: ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPO }}:${{github.sha}} - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v1 From fb08f87e4d299b2bd63d81b4cd005ec39ca7201b Mon Sep 17 00:00:00 2001 From: gckang <76260739+gckang@users.noreply.github.com> Date: Sun, 2 Mar 2025 15:51:34 -0500 Subject: [PATCH 08/32] update geting task definition dynamically --- .github/workflows/master-to-app-engine.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/master-to-app-engine.yml b/.github/workflows/master-to-app-engine.yml index d4658dc9a..26dac30c2 100644 --- a/.github/workflows/master-to-app-engine.yml +++ b/.github/workflows/master-to-app-engine.yml @@ -105,12 +105,18 @@ jobs: push: true tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPO }}:${{github.sha}} + - name: Get latest task definition + run: | + aws ecs describe-task-definition \ + --task-definition tcf-prod-task \ + --query taskDefinition > task-definition.json + - name: Add image in ECS task definition id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 with: - task-definition: $(aws ecs list-task-definitions --family-prefix tcf-prod-task --sort DESC --max-items 1 --region=us-east-1 --profile tcf | jq -r '.taskDefinitionArns[0] | split("/")[-1]') - container-name: ${{ env.ECR_REPO }} + task-definition: task-definition.json + container-name: ${{ env.ECR_REPO }}:${{github.sha}} image: ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPO }}:${{github.sha}} - name: Deploy Amazon ECS task definition From 4289102c0a48733cc845c51ab22af2b4837af374 Mon Sep 17 00:00:00 2001 From: Lucas Kohler Date: Sun, 2 Mar 2025 15:48:57 -0500 Subject: [PATCH 09/32] update requirements to include django-storages and boto3 --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 597b365e7..2db5c066e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,13 @@ Django~=4.2.8 backoff~=2.2.1 black~=24.1.1 +boto3~=1.37.4 coverage~=7.3.3 django-cachalot~=2.6.1 django-environ~=0.11.2 django-filter~=23.5 django-heroku~=0.3.1 +django-storages~=1.14.5 django-stubs~=4.2.7 djangorestframework~=3.14.0 gunicorn~=21.2.0 From 7c6f4a7a30ca440eb6b0f02ed153d2fb715cfafc Mon Sep 17 00:00:00 2001 From: gckang <76260739+gckang@users.noreply.github.com> Date: Sun, 2 Mar 2025 15:58:13 -0500 Subject: [PATCH 10/32] update ecr container name cicd --- .github/workflows/master-to-app-engine.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/master-to-app-engine.yml b/.github/workflows/master-to-app-engine.yml index 26dac30c2..a6951e262 100644 --- a/.github/workflows/master-to-app-engine.yml +++ b/.github/workflows/master-to-app-engine.yml @@ -36,6 +36,7 @@ env: ECS_SERVICE: barrett-fogle-love-v1 ECS_CLUSTER: tcf-fargate-cluster ECR_REPO: tcf/thecourseforum2 + ECR_CONTAINER_NAME: tcf-container AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} @@ -116,7 +117,7 @@ jobs: uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: task-definition.json - container-name: ${{ env.ECR_REPO }}:${{github.sha}} + container-name: ${{ env.ECR_CONTAINER_NAME }} image: ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPO }}:${{github.sha}} - name: Deploy Amazon ECS task definition From 581bb48460511277f541174b1cb5cb0eea3c7a0a Mon Sep 17 00:00:00 2001 From: Lucas Kohler Date: Sun, 2 Mar 2025 16:03:09 -0500 Subject: [PATCH 11/32] delete .env.example --- .config/.env.example | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 .config/.env.example diff --git a/.config/.env.example b/.config/.env.example deleted file mode 100644 index 92d49d8c6..000000000 --- a/.config/.env.example +++ /dev/null @@ -1,30 +0,0 @@ -# Django -DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE -DEBUG=$DEBUG -SECRET_KEY=$SECRET_KEY - -ENVIRONMENT=$ENVIRONMENT - -# review drive account -REVIEW_DRIVE_ID=$REVIEW_DRIVE_ID -REVIEW_DRIVE_EMAIL=$REVIEW_DRIVE_EMAIL -REVIEW_DRIVE_PASSWORD=$REVIEW_DRIVE_PASSWORD - -# social-auth-app-django library -SOCIAL_AUTH_GOOGLE_OAUTH2_KEY=$SOCIAL_AUTH_GOOGLE_OAUTH2_KEY -SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET=$SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET - -# django-microsoft-auth library -SOCIAL_AUTH_MICROSOFT_CLIENT_ID=$SOCIAL_AUTH_MICROSOFT_CLIENT_ID -SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET=$SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET - -# database -DB_NAME=$DB_NAME -DB_USER=$DB_USER -DB_PASSWORD=$DB_PASSWORD -DB_HOST=$DB_HOST -DB_PORT=$DB_PORT - -# feedback -EMAIL_HOST_USER=$EMAIL_HOST_USER -EMAIL_HOST_PASSWORD=$EMAIL_HOST_PASSWORD \ No newline at end of file From bd535ee50d07fe91bb1f3f5f4aa90a561ddd45dd Mon Sep 17 00:00:00 2001 From: Lucas Kohler Date: Sun, 2 Mar 2025 16:03:26 -0500 Subject: [PATCH 12/32] revise cicd and rename yaml --- .github/workflows/{master-to-app-engine.yml => aws.yml} | 7 ------- 1 file changed, 7 deletions(-) rename .github/workflows/{master-to-app-engine.yml => aws.yml} (95%) diff --git a/.github/workflows/master-to-app-engine.yml b/.github/workflows/aws.yml similarity index 95% rename from .github/workflows/master-to-app-engine.yml rename to .github/workflows/aws.yml index a6951e262..7b4f71d27 100644 --- a/.github/workflows/master-to-app-engine.yml +++ b/.github/workflows/aws.yml @@ -77,13 +77,6 @@ jobs: service_account_key: ${{ secrets.GCP_SA_KEY }} export_default_credentials: true - - name: Collect static files - run: python3 manage.py collectstatic --noinput --clear - - - name: Upload static files to S3 - run: | - aws s3 sync ./static/ s3://${{ env.S3_BUCKET_NAME }}/ --delete - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: From 62fe9e96b2a872f3e5394c0d03f7faec635bdf76 Mon Sep 17 00:00:00 2001 From: Lucas Kohler Date: Sun, 2 Mar 2025 16:03:37 -0500 Subject: [PATCH 13/32] delete old app.yaml template --- .config/app.yaml.template | 47 --------------------------------------- 1 file changed, 47 deletions(-) delete mode 100644 .config/app.yaml.template diff --git a/.config/app.yaml.template b/.config/app.yaml.template deleted file mode 100644 index 940126483..000000000 --- a/.config/app.yaml.template +++ /dev/null @@ -1,47 +0,0 @@ -# Google App Engine Config - -runtime: python311 - -# The --http flag exposes the app with uWSGI only and will act as the router/proxy/load balancer -# If we want an nginx webserver upstream for more features, we would have to use a --socket flag so that nginx can communicate with uWSGI -# More info here: https://uwsgi-docs.readthedocs.io/en/latest/tutorials/Django_and_nginx.html -entrypoint: uwsgi --http :8080 --module tcf_core.wsgi --master --processes 1 --threads 2 - -handlers: -# This configures Google App Engine to serve the files in the app's static -# directory. -- url: /static - static_dir: static/ - -# This handler routes all requests not caught above to your main app. It is -# required when static routes are defined, but can be omitted (along with -# the entire handlers section) when there are no static files defined. -- url: /.* - script: auto - secure: always - -env_variables: - DJANGO_SETTINGS_MODULE: tcf_core.settings.prod - # Django - SECRET_KEY: "${SECRET_KEY}" - DEBUG: 0 - # database - DB_NAME: "${DB_NAME}" - DB_USER: "${DB_USER}" - DB_PASSWORD: "${DB_PASSWORD}" - DB_HOST: "${DB_HOST}" - DB_PORT: "${DB_PORT}" - # social-auth-app-django - SOCIAL_AUTH_GOOGLE_OAUTH2_KEY: "${SOCIAL_AUTH_GOOGLE_OAUTH2_KEY}" - SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET: "${SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET}" - # elasticsearch - ELASTICSEARCH_ENDPOINT: "${ELASTICSEARCH_ENDPOINT}" - ES_PRIVATE_API_KEY: "${ES_PRIVATE_API_KEY}" - ES_PUBLIC_API_KEY: "${ES_PUBLIC_API_KEY}" - # email for account verification - EMAIL_HOST_USER: "${EMAIL_HOST_USER}" - EMAIL_HOST_PASSWORD: "${EMAIL_HOST_PASSWORD}" - # review drive information - REVIEW_DRIVE_ID: "${REVIEW_DRIVE_ID}" - REVIEW_DRIVE_EMAIL: "${REVIEW_DRIVE_EMAIL}" - REVIEW_DRIVE_PASSWORD: "${REVIEW_DRIVE_PASSWORD}" From e3366779ccf8989118e011a86c55f3edb890139e Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Sun, 2 Mar 2025 16:26:56 -0500 Subject: [PATCH 14/32] feat(aws): update django-storages config --- tcf_core/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcf_core/settings/base.py b/tcf_core/settings/base.py index 8cf792042..a7b1f3bd5 100644 --- a/tcf_core/settings/base.py +++ b/tcf_core/settings/base.py @@ -54,7 +54,7 @@ AWS_STORAGE_BUCKET_NAME = env.str("AWS_STORAGE_BUCKET_NAME") AWS_S3_REGION_NAME = env.str("AWS_S3_REGION_NAME", default="us-east-1") AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" -AWS_DEFAULT_ACL = "public-read" +AWS_DEFAULT_ACL = None AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"} STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/static/" From 08ad2a5ebe39fed97bc959397d03b5c998aca9e9 Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Sun, 2 Mar 2025 16:48:58 -0500 Subject: [PATCH 15/32] fix(s3): use the latest boto-storages api --- tcf_core/settings/base.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tcf_core/settings/base.py b/tcf_core/settings/base.py index a7b1f3bd5..e36102a36 100644 --- a/tcf_core/settings/base.py +++ b/tcf_core/settings/base.py @@ -54,14 +54,17 @@ AWS_STORAGE_BUCKET_NAME = env.str("AWS_STORAGE_BUCKET_NAME") AWS_S3_REGION_NAME = env.str("AWS_S3_REGION_NAME", default="us-east-1") AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" -AWS_DEFAULT_ACL = None +AWS_DEFAULT_ACL = 'public-read' AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"} -STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/static/" -STATICFILES_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" - -MEDIA_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/media/" -DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" +STORAGES = { + "default": { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": { + }, + }, +} +STATICFILES_STORAGE = "storages.backends.s3.S3Storage" MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", From ba2a329c3dd69e7a2828873d1332b41f649b34f5 Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Sun, 2 Mar 2025 16:58:21 -0500 Subject: [PATCH 16/32] prayge --- tcf_core/settings/base.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tcf_core/settings/base.py b/tcf_core/settings/base.py index e36102a36..c24b1f0c4 100644 --- a/tcf_core/settings/base.py +++ b/tcf_core/settings/base.py @@ -54,17 +54,16 @@ AWS_STORAGE_BUCKET_NAME = env.str("AWS_STORAGE_BUCKET_NAME") AWS_S3_REGION_NAME = env.str("AWS_S3_REGION_NAME", default="us-east-1") AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" -AWS_DEFAULT_ACL = 'public-read' +AWS_DEFAULT_ACL = "public-read" AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"} STORAGES = { "default": { "BACKEND": "storages.backends.s3.S3Storage", - "OPTIONS": { - }, + "OPTIONS": {}, }, + "staticfiles": "storages.backends.s3.S3Storage", } -STATICFILES_STORAGE = "storages.backends.s3.S3Storage" MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", @@ -183,9 +182,7 @@ WHITELISTED_DOMAINS = ["virginia.edu"] -SOCIAL_AUTH_GOOGLE_OAUTH2_LOGIN_URL = reverse_lazy( - "social:begin", args=["google-oauth2"] -) +SOCIAL_AUTH_GOOGLE_OAUTH2_LOGIN_URL = reverse_lazy("social:begin", args=["google-oauth2"]) SOCIAL_AUTH_RAISE_EXCEPTIONS = False SOCIAL_AUTH_PIPELINE = ( "tcf_core.auth_pipeline.password_validation", From 838c7d88becce6a687195454d0ae015463f5d2d4 Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Sun, 2 Mar 2025 17:02:20 -0500 Subject: [PATCH 17/32] fix django remove staticfiles --- tcf_core/settings/base.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tcf_core/settings/base.py b/tcf_core/settings/base.py index c24b1f0c4..c8d650432 100644 --- a/tcf_core/settings/base.py +++ b/tcf_core/settings/base.py @@ -136,13 +136,6 @@ USE_TZ = True -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.0/howto/static-files/ - -STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/static/" -STATICFILES_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" - - # Database # https://docs.djangoproject.com/en/3.0/ref/settings/#databases From 203f082aa22b59b22aabed29ea458ecf51bcb382 Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Sun, 2 Mar 2025 17:19:46 -0500 Subject: [PATCH 18/32] prayge --- tcf_core/settings/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tcf_core/settings/base.py b/tcf_core/settings/base.py index c8d650432..38014411e 100644 --- a/tcf_core/settings/base.py +++ b/tcf_core/settings/base.py @@ -62,7 +62,6 @@ "BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": {}, }, - "staticfiles": "storages.backends.s3.S3Storage", } MIDDLEWARE = [ From b661329caa99d745abda37047f3f4ca020f57efc Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Sun, 2 Mar 2025 17:26:57 -0500 Subject: [PATCH 19/32] pray --- tcf_core/settings/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tcf_core/settings/base.py b/tcf_core/settings/base.py index 38014411e..926b8c1cb 100644 --- a/tcf_core/settings/base.py +++ b/tcf_core/settings/base.py @@ -62,6 +62,9 @@ "BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": {}, }, + "staticfiles": { + "BACKEND": "storages.backends.s3.S3Storage", + } } MIDDLEWARE = [ From 9bdb3afe17ee6ed5c393b71eebc4acd5a07a7efe Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Wed, 5 Mar 2025 12:13:37 -0500 Subject: [PATCH 20/32] feat(aws/ci): temporarily change branch to this one to test --- .github/workflows/aws.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 7b4f71d27..80611b050 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -5,7 +5,7 @@ on: # CI must pass on the master branch workflow_run: workflows: ["Continuous Integration"] - branches: [master] + branches: [aws/cicd] types: - completed @@ -32,15 +32,14 @@ env: REVIEW_DRIVE_EMAIL: ${{ secrets.REVIEW_DRIVE_EMAIL }} REVIEW_DRIVE_PASSWORD: ${{ secrets.REVIEW_DRIVE_PASSWORD }} # aws ecs image - AWS_REGION: us-east-1 - ECS_SERVICE: barrett-fogle-love-v1 - ECS_CLUSTER: tcf-fargate-cluster - ECR_REPO: tcf/thecourseforum2 + AWS_REGION: us-east-1 + ECS_SERVICE: barrett-fogle-love-v1 + ECS_CLUSTER: tcf-fargate-cluster + ECR_REPO: tcf/thecourseforum2 ECR_CONTAINER_NAME: tcf-container AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - # aws s3 bucket S3_BUCKET_NAME: tcf-static-files jobs: @@ -87,7 +86,7 @@ jobs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -112,7 +111,7 @@ jobs: task-definition: task-definition.json container-name: ${{ env.ECR_CONTAINER_NAME }} image: ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPO }}:${{github.sha}} - + - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v1 with: @@ -120,4 +119,3 @@ jobs: service: ${{ env.ECS_SERVICE }} cluster: ${{ env.ECS_CLUSTER }} wait-for-service-stability: true - From ce2d210315b6dbce1199ed8e07678d2d397fc417 Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Wed, 5 Mar 2025 12:18:24 -0500 Subject: [PATCH 21/32] fix(ci): expose aws info to ci environment --- .github/workflows/aws.yml | 2 +- .github/workflows/{continuous-integration.yml => ci.yml} | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) rename .github/workflows/{continuous-integration.yml => ci.yml} (95%) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 80611b050..9335d9c01 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -1,5 +1,5 @@ # Based on https://github.com/actions/starter-workflows/blob/main/ci/django.yml -name: Google App Engine Deployment +name: AWS Deployment on: # CI must pass on the master branch diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/ci.yml similarity index 95% rename from .github/workflows/continuous-integration.yml rename to .github/workflows/ci.yml index 02a121130..21f9615e6 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,9 @@ env: REVIEW_DRIVE_ID: ${{ secrets.REVIEW_DRIVE_ID }} REVIEW_DRIVE_EMAIL: ${{ secrets.REVIEW_DRIVE_EMAIL }} REVIEW_DRIVE_PASSWORD: ${{ secrets.REVIEW_DRIVE_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} jobs: pylint: From 7016dc31f71a0b892abea99b6c87a40dbdfececd Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Wed, 5 Mar 2025 12:24:38 -0500 Subject: [PATCH 22/32] fix(ci): extract s3 bucket name to env var --- .github/workflows/aws.yml | 2 +- .github/workflows/ci.yml | 1 + tcf_core/settings/base.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 9335d9c01..abec4269d 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -40,7 +40,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - S3_BUCKET_NAME: tcf-static-files + AWS_S3_BUCKET_NAME: ${{ secrets.AWS_S3_BUCKET_NAME }} jobs: deploy: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21f9615e6..87faadf1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_S3_BUCKET_NAME: ${{ secrets.AWS_S3_BUCKET_NAME }} jobs: pylint: diff --git a/tcf_core/settings/base.py b/tcf_core/settings/base.py index 926b8c1cb..bdb7a3294 100644 --- a/tcf_core/settings/base.py +++ b/tcf_core/settings/base.py @@ -51,9 +51,9 @@ AWS_ACCESS_KEY_ID = env.str("AWS_ACCESS_KEY_ID") AWS_SECRET_ACCESS_KEY = env.str("AWS_SECRET_ACCESS_KEY") -AWS_STORAGE_BUCKET_NAME = env.str("AWS_STORAGE_BUCKET_NAME") +AWS_S3_BUCKET_NAME = env.str("AWS_S3_BUCKET_NAME") AWS_S3_REGION_NAME = env.str("AWS_S3_REGION_NAME", default="us-east-1") -AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" +AWS_S3_CUSTOM_DOMAIN = f"{AWS_S3_BUCKET_NAME}.s3.amazonaws.com" AWS_DEFAULT_ACL = "public-read" AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"} From ebbd7c951f4b442479751a2fc8866df25c658df5 Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Wed, 5 Mar 2025 12:32:16 -0500 Subject: [PATCH 23/32] .env.example needed for ci coverage --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87faadf1c..2fb1ada09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,7 +88,7 @@ jobs: - name: Migrations & Tests run: | - envsubst < .config/.env.example > .env + envsubst < .env.example > .env python manage.py migrate coverage run manage.py test From 3ebe8178d858b4a1ce398d2f1d57d85df47a351c Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Wed, 5 Mar 2025 12:36:45 -0500 Subject: [PATCH 24/32] feat(env): force include env example --- .env.example | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..e2858f903 --- /dev/null +++ b/.env.example @@ -0,0 +1,36 @@ +# Django +DJANGO_SETTINGS_MODULE=? +DEBUG=? +SECRET_KEY=? + +ENVIRONMENT=? + +# review drive account +REVIEW_DRIVE_ID=? +REVIEW_DRIVE_EMAIL=? +REVIEW_DRIVE_PASSWORD=? + +# social-auth-app-django library +SOCIAL_AUTH_GOOGLE_OAUTH2_KEY=? +SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET=? + +# database +DB_USER=? +DB_NAME=? +DB_PASSWORD=? +DB_HOST=? +DB_PORT=? +sslmode=? + +# email +EMAIL_HOST_USER=? +EMAIL_HOST_PASSWORD=? + + +# aws +AWS_ACCESS_KEY_ID=? +AWS_SECRET_ACCESS_KEY=? + +# s3 +AWS_STORAGE_BUCKET_NAME=? +AWS_S3_REGION_NAME=? From dfec13457989a3e86b6a4775aed051fba6856ad7 Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Wed, 5 Mar 2025 12:53:20 -0500 Subject: [PATCH 25/32] fix(ci/aws): remove superfluous gcloud step --- .github/workflows/aws.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index abec4269d..2079f50b9 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -70,12 +70,6 @@ jobs: run: | python manage.py migrate - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: From 8df27f1d1a966d8493ff84bdc01d806ab76f027b Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Wed, 5 Mar 2025 12:54:26 -0500 Subject: [PATCH 26/32] feat(aws/ci): run ci on non-default branch --- .github/workflows/aws.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 2079f50b9..02f20bdd6 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -44,7 +44,7 @@ env: jobs: deploy: - # if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest steps: # A workaround because this workflow runs on the default branch (`dev`) From 5315a2bc77689f4327f5c351bd266e6dbdcba25a Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Wed, 5 Mar 2025 13:01:12 -0500 Subject: [PATCH 27/32] revert(ci/aws): remove the if statement, idek --- .github/workflows/aws.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 02f20bdd6..2079f50b9 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -44,7 +44,7 @@ env: jobs: deploy: - if: ${{ github.event.workflow_run.conclusion == 'success' }} + # if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest steps: # A workaround because this workflow runs on the default branch (`dev`) From d3ac6caa9a3dce09930c6d31aefadafff619de98 Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Wed, 5 Mar 2025 23:30:22 -0500 Subject: [PATCH 28/32] fix(aws): use master branch only --- .github/workflows/aws.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 2079f50b9..03f4f9c6b 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -5,7 +5,7 @@ on: # CI must pass on the master branch workflow_run: workflows: ["Continuous Integration"] - branches: [aws/cicd] + branches: [master] types: - completed From 2678e5cf050b91415c4ed2a04554eddcbdffaf17 Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Wed, 5 Mar 2025 23:31:34 -0500 Subject: [PATCH 29/32] fix(aws): update env var name --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index e2858f903..9067133f5 100644 --- a/.env.example +++ b/.env.example @@ -32,5 +32,5 @@ AWS_ACCESS_KEY_ID=? AWS_SECRET_ACCESS_KEY=? # s3 -AWS_STORAGE_BUCKET_NAME=? +AWS_S3_BUCKET_NAME=? AWS_S3_REGION_NAME=? From a11218bec3da54c0f7d1ab3f4358b9085d894ec1 Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Thu, 6 Mar 2025 00:09:30 -0500 Subject: [PATCH 30/32] fix env var names --- .env.example | 2 +- tcf_core/settings/base.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 9067133f5..e2858f903 100644 --- a/.env.example +++ b/.env.example @@ -32,5 +32,5 @@ AWS_ACCESS_KEY_ID=? AWS_SECRET_ACCESS_KEY=? # s3 -AWS_S3_BUCKET_NAME=? +AWS_STORAGE_BUCKET_NAME=? AWS_S3_REGION_NAME=? diff --git a/tcf_core/settings/base.py b/tcf_core/settings/base.py index bdb7a3294..926b8c1cb 100644 --- a/tcf_core/settings/base.py +++ b/tcf_core/settings/base.py @@ -51,9 +51,9 @@ AWS_ACCESS_KEY_ID = env.str("AWS_ACCESS_KEY_ID") AWS_SECRET_ACCESS_KEY = env.str("AWS_SECRET_ACCESS_KEY") -AWS_S3_BUCKET_NAME = env.str("AWS_S3_BUCKET_NAME") +AWS_STORAGE_BUCKET_NAME = env.str("AWS_STORAGE_BUCKET_NAME") AWS_S3_REGION_NAME = env.str("AWS_S3_REGION_NAME", default="us-east-1") -AWS_S3_CUSTOM_DOMAIN = f"{AWS_S3_BUCKET_NAME}.s3.amazonaws.com" +AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" AWS_DEFAULT_ACL = "public-read" AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"} From 542820f6270ee2b080e4a83093d85263484f92bd Mon Sep 17 00:00:00 2001 From: Lucas Kohler Date: Thu, 6 Mar 2025 00:29:56 -0500 Subject: [PATCH 31/32] switch default ACL to none --- tcf_core/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcf_core/settings/base.py b/tcf_core/settings/base.py index 926b8c1cb..261839999 100644 --- a/tcf_core/settings/base.py +++ b/tcf_core/settings/base.py @@ -54,7 +54,7 @@ AWS_STORAGE_BUCKET_NAME = env.str("AWS_STORAGE_BUCKET_NAME") AWS_S3_REGION_NAME = env.str("AWS_S3_REGION_NAME", default="us-east-1") AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" -AWS_DEFAULT_ACL = "public-read" +AWS_DEFAULT_ACL = None AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"} STORAGES = { From 6fc2531a0a8f4672b1e1e5fa611ac71f7a7c3f60 Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Thu, 6 Mar 2025 11:30:25 -0500 Subject: [PATCH 32/32] fix(ci): proper s3 env vars --- .github/workflows/aws.yml | 5 +---- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 03f4f9c6b..74aec52c6 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -40,16 +40,13 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - AWS_S3_BUCKET_NAME: ${{ secrets.AWS_S3_BUCKET_NAME }} + AWS_STORAGE_BUCKET_NAME: ${{ secrets.AWS_STORAGE_BUCKET_NAME }} jobs: deploy: # if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest steps: - # A workaround because this workflow runs on the default branch (`dev`) - # because that's how `workflow_run` works at the moment - # https://github.community/t/workflow-run-not-working-as-expected/139342 - name: Checkout `master` uses: actions/checkout@v2 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fb1ada09..3de904102 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - AWS_S3_BUCKET_NAME: ${{ secrets.AWS_S3_BUCKET_NAME }} + AWS_STORAGE_BUCKET_NAME: ${{ secrets.AWS_STORAGE_BUCKET_NAME }} jobs: pylint: