diff --git a/.flake8 b/.flake8 index da8640d1..a9acfb2c 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] max-line-length = 88 select = C,E,F,W,B,B9 -ignore = E203, E501, W503, B006 +ignore = E203, E501, W503, B006, E712 exclude = .hg, .git, diff --git a/landoapi/api/repos.py b/landoapi/api/repos.py new file mode 100644 index 00000000..a3fd2c96 --- /dev/null +++ b/landoapi/api/repos.py @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import logging + +from flask import current_app, g + +from landoapi import auth +from landoapi.models.repo import RepoNotice +from landoapi.repos import get_repos_for_env, SCM_ALLOW_DIRECT_PUSH +from landoapi.storage import db + +logger = logging.getLogger(__name__) +auth_params = {"scopes": ("lando", "profile", "email"), "userinfo": True} + + +@auth.require_auth0(**auth_params) +def get_repo_notices(): + if SCM_ALLOW_DIRECT_PUSH.active_group not in g.auth0_user.groups: + raise auth._not_authorized_problem_exception() + supported_repos = get_repos_for_env(current_app.config.get("ENVIRONMENT")) + repos = list(supported_repos.keys()) + notices = RepoNotice.query.filter(RepoNotice.is_archived == False).order_by( + RepoNotice.updated_at.desc() + ) + + return {"notices": [n.serialize() for n in notices], "repos": repos}, 200 + + +@auth.require_auth0(**auth_params) +def post_repo_notice(data): + if SCM_ALLOW_DIRECT_PUSH.active_group not in g.auth0_user.groups: + raise auth._not_authorized_problem_exception() + notice = RepoNotice() + for attr in data: + setattr(notice, attr, data[attr]) + db.session.add(notice) + db.session.commit() + logger.info(f"{notice.id} was created.") + return notice.serialize(), 201 + + +@auth.require_auth0(**auth_params) +def put_repo_notice(notice_id, data): + if SCM_ALLOW_DIRECT_PUSH.active_group not in g.auth0_user.groups: + raise auth._not_authorized_problem_exception() + notice = RepoNotice.get(notice_id) + for attr in data: + setattr(notice, attr, data[attr]) + db.session.add(notice) + db.session.commit() + return notice.serialize(), 200 + + +@auth.require_auth0(**auth_params) +def delete_repo_notice(notice_id): + if SCM_ALLOW_DIRECT_PUSH.active_group not in g.auth0_user.groups: + raise auth._not_authorized_problem_exception() + notice = RepoNotice.query.get(notice_id) + notice.is_archived = True + db.session.add(notice) + db.session.commit() + return notice.serialize(), 200 diff --git a/landoapi/api/stacks.py b/landoapi/api/stacks.py index b8990e26..a5138f06 100644 --- a/landoapi/api/stacks.py +++ b/landoapi/api/stacks.py @@ -30,6 +30,7 @@ serialize_diff, serialize_status, ) +from landoapi.models.repo import RepoNotice from landoapi.stacks import ( build_stack_graph, calculate_landable_subgraphs, @@ -155,12 +156,14 @@ def get(revision_id): ) repositories = [] + repo_notices = {} for phid in stack_data.repositories.keys(): short_name = PhabricatorClient.expect( stack_data.repositories[phid], "fields", "shortName" ) repo = supported_repos.get(short_name) + repo_notices[short_name] = RepoNotice.get_active_repo_notices(short_name) landing_supported = repo is not None url = ( repo.url @@ -181,6 +184,7 @@ def get(revision_id): return { "repositories": repositories, + "repo_notices": repo_notices, "revisions": revisions_response, "edges": [e for e in edges], "landable_paths": landable, diff --git a/landoapi/api/transplants.py b/landoapi/api/transplants.py index cb12fcd6..54e4e611 100644 --- a/landoapi/api/transplants.py +++ b/landoapi/api/transplants.py @@ -15,6 +15,7 @@ from landoapi.hgexports import build_patch_for_revision from landoapi.models.transplant import Transplant, TransplantStatus from landoapi.models.landing_job import LandingJob, LandingJobStatus +from landoapi.models.repo import RepoNotice from landoapi.patches import upload from landoapi.phabricator import PhabricatorClient from landoapi.projects import ( @@ -210,6 +211,7 @@ def _assess_transplant_request(phab, landing_path): get_secure_project_phid(phab), get_testing_tag_project_phids(phab), get_testing_policy_phid(phab), + RepoNotice.get_active_repo_notices(landing_repo.short_name), ) return (assessment, to_land, landing_repo, stack_data) diff --git a/landoapi/auth.py b/landoapi/auth.py index 40eae040..d68ee2dc 100644 --- a/landoapi/auth.py +++ b/landoapi/auth.py @@ -316,6 +316,8 @@ def _mock_userinfo_claims(userinfo): "all_scm_level_2", "active_scm_level_1", "all_scm_level_1", + "active_scm_allow_direct_push", + "all_scm_allow_direct_push", ] elif a0_mock_option == "inject_invalid": userinfo["https://sso.mozilla.com/claim/groups"] = ["invalid_group"] diff --git a/landoapi/models/__init__.py b/landoapi/models/__init__.py index 09ceb29a..1ca59790 100644 --- a/landoapi/models/__init__.py +++ b/landoapi/models/__init__.py @@ -1,6 +1,14 @@ +from landoapi.models.configuration import ConfigurationVariable from landoapi.models.landing_job import LandingJob +from landoapi.models.repo import RepoNotice from landoapi.models.secapproval import SecApprovalRequest from landoapi.models.transplant import Transplant -from landoapi.models.configuration import ConfigurationVariable -__all__ = ["LandingJob", "SecApprovalRequest", "Transplant", "ConfigurationVariable"] + +__all__ = [ + "ConfigurationVariable", + "LandingJob", + "RepoNotice", + "SecApprovalRequest", + "Transplant", +] diff --git a/landoapi/models/base.py b/landoapi/models/base.py index 5971ddc8..a3b2bebc 100644 --- a/landoapi/models/base.py +++ b/landoapi/models/base.py @@ -1,6 +1,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import datetime import re from sqlalchemy.ext.declarative import declared_attr @@ -44,3 +45,19 @@ def __repr__(self): For example, ``. """ return f"<{self.__class__.__name__}: {self.id}>" + + def serialize(self): + """Return a JSON compatible dictionary. + + This method should be extended in subclasses in order to include additional + fields. + """ + return { + "id": self.id, + "created_at": ( + self.created_at.astimezone(datetime.timezone.utc).isoformat() + ), + "updated_at": ( + self.updated_at.astimezone(datetime.timezone.utc).isoformat() + ), + } diff --git a/landoapi/models/repo.py b/landoapi/models/repo.py new file mode 100644 index 00000000..8ee92258 --- /dev/null +++ b/landoapi/models/repo.py @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import datetime +import logging + +from sqlalchemy import or_, and_ + +from landoapi.models.base import Base +from landoapi.storage import db + +logger = logging.getLogger(__name__) + + +class RepoNotice(Base): + """A scheduled notice that is associated with a repository.""" + + # This currently matches the keys in `landoapi.repos.REPO_CONFIG`. + # In the future, a `Repo` model should be created to house repos. + repo_identifier = db.Column(db.String(254), nullable=False) + start_date = db.Column(db.DateTime(timezone=True), nullable=True) + end_date = db.Column(db.DateTime(timezone=True), nullable=True) + message = db.Column(db.Text(), default="") + is_archived = db.Column(db.Boolean, default=False) + + # When set to `True`, results in a landing warning. + is_warning = db.Column(db.Boolean, default=False) + + @classmethod + def get_active_repo_notices(cls, repo_short_name): + now = datetime.datetime.now() + notices = ( + cls.query.filter( + and_(cls.repo_identifier == repo_short_name, cls.is_archived == False) + ) + .filter( + or_( + and_(cls.start_date <= now, cls.end_date >= now), + and_(cls.start_date <= now, cls.end_date == None), + and_(cls.start_date == None, cls.end_date >= now), + and_(cls.start_date == None, cls.end_date == None), + ) + ) + .order_by(cls.updated_at.desc()) + ) + return [n.serialize() for n in notices.all()] + + def serialize(self): + data = super().serialize() + data.update( + { + "repo_identifier": self.repo_identifier, + "start_date": self.start_date.astimezone( + datetime.timezone.utc + ).isoformat() + if self.start_date + else None, + "end_date": self.end_date.astimezone(datetime.timezone.utc).isoformat() + if self.end_date + else None, + "message": self.message, + "is_archived": self.is_archived, + "is_warning": self.is_warning, + } + ) + return data diff --git a/landoapi/repos.py b/landoapi/repos.py index 68a40fd6..f034540a 100644 --- a/landoapi/repos.py +++ b/landoapi/repos.py @@ -130,6 +130,12 @@ def __post_init__(self): display_name="scm_firefoxci", ) +SCM_ALLOW_DIRECT_PUSH = AccessGroup( + active_group="active_scm_allow_direct_push", + membership_group="all_scm_allow_direct_push", + display_name="scm_allow_direct_push", +) + # DONTBUILD flag and help text. DONTBUILD = ( "DONTBUILD", diff --git a/landoapi/spec/swagger.yml b/landoapi/spec/swagger.yml index ac3920df..28837196 100644 --- a/landoapi/spec/swagger.yml +++ b/landoapi/spec/swagger.yml @@ -236,6 +236,84 @@ paths: schema: allOf: - $ref: '#/definitions/Error' + /repos/notices: + get: + operationId: landoapi.api.repos.get_repo_notices + description: Get a list of all repo notices + responses: + 200: + description: OK + schema: + type: object + default: + description: Unexpected error + schema: + allOf: + - $ref: '#/definitions/Error' + post: + operationId: landoapi.api.repos.post_repo_notice + description: Post a new notice on a repo. + parameters: + - name: data + required: true + in: body + schema: + allOf: + - $ref: '#/definitions/RepoNotice' + responses: + 201: + description: OK + schema: + type: object + default: + description: Unexpected error + schema: + allOf: + - $ref: '#/definitions/Error' + /repos/notices/{notice_id}: + put: + operationId: landoapi.api.repos.put_repo_notice + description: Edit a repo notice. + parameters: + - name: notice_id + in: path + type: string + description: The primary key of the repo notice to edit. + required: true + - name: data + required: true + in: body + schema: + type: object + responses: + 200: + description: OK + schema: + type: object + default: + description: Unexpected error + schema: + allOf: + - $ref: '#/definitions/Error' + delete: + operationId: landoapi.api.repos.delete_repo_notice + description: Archive a repo notice. + parameters: + - name: notice_id + in: path + type: string + description: The primary key of the repo notice to archive. + required: true + responses: + 200: + description: OK + schema: + type: object + default: + description: Unexpected error + schema: + allOf: + - $ref: '#/definitions/Error' /stacks/{revision_id}: get: description: | @@ -335,6 +413,27 @@ paths: allOf: - $ref: '#/definitions/Error' definitions: + RepoNotice: + type: object + properties: + id: + type: integer + message: + type: string + start_date: + type: string + x-nullable: true + end_date: + type: string + x-nullable: true + created_at: + type: string + updated_at: + type: string + is_warning: + type: boolean + is_archived: + type: boolean LandingPath: type: array description: | diff --git a/landoapi/transplants.py b/landoapi/transplants.py index 73b0fbed..0f3f9f2b 100644 --- a/landoapi/transplants.py +++ b/landoapi/transplants.py @@ -218,7 +218,7 @@ def warning_previously_landed(*, revision, diff, **kwargs): ) -@RevisionWarningCheck(2, "Is not Accepted.") +@RevisionWarningCheck(2, "Revision is not accepted.") def warning_not_accepted(*, revision, **kwargs): status = RevisionStatus.from_status( PhabricatorClient.expect(revision, "fields", "status", "value") @@ -229,7 +229,7 @@ def warning_not_accepted(*, revision, **kwargs): return status.output_name -@RevisionWarningCheck(3, "No reviewer has accepted the current diff.") +@RevisionWarningCheck(3, "Current diff is not accepted.") def warning_reviews_not_current(*, diff, reviewers, **kwargs): for _, r in reviewers.items(): extra = calculate_review_extra_state( @@ -239,11 +239,11 @@ def warning_reviews_not_current(*, diff, reviewers, **kwargs): if r["status"] is ReviewerStatus.ACCEPTED and not extra["for_other_diff"]: return None - return "Has no accepted review on the current diff." + return "Revision has no accepted review on the current diff." @RevisionWarningCheck( - 4, "Is a secure revision and should follow the Security Bug Approval Process." + 4, "Revision is secure and should follow the Security Bug Approval Process." ) def warning_revision_secure(*, revision, secure_project_phid, **kwargs): if secure_project_phid is None: @@ -275,6 +275,15 @@ def warning_revision_missing_testing_tag( ) +@RevisionWarningCheck(6, "Repo has a warning notice.") +def warning_repo_notices(*, revision, repo_notices, **kwargs): + # Filter on notices that are marked as being a warning. + warning_notices = [n for n in repo_notices if n["is_warning"]] + + if warning_notices: + return ". ".join((n["message"] for n in warning_notices)) + + def user_block_no_auth0_email(*, auth0_user, **kwargs): """Check the user has a proper auth0 email.""" return ( @@ -309,6 +318,7 @@ def check_landing_warnings( secure_project_phid, testing_tag_project_phids, testing_policy_phid, + repo_notices, *, revision_warnings=[ warning_blocking_reviews, @@ -317,6 +327,7 @@ def check_landing_warnings( warning_reviews_not_current, warning_revision_secure, warning_revision_missing_testing_tag, + warning_repo_notices, ] ): assessment = TransplantAssessment() @@ -333,6 +344,7 @@ def check_landing_warnings( secure_project_phid=secure_project_phid, testing_tag_project_phids=testing_tag_project_phids, testing_policy_phid=testing_policy_phid, + repo_notices=repo_notices, ) if result is not None: diff --git a/migrations/941d85e9cae8_add_repo_notice.py b/migrations/941d85e9cae8_add_repo_notice.py new file mode 100644 index 00000000..e93058f4 --- /dev/null +++ b/migrations/941d85e9cae8_add_repo_notice.py @@ -0,0 +1,40 @@ +"""add_repo_notice + +Revision ID: 941d85e9cae8 +Revises: 0bf5ff89b7fd +Create Date: 2021-04-19 18:36:15.510007 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "941d85e9cae8" +down_revision = "0bf5ff89b7fd" +branch_labels = () +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "repo_notice", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("repo_identifier", sa.String(length=254), nullable=False), + sa.Column("start_date", sa.DateTime(timezone=True), nullable=True), + sa.Column("end_date", sa.DateTime(timezone=True), nullable=True), + sa.Column("message", sa.Text(), nullable=True), + sa.Column("is_archived", sa.Boolean(), nullable=True), + sa.Column("is_warning", sa.Boolean(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("repo_notice") + # ### end Alembic commands ### diff --git a/requirements.in b/requirements.in index 83f7a2a0..7e466cb9 100644 --- a/requirements.in +++ b/requirements.in @@ -1,11 +1,11 @@ Flask-Alembic==2.0.1 Flask-Caching==1.7.1 -Flask==1.0.2 +Flask==1.1.2 atomicwrites==1.3.0 black==19.3b0 blinker==1.4 celery==4.3.0 -connexion==2.2.0 +connexion==2.7.0 datadog==0.28.0 flake8-bugbear==19.3.0 flake8==3.7.7 diff --git a/requirements.txt b/requirements.txt index dbb2cb05..8cf42a62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -120,9 +120,9 @@ clickclick==1.2.2 \ --hash=sha256:4a890aaa9c3990cfabd446294eb34e3dc89701101ac7b41c1bff85fc210f6d23 \ --hash=sha256:ab8f229fb9906a86634bdfc6fabfc2b665f44804170720db4f6e1d98f8a58f3d # via connexion -connexion==2.2.0 \ - --hash=sha256:24a0f02e601c37de81840a91dff68bbfa48df819ac75b7f8a9cd7e0e2ec8af95 \ - --hash=sha256:3511c369fb1fd1be56ac1d7ba2910f710f59b9cdb36338135afbf0fe51af189e +connexion==2.7.0 \ + --hash=sha256:1ccfac57d4bb7adf4295ba6f5e48f5a1f66057df6a0713417766c9b5235182ee \ + --hash=sha256:5439e9659a89c4380d93a07acfbf3380d70be4130574de8881e5f0dfec7ad0e2 # via -r requirements.in cryptography==3.4.7 \ --hash=sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d \ @@ -189,9 +189,9 @@ flask-sqlalchemy==2.4.0 \ --hash=sha256:0c9609b0d72871c540a7945ea559c8fdf5455192d2db67219509aed680a3d45a \ --hash=sha256:8631bbea987bc3eb0f72b1f691d47bd37ceb795e73b59ab48586d76d75a7c605 # via flask-alembic -flask==1.0.2 \ - --hash=sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48 \ - --hash=sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05 +flask==1.1.2 \ + --hash=sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060 \ + --hash=sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557 # via # -r requirements.in # connexion @@ -481,7 +481,6 @@ six==1.12.0 \ # via # aws-sam-translator # cfn-lint - # connexion # docker # docker-pycreds # mock diff --git a/tests/test_landing_job.py b/tests/test_landing_job.py index 7f60d111..4b6f8ab6 100644 --- a/tests/test_landing_job.py +++ b/tests/test_landing_job.py @@ -71,7 +71,7 @@ def test_cancel_landing_job_fails_not_owner(db, client, landing_job, auth0_mock) def test_cancel_landing_job_fails_not_found(db, client, landing_job, auth0_mock): """Test trying to cancel a job that does not exist.""" response = client.put( - f"/landing_jobs/1", + "/landing_jobs/1", json={"status": LandingJobStatus.CANCELLED.value}, headers=auth0_mock.mock_headers, ) @@ -90,7 +90,9 @@ def test_cancel_landing_job_fails_bad_input(db, client, landing_job, auth0_mock) ) assert response.status_code == 400 - assert response.json["detail"] == ("'IN_PROGRESS' is not one of ['CANCELLED']") + assert response.json["detail"] == ( + "'IN_PROGRESS' is not one of ['CANCELLED'] - 'status'" + ) assert job.status == LandingJobStatus.SUBMITTED diff --git a/tests/test_stacks.py b/tests/test_stacks.py index 99dafb88..6f55c126 100644 --- a/tests/test_stacks.py +++ b/tests/test_stacks.py @@ -594,7 +594,7 @@ def test_get_landable_repos_for_revision_data(phabdouble, mocked_repo_config): assert landable_repos[repo1["phid"]].tree == "mozilla-central" -def test_integrated_stack_endpoint_simple(client, phabdouble, mocked_repo_config): +def test_integrated_stack_endpoint_simple(client, phabdouble, mocked_repo_config, db): repo = phabdouble.repo() unsupported_repo = phabdouble.repo(name="not-mozilla-central") r1 = phabdouble.revision(repo=repo) @@ -627,7 +627,7 @@ def test_integrated_stack_endpoint_simple(client, phabdouble, mocked_repo_config ) -def test_integrated_stack_endpoint_repos(client, phabdouble, mocked_repo_config): +def test_integrated_stack_endpoint_repos(client, phabdouble, mocked_repo_config, db): repo = phabdouble.repo() unsupported_repo = phabdouble.repo(name="not-mozilla-central") r1 = phabdouble.revision(repo=repo)