From 9865410e52d7802a51dbc88734647769b42fc55f Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Mon, 21 Apr 2025 08:09:43 -0400 Subject: [PATCH 01/24] Modify cloudformation template and api handler to create multiple api stack for each version and handle the mapping of version to invoke url --- api/PclusterApiHandler.py | 24 ++- frontend/locales/en/strings.json | 6 + frontend/src/model.tsx | 4 + frontend/src/old-pages/Configure/Create.tsx | 6 + infrastructure/parallelcluster-ui.yaml | 212 ++++++++++++++++---- 5 files changed, 205 insertions(+), 47 deletions(-) diff --git a/api/PclusterApiHandler.py b/api/PclusterApiHandler.py index c29714ab2..121549205 100644 --- a/api/PclusterApiHandler.py +++ b/api/PclusterApiHandler.py @@ -38,7 +38,7 @@ CLIENT_ID = os.getenv("CLIENT_ID") CLIENT_SECRET = os.getenv("CLIENT_SECRET") SECRET_ID = os.getenv("SECRET_ID") -SITE_URL = os.getenv("SITE_URL", API_BASE_URL) +SITE_URL = os.getenv("SITE_URL") SCOPES_LIST = os.getenv("SCOPES_LIST") REGION = os.getenv("AWS_DEFAULT_REGION") TOKEN_URL = os.getenv("TOKEN_URL", f"{AUTH_PATH}/oauth2/token") @@ -62,6 +62,12 @@ if not JWKS_URL: JWKS_URL = os.getenv("JWKS_URL", f"https://cognito-idp.{REGION}.amazonaws.com/{USER_POOL_ID}/" ".well-known/jwks.json") +API_BASE_URL_MAPPING = {} + +for url in API_BASE_URL.split(","): + pair=url.split("=") + API_BASE_URL_MAPPING[pair[0]] = pair[1] + def jwt_decode(token, audience=None, access_token=None): @@ -196,7 +202,7 @@ def get_redirect_uri(): def get_version(): - return {"version": API_VERSION} + return {"version": _get_version(request)} def get_app_config(): return { @@ -233,9 +239,9 @@ def ec2_action(): def get_cluster_config_text(cluster_name, region=None): url = f"/v3/clusters/{cluster_name}" if region: - info_resp = sigv4_request("GET", API_BASE_URL, url, params={"region": region}) + info_resp = sigv4_request("GET", API_BASE_URL_MAPPING[_get_version(request)], url, params={"region": region}) else: - info_resp = sigv4_request("GET", API_BASE_URL, url) + info_resp = sigv4_request("GET", API_BASE_URL_MAPPING[_get_version(request)], url) if info_resp.status_code != 200: abort(info_resp.status_code) @@ -484,7 +490,7 @@ def get_dcv_session(): def get_custom_image_config(): - image_info = sigv4_request("GET", API_BASE_URL, f"/v3/images/custom/{request.args.get('image_id')}").json() + image_info = sigv4_request("GET", API_BASE_URL_MAPPING[_get_version(request)], f"/v3/images/custom/{request.args.get('image_id')}").json() configuration = requests.get(image_info["imageConfiguration"]["url"]) return configuration.text @@ -735,6 +741,10 @@ def _get_params(_request): params.pop("path") return params +def _get_version(request): + params = _get_params(request) + return params.get('version') + pc = Blueprint('pc', __name__) @@ -742,7 +752,7 @@ def _get_params(_request): @authenticated({'admin'}) @validated(params=PCProxyArgs) def pc_proxy_get(): - response = sigv4_request(request.method, API_BASE_URL, request.args.get("path"), _get_params(request)) + response = sigv4_request(request.method, API_BASE_URL_MAPPING[_get_version(request)], request.args.get("path"), _get_params(request)) return response.json(), response.status_code @pc.route('/', methods=['POST','PUT','PATCH','DELETE'], strict_slashes=False) @@ -756,5 +766,5 @@ def pc_proxy(): except: pass - response = sigv4_request(request.method, API_BASE_URL, request.args.get("path"), _get_params(request), body=body) + response = sigv4_request(request.method, API_BASE_URL_MAPPING[_get_version(request)], request.args.get("path"), _get_params(request), body=body) return response.json(), response.status_code diff --git a/frontend/locales/en/strings.json b/frontend/locales/en/strings.json index 855dd9e54..d40bdd7e0 100644 --- a/frontend/locales/en/strings.json +++ b/frontend/locales/en/strings.json @@ -527,6 +527,12 @@ "alreadyExists": "Cluster with name {{clusterName}} already exists. Choose a unique name.", "doesntMatchRegex": "Cluster name '{{clusterName}}' doesn't match the the required format. Enter a name that starts with an alphabetical character and has up to 60 characters. If Slurm accounting is configured, the name can have up to 40 characters. Valid characters: A-Z, a-z, 0-9, and - (hyphen)" }, + "version": { + "label": "Version", + "description": "The ParallelCluster version that will be used for the cluster.", + "placeholder": "Enter your cluster version", + "cannotBeBlank": "Cluster version must not be blank." + }, "region": { "label": "Region", "description": "The AWS Region for the cluster.", diff --git a/frontend/src/model.tsx b/frontend/src/model.tsx index b7959d5cf..eff4d07cc 100644 --- a/frontend/src/model.tsx +++ b/frontend/src/model.tsx @@ -103,6 +103,7 @@ function CreateCluster( clusterConfig: string, region: string, selectedRegion: string, + version: string, dryrun = false, successCallback?: Callback, errorCallback?: Callback, @@ -110,6 +111,7 @@ function CreateCluster( var url = 'api?path=/v3/clusters' url += dryrun ? '&dryrun=true' : '' url += region ? `®ion=${region}` : '' + url += version ? `&version=${version}` : '' var body = { clusterName: clusterName, clusterConfiguration: mapAndApplyTags(clusterConfig), @@ -159,12 +161,14 @@ function UpdateCluster( clusterConfig: any, dryrun = false, forceUpdate: any, + version: any, successCallback?: Callback, errorCallback?: Callback, ) { var url = `api?path=/v3/clusters/${clusterName}` url += dryrun ? '&dryrun=true' : '' url += forceUpdate ? '&forceUpdate=true' : '' + url += version ? `&version=${version}` : '' var body = {clusterConfiguration: clusterConfig} request('put', url, body) .then((response: any) => { diff --git a/frontend/src/old-pages/Configure/Create.tsx b/frontend/src/old-pages/Configure/Create.tsx index 032fb5831..8364a37f9 100644 --- a/frontend/src/old-pages/Configure/Create.tsx +++ b/frontend/src/old-pages/Configure/Create.tsx @@ -96,6 +96,7 @@ function handleCreate( const dryRun = false const region = getState(['app', 'wizard', 'config', 'Region']) const selectedRegion = getState(['app', 'selectedRegion']) + const version = getState(['app', 'wizard', 'version']) setClusterLoadingMsg(clusterName, editing, dryRun) setState(wizardSubmissionLoading, true) @@ -120,6 +121,7 @@ function handleCreate( UpdateCluster( clusterName, clusterConfig, + version, dryRun, forceUpdate, successHandler, @@ -131,6 +133,7 @@ function handleCreate( clusterConfig, region, selectedRegion, + version, dryRun, successHandler, errHandler, @@ -145,6 +148,7 @@ function handleDryRun() { const clusterConfig = getState(configPath) || '' const region = getState(['app', 'wizard', 'config', 'Region']) const selectedRegion = getState(['app', 'selectedRegion']) + const version = getState(['app', 'wizard', 'version']) const dryRun = true setClusterLoadingMsg(clusterName, editing, dryRun) setState(wizardSubmissionLoading, true) @@ -163,6 +167,7 @@ function handleDryRun() { UpdateCluster( clusterName, clusterConfig, + version, dryRun, forceUpdate, successHandler, @@ -174,6 +179,7 @@ function handleDryRun() { clusterConfig, region, selectedRegion, + version, dryRun, successHandler, errHandler, diff --git a/infrastructure/parallelcluster-ui.yaml b/infrastructure/parallelcluster-ui.yaml index a49bf262d..898315502 100644 --- a/infrastructure/parallelcluster-ui.yaml +++ b/infrastructure/parallelcluster-ui.yaml @@ -1,3 +1,4 @@ +Transform: 'AWS::LanguageExtensions' Parameters: AdminUserEmail: Description: Email address of administrative user to setup by default (only with new Cognito instances). @@ -161,17 +162,6 @@ Conditions: - !Not [!Equals [!Ref SNSRole, ""]] UseNewCognito: !Not [ Condition: UseExistingCognito] - UseNonDockerizedPCAPI: - !Not [ Condition: UseDockerizedPCAPI] - UseDockerizedPCAPI: !And - - !Equals ['3', !Select [ 0, !Split ['.', !Ref Version] ] ] # Check PC version major is 3 and PC version minor is 0-5 - - !Or - - !Equals ['0', !Select [ 1, !Split ['.', !Ref Version] ] ] - - !Equals ['1', !Select [ 1, !Split ['.', !Ref Version] ] ] - - !Equals ['2', !Select [ 1, !Split ['.', !Ref Version] ] ] - - !Equals ['3', !Select [ 1, !Split ['.', !Ref Version] ] ] - - !Equals ['4', !Select [ 1, !Split ['.', !Ref Version] ] ] - - !Equals ['5', !Select [ 1, !Split ['.', !Ref Version] ] ] InGovCloud: !Equals ['us-gov-west-1', !Ref "AWS::Region"] UsePermissionBoundary: !Not [!Equals [!Ref PermissionsBoundaryPolicy, '']] UsePermissionBoundaryPCAPI: !Not [!Equals [!Ref PermissionsBoundaryPolicyPCAPI, '']] @@ -208,36 +198,178 @@ Resources: TimeoutInMinutes: 10 - ParallelClusterApi: - Type: AWS::CloudFormation::Stack + Fn::ForEach::ParallelClusterApi: + - ApiVersion + - !Split [",", !Ref Version] + - ParallelClusterApi&{ApiVersion}: + Type: AWS::CloudFormation::Stack + Properties: + Parameters: + PermissionsBoundaryPolicy: !If [ UsePermissionBoundaryPCAPI, !Ref PermissionsBoundaryPolicyPCAPI, !Ref AWS::NoValue ] + IAMRoleAndPolicyPrefix: !If [ UseIAMRoleAndPolicyPrefix, !Ref IAMRoleAndPolicyPrefix, !Ref AWS::NoValue ] + ParallelClusterFunctionAdditionalPolicies: !If [ UseAdditionalPoliciesPCAPI, !Ref AdditionalPoliciesPCAPI, !Ref AWS::NoValue ] + ApiDefinitionS3Uri: !Sub s3://${AWS::Region}-aws-parallelcluster/parallelcluster/${ApiVersion}/api/ParallelCluster.openapi.yaml + CreateApiUserRole: False + EnableIamAdminAccess: True + VpcEndpointId: !If [ IsPrivate, !Ref VpcEndpointId, !Ref AWS::NoValue ] + TemplateURL: !Sub https://${AWS::Region}-aws-parallelcluster.s3.${AWS::Region}.amazonaws.com/parallelcluster/${ApiVersion}/api/parallelcluster-api.yaml + TimeoutInMinutes: 30 + Tags: + - Key: 'parallelcluster:api-id' + Value: !Ref ApiGatewayRestApi + +# ParallelClusterApi: +# Type: AWS::CloudFormation::Stack +# Properties: +# Parameters: +# PermissionsBoundaryPolicy: !If [ UsePermissionBoundaryPCAPI, !Ref PermissionsBoundaryPolicyPCAPI, !Ref AWS::NoValue ] +# IAMRoleAndPolicyPrefix: !If [ UseIAMRoleAndPolicyPrefix, !Ref IAMRoleAndPolicyPrefix, !Ref AWS::NoValue ] +# ParallelClusterFunctionAdditionalPolicies: !If [ UseAdditionalPoliciesPCAPI, !Ref AdditionalPoliciesPCAPI, !Ref AWS::NoValue ] +# ApiDefinitionS3Uri: !Sub s3://${AWS::Region}-aws-parallelcluster/parallelcluster/${Version}/api/ParallelCluster.openapi.yaml +# CreateApiUserRole: False +# EnableIamAdminAccess: True +# VpcEndpointId: !If [ IsPrivate, !Ref VpcEndpointId, !Ref AWS::NoValue ] +# ImageBuilderSubnetId: !If +# - UseNonDockerizedPCAPI +# - !Ref AWS::NoValue +# - Fn::If: +# - NonDefaultVpc +# - !Ref ImageBuilderSubnetId +# - !Ref AWS::NoValue +# ImageBuilderVpcId: !If +# - UseNonDockerizedPCAPI +# - !Ref AWS::NoValue +# - Fn::If: +# - NonDefaultVpc +# - !Ref ImageBuilderVpcId +# - !Ref AWS::NoValue +# TemplateURL: !Sub https://${AWS::Region}-aws-parallelcluster.s3.${AWS::Region}.amazonaws.com/parallelcluster/${Version}/api/parallelcluster-api.yaml +# TimeoutInMinutes: 30 + + ApiVersionMap: + Type: Custom::ApiVersionMap Properties: - Parameters: - PermissionsBoundaryPolicy: !If [ UsePermissionBoundaryPCAPI, !Ref PermissionsBoundaryPolicyPCAPI, !Ref AWS::NoValue ] - IAMRoleAndPolicyPrefix: !If [ UseIAMRoleAndPolicyPrefix, !Ref IAMRoleAndPolicyPrefix, !Ref AWS::NoValue ] - ParallelClusterFunctionAdditionalPolicies: !If [ UseAdditionalPoliciesPCAPI, !Ref AdditionalPoliciesPCAPI, !Ref AWS::NoValue ] - ApiDefinitionS3Uri: !Sub s3://${AWS::Region}-aws-parallelcluster/parallelcluster/${Version}/api/ParallelCluster.openapi.yaml - CreateApiUserRole: False - EnableIamAdminAccess: True - VpcEndpointId: !If [ IsPrivate, !Ref VpcEndpointId, !Ref AWS::NoValue ] - ImageBuilderSubnetId: !If - - UseNonDockerizedPCAPI - - !Ref AWS::NoValue - - Fn::If: - - NonDefaultVpc - - !Ref ImageBuilderSubnetId - - !Ref AWS::NoValue - ImageBuilderVpcId: !If - - UseNonDockerizedPCAPI - - !Ref AWS::NoValue - - Fn::If: - - NonDefaultVpc - - !Ref ImageBuilderVpcId - - !Ref AWS::NoValue - TemplateURL: !Sub https://${AWS::Region}-aws-parallelcluster.s3.${AWS::Region}.amazonaws.com/parallelcluster/${Version}/api/parallelcluster-api.yaml - TimeoutInMinutes: 30 + ServiceToken: !GetAtt ApiVersionMapFunction.Arn + Version: !Ref Version + ApiGatewayRestApiId: !Ref ApiGatewayRestApi + + ApiVersionMapFunction: + Type: AWS::Lambda::Function + Properties: + Handler: index.handler + Runtime: python3.12 + TracingConfig: + Mode: Active + Role: !GetAtt ApiVersionMapFunctionRole.Arn + Code: + ZipFile: | + import boto3 + import cfnresponse + import os + import re + import time + + def handler(event, context): + response_data = {} + response_status = cfnresponse.SUCCESS + reason = None + try: + + if event['RequestType'] in ['Create', 'Update']: + response_data["Message"] = "Resource creation successful!" + cfn = boto3.client('cloudformation') + result = "" + + api_id = event['ResourceProperties'].get('ApiGatewayRestApiId') + print(f"ApiGatewayRestApiId: {api_id}") + versions = event['ResourceProperties'].get('Version').split(",") + print(f"Versions: {versions}") + + paginator = cfn.get_paginator('list_stacks') + for page in paginator.paginate( + StackStatusFilter=['CREATE_COMPLETE', 'UPDATE_COMPLETE', 'CREATE_IN_PROGRESS'] + ): + for stack in page['StackSummaries']: + try: + # Check if stack name matches the pattern + print(f"Checking stack {stack['StackName']}") + + for version in versions: + clean_version = version.replace('.','') + pattern = f'.*-ParallelClusterApi{clean_version}-[^-]*$' + stack_pattern = re.compile(pattern) + match = stack_pattern.match(stack['StackName']) + if not match: + continue + + # Get stack details including tags + stack_response = cfn.describe_stacks( + StackName=stack['StackName'] + ) + + stack_tags = stack_response['Stacks'][0].get('Tags', []) + print(f"Match: {match}, tags: {stack_tags}") + # Check if stack has the specific tag and value + for tag in stack_tags: + if (tag.get('Key') == 'parallelcluster:api-id' and + tag.get('Value') == api_id): + # Get stack outputs + waiter = cfn.get_waiter('stack_create_complete') + waiter.wait(StackName=stack['StackName']) + stack_response = cfn.describe_stacks( + StackName=stack['StackName'] + ) + print(f"Found stack {stack['StackName']} with outputs: {stack_response['Stacks'][0]['Outputs']}") + + for output in stack_response['Stacks'][0]['Outputs']: + if output['OutputKey'] == 'ParallelClusterApiInvokeUrl': + # Construct the result string + result = f"{result}{version}={output['OutputValue']}," + print(f"Version={version}, ApiURL={output['OutputValue']}") + break + + except Exception as e: + print(f"Error processing stack {stack['StackName']}: {str(e)}") + continue + print(f"Result: {result}") + + response_data = {"ApiVersionMapping": result} + cfnresponse.send(event, context, cfnresponse.SUCCESS, response_data, result) + + except Exception as e: + response_status = cfnresponse.FAILED + reason = "Failed {}: {}".format(event["RequestType"], e) + cfnresponse.send(event, context, response_status, response_data, reason) + + Timeout: 300 + MemorySize: 128 + + ApiVersionMapFunctionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: CloudFormationAccess + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - cloudformation:ListStacks + - cloudformation:DescribeStacks + Resource: '*' ParallelClusterUIFun: Type: AWS::Lambda::Function + DependsOn: ApiVersionMap Properties: Role: !GetAtt ParallelClusterUIUserRole.Arn PackageType: Image @@ -256,7 +388,7 @@ Resources: - !Ref AWS::NoValue Environment: Variables: - API_BASE_URL: !GetAtt [ ParallelClusterApi, Outputs.ParallelClusterApiInvokeUrl ] + API_BASE_URL: !GetAtt ApiVersionMap.ApiVersionMapping API_VERSION: !Ref Version SITE_URL: !If - UseCustomDomain @@ -580,7 +712,7 @@ Resources: PermissionsBoundary: !If [UsePermissionBoundary, !Ref PermissionsBoundaryPolicy, !Ref 'AWS::NoValue'] PrivateEcrRepository: - DependsOn: ParallelClusterApi + DependsOn: ParallelClusterApi3120 Type: AWS::ECR::Repository Properties: RepositoryName: !Sub @@ -899,7 +1031,7 @@ Resources: Effect: Allow Resource: !Sub - arn:${AWS::Partition}:execute-api:${AWS::Region}:${AWS::AccountId}:${PCApiGateway}/*/* - - { PCApiGateway: !Select [2, !Split ['/', !Select [0, !Split ['.', !GetAtt [ ParallelClusterApi, Outputs.ParallelClusterApiInvokeUrl ]]]]] } + - { PCApiGateway: !Select [2, !Split ['/', !Select [0, !Split ['.', !GetAtt [ ParallelClusterApi3120, Outputs.ParallelClusterApiInvokeUrl ]]]]] } CognitoPolicy: Type: AWS::IAM::ManagedPolicy From f84a92d5d7ec8b7dd518e5effd7f00e5d8f1aa0b Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Mon, 21 Apr 2025 09:57:45 -0400 Subject: [PATCH 02/24] Add version field to cluster create --- api/PclusterApiHandler.py | 31 ++++++----- frontend/src/old-pages/Clusters/Actions.tsx | 2 +- frontend/src/old-pages/Clusters/Details.tsx | 2 +- .../src/old-pages/Clusters/Properties.tsx | 1 + frontend/src/old-pages/Configure/Cluster.tsx | 2 + .../Configure/Cluster/ClusterVersionField.tsx | 53 +++++++++++++++++++ infrastructure/parallelcluster-ui.yaml | 5 +- 7 files changed, 77 insertions(+), 19 deletions(-) create mode 100644 frontend/src/old-pages/Configure/Cluster/ClusterVersionField.tsx diff --git a/api/PclusterApiHandler.py b/api/PclusterApiHandler.py index 121549205..e1eafc50f 100644 --- a/api/PclusterApiHandler.py +++ b/api/PclusterApiHandler.py @@ -33,6 +33,7 @@ AUTH_PATH = os.getenv("AUTH_PATH") API_BASE_URL = os.getenv("API_BASE_URL") API_VERSION = os.getenv("API_VERSION", "3.1.0") +DEFAULT_API_VERSION = API_VERSION.split(",")[0] API_USER_ROLE = os.getenv("API_USER_ROLE") OIDC_PROVIDER = os.getenv("OIDC_PROVIDER") CLIENT_ID = os.getenv("CLIENT_ID") @@ -65,8 +66,9 @@ API_BASE_URL_MAPPING = {} for url in API_BASE_URL.split(","): - pair=url.split("=") - API_BASE_URL_MAPPING[pair[0]] = pair[1] + if url: + pair=url.split("=") + API_BASE_URL_MAPPING[pair[0]] = pair[1] @@ -171,7 +173,7 @@ def authenticate(groups): if (not groups): return abort(403) - + jwt_roles = set(decoded.get(USER_ROLES_CLAIM, [])) groups_granted = groups.intersection(jwt_roles) if len(groups_granted) == 0: @@ -197,12 +199,12 @@ def get_scopes_list(): def get_redirect_uri(): return f"{SITE_URL}/login" - + # Local Endpoints def get_version(): - return {"version": _get_version(request)} + return {"version": API_VERSION} def get_app_config(): return { @@ -371,7 +373,7 @@ def sacct(): user, f"sacct {sacct_args} --json " + "| jq -c .jobs[0:120]\\|\\map\\({name,user,partition,state,job_id,exit_code\\}\\)", - ) + ) if type(accounting) is tuple: return accounting else: @@ -602,9 +604,9 @@ def _get_identity_from_token(decoded, claims): identity["username"] = decoded["username"] for claim in claims: - if claim in decoded: - identity["attributes"][claim] = decoded[claim] - + if claim in decoded: + identity["attributes"][claim] = decoded[claim] + return identity def get_identity(): @@ -741,9 +743,10 @@ def _get_params(_request): params.pop("path") return params -def _get_version(request): - params = _get_params(request) - return params.get('version') +def _get_version(v): + if v and str(v) in API_VERSION: + return str(v) + return DEFAULT_API_VERSION pc = Blueprint('pc', __name__) @@ -752,7 +755,7 @@ def _get_version(request): @authenticated({'admin'}) @validated(params=PCProxyArgs) def pc_proxy_get(): - response = sigv4_request(request.method, API_BASE_URL_MAPPING[_get_version(request)], request.args.get("path"), _get_params(request)) + response = sigv4_request(request.method, API_BASE_URL_MAPPING[_get_version(request.args.get("version"))], request.args.get("path"), _get_params(request)) return response.json(), response.status_code @pc.route('/', methods=['POST','PUT','PATCH','DELETE'], strict_slashes=False) @@ -766,5 +769,5 @@ def pc_proxy(): except: pass - response = sigv4_request(request.method, API_BASE_URL_MAPPING[_get_version(request)], request.args.get("path"), _get_params(request), body=body) + response = sigv4_request(request.method, API_BASE_URL_MAPPING[_get_version(request.args.get("version"))], request.args.get("path"), _get_params(request), body=body) return response.json(), response.status_code diff --git a/frontend/src/old-pages/Clusters/Actions.tsx b/frontend/src/old-pages/Clusters/Actions.tsx index 66765a1f1..add723ba9 100644 --- a/frontend/src/old-pages/Clusters/Actions.tsx +++ b/frontend/src/old-pages/Clusters/Actions.tsx @@ -75,7 +75,7 @@ export default function Actions() { clusterStatus === ClusterStatus.DeleteInProgress || clusterStatus === ClusterStatus.UpdateInProgress || clusterStatus === ClusterStatus.CreateFailed || - clusterVersion !== apiVersion + !apiVersion.split(",").includes(clusterVersion) const isStartFleetDisabled = fleetStatus !== 'STOPPED' const isStopFleetDisabled = fleetStatus !== 'RUNNING' const isDeleteDisabled = diff --git a/frontend/src/old-pages/Clusters/Details.tsx b/frontend/src/old-pages/Clusters/Details.tsx index a94faafe8..6dc34cbba 100644 --- a/frontend/src/old-pages/Clusters/Details.tsx +++ b/frontend/src/old-pages/Clusters/Details.tsx @@ -56,7 +56,7 @@ export default function ClusterTabs() { return cluster ? ( <> - {cluster.version !== apiVersion ? ( + {!apiVersion.split(",").includes(cluster.version) ? ( {t('cluster.editAlert')} ) : null} + = + useCallback(({detail}) => { + setState(clusterVersionPath, detail.value) + }, []) + + return ( + + + + ) +} diff --git a/infrastructure/parallelcluster-ui.yaml b/infrastructure/parallelcluster-ui.yaml index 898315502..327d2ea7a 100644 --- a/infrastructure/parallelcluster-ui.yaml +++ b/infrastructure/parallelcluster-ui.yaml @@ -1029,9 +1029,8 @@ Resources: - Action: - execute-api:Invoke Effect: Allow - Resource: !Sub - - arn:${AWS::Partition}:execute-api:${AWS::Region}:${AWS::AccountId}:${PCApiGateway}/*/* - - { PCApiGateway: !Select [2, !Split ['/', !Select [0, !Split ['.', !GetAtt [ ParallelClusterApi3120, Outputs.ParallelClusterApiInvokeUrl ]]]]] } + Resource: !Sub "arn:${AWS::Partition}:execute-api:${AWS::Region}:${AWS::AccountId}:*/*/*" +# - { PCApiGateway: !Select [2, !Split ['/', !Select [0, !Split ['.', !GetAtt [ ParallelClusterApi3120, Outputs.ParallelClusterApiInvokeUrl ]]]]] } CognitoPolicy: Type: AWS::IAM::ManagedPolicy From 96dfef425a4d4ed8af656500c25e67630acc78e9 Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Wed, 23 Apr 2025 19:53:36 -0400 Subject: [PATCH 03/24] Add dropdown menu to select the image version for the Official Images page --- frontend/locales/en/strings.json | 8 +++ frontend/src/model.tsx | 4 +- .../Images/OfficialImages/OfficialImages.tsx | 71 ++++++++++++++----- 3 files changed, 64 insertions(+), 19 deletions(-) diff --git a/frontend/locales/en/strings.json b/frontend/locales/en/strings.json index d40bdd7e0..f974dc19a 100644 --- a/frontend/locales/en/strings.json +++ b/frontend/locales/en/strings.json @@ -1436,6 +1436,14 @@ "href": "https://docs.aws.amazon.com/parallelcluster/latest/ug/support-policy.html" } }, + "actions": { + "versionSelect": { + "selectedAriaLabel": "Selected version", + "versionText": "Version {{version}}", + "placeholder": "Select a version" + }, + "refresh": "Refresh" + }, "list": { "columns": { "id": "ID", diff --git a/frontend/src/model.tsx b/frontend/src/model.tsx index eff4d07cc..7bb4c4311 100644 --- a/frontend/src/model.tsx +++ b/frontend/src/model.tsx @@ -459,8 +459,8 @@ async function BuildImage(imageId: string, imageConfig: string) { return data } -async function ListOfficialImages(region?: string) { - const url = `api?path=/v3/images/official${region ? `®ion=${region}` : ''}` +async function ListOfficialImages(region?: string, version?: string) { + const url = `api?path=/v3/images/official${region ? `®ion=${region}` : ''}${version ? `&version=${version}` : ''}` try { const {data} = await request('get', url) return data?.images || [] diff --git a/frontend/src/old-pages/Images/OfficialImages/OfficialImages.tsx b/frontend/src/old-pages/Images/OfficialImages/OfficialImages.tsx index f4b451d81..8a3b8fb4a 100644 --- a/frontend/src/old-pages/Images/OfficialImages/OfficialImages.tsx +++ b/frontend/src/old-pages/Images/OfficialImages/OfficialImages.tsx @@ -9,6 +9,8 @@ // OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and // limitations under the License. import React, {useMemo} from 'react' +import {Select, SpaceBetween} from '@cloudscape-design/components' + import {ListOfficialImages} from '../../../model' import {useCollection} from '@cloudscape-design/collection-hooks' @@ -25,7 +27,7 @@ import { // Components import EmptyState from '../../../components/EmptyState' import {useQuery} from 'react-query' -import {useState} from '../../../store' +import {useState, setState} from '../../../store' import {useHelpPanel} from '../../../components/help-panel/HelpPanel' import {Trans, useTranslation} from 'react-i18next' import TitleDescriptionHelpPanel from '../../../components/help-panel/TitleDescriptionHelpPanel' @@ -39,6 +41,28 @@ type Image = { version: string } +function VersionSelect() { + const {t} = useTranslation() + const versions = useState(['app', 'version', 'full']).split(',') + const [selectedVersion, setSelectedVersion] = React.useState(versions[0]) + + return ( + - - ) -} diff --git a/frontend/src/old-pages/Configure/Configure.tsx b/frontend/src/old-pages/Configure/Configure.tsx index 267271e3d..726c7e170 100644 --- a/frontend/src/old-pages/Configure/Configure.tsx +++ b/frontend/src/old-pages/Configure/Configure.tsx @@ -27,6 +27,7 @@ import { HeadNodePropertiesHelpPanel, headNodeValidate, } from './HeadNode' +import {Version, versionValidate} from './Version' import {Storage, StorageHelpPanel, storageValidate} from './Storage' import { useClusterResourcesLimits, @@ -60,6 +61,7 @@ import InfoLink from '../../components/InfoLink' import {useFeatureFlag} from '../../feature-flags/useFeatureFlag' const validators: {[key: string]: (...args: any[]) => boolean} = { + version: versionValidate, cluster: clusterValidate, headNode: headNodeValidate, storage: storageValidate, @@ -89,6 +91,7 @@ function clearWizardState( clearErrorsOnly: boolean, ) { if (!clearErrorsOnly) { + clearState(['app', 'wizard', 'version']) clearState(['app', 'wizard', 'config']) clearState(['app', 'wizard', 'clusterConfigYaml']) clearState(['app', 'wizard', 'clusterName']) @@ -251,6 +254,11 @@ function Configure() { refreshing || loadingExistingConfiguration || isSubmittingWizard } steps={[ + { + title: t('wizard.version.title'), + description: t('wizard.version.description'), + content: , + }, { title: t('wizard.cluster.title'), description: t('wizard.cluster.description'), diff --git a/frontend/src/old-pages/Configure/Version.tsx b/frontend/src/old-pages/Configure/Version.tsx new file mode 100644 index 000000000..4898bbc30 --- /dev/null +++ b/frontend/src/old-pages/Configure/Version.tsx @@ -0,0 +1,64 @@ +import React, {useEffect} from 'react' +import i18next from 'i18next' +import {useTranslation} from 'react-i18next' +import {Container, Header, SpaceBetween} from '@cloudscape-design/components' +import {ClusterVersionField} from './Version/ClusterVersionField' +import InfoLink from '../../components/InfoLink' +import TitleDescriptionHelpPanel from '../../components/help-panel/TitleDescriptionHelpPanel' +import {getState, setState, clearState, useState} from '../../store' + +const errorsPath = ['app', 'wizard', 'errors', 'version'] + +function versionValidate() { + const version = getState(['app', 'wizard', 'version']) + if (!version) { + setState( + [...errorsPath, 'version'], + i18next.t('wizard.version.validation.versionSelect'), + ) + return false + } + clearState([...errorsPath, 'version']) + return true +} + + +function Version() { + const {t} = useTranslation() + + useEffect(() => { + // Clear any existing version when the component mounts + setState(['app', 'wizard', 'version'], null) + }, []) + + return ( + + } />} + > + {t('wizard.version.label')} + + } + > + + + + + + ) +} + +const VersionHelpPanel = () => { + const {t} = useTranslation() + return ( + + ) +} + +export {Version, versionValidate} \ No newline at end of file diff --git a/frontend/src/old-pages/Configure/Version/ClusterVersionField.tsx b/frontend/src/old-pages/Configure/Version/ClusterVersionField.tsx new file mode 100644 index 000000000..c399fab14 --- /dev/null +++ b/frontend/src/old-pages/Configure/Version/ClusterVersionField.tsx @@ -0,0 +1,77 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +// with the License. A copy of the License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +// OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and +// limitations under the License. + +import { + FormField, + Input, + InputProps, + NonCancelableCustomEvent, + Select, + SelectProps +} from '@cloudscape-design/components' +import {NonCancelableEventHandler} from '@cloudscape-design/components/internal/events' +import {useCallback} from 'react' +import {useTranslation} from 'react-i18next' +import {setState, useState} from '../../../store' + +const clusterVersionPath = ['app', 'wizard', 'version'] +const clusterVersionErrorPath = [ + 'app', + 'wizard', + 'errors', + 'source', + 'version', +] +const editingPath = ['app', 'wizard', 'editing'] + +interface ClusterVersionFieldProps { + hideLabel?: boolean; +} + +export function ClusterVersionField({ hideLabel = false }: ClusterVersionFieldProps) { + const {t} = useTranslation() + const version = useState(clusterVersionPath) || '' + const clusterVersionError = useState(clusterVersionErrorPath) + const editing = !!useState(editingPath) + const versions = useState(['app', 'version', 'full']) + + const options = versions.map((version: any) => ({ + label: version, + value: version + })) + + const onChange = useCallback( + ({detail}: NonCancelableCustomEvent) => { + setState(clusterVersionPath, detail.selectedOption.value) + }, + [] + ) + + return ( + + { setSelectedVersion(detail.selectedOption.value) From 4199108c4bd27f2da7d83b0f8a47d63c78f1ce2b Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Mon, 5 May 2025 13:16:34 -0400 Subject: [PATCH 14/24] Wrap url mapping logic in a function --- api/PclusterApiHandler.py | 38 +++++++++++++++----------- api/tests/test_pcluster_api_handler.py | 12 ++++++-- frontend/locales/en/strings.json | 2 +- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/api/PclusterApiHandler.py b/api/PclusterApiHandler.py index dbf689bcb..e5ac14e66 100644 --- a/api/PclusterApiHandler.py +++ b/api/PclusterApiHandler.py @@ -32,14 +32,13 @@ USER_POOL_ID = os.getenv("USER_POOL_ID") AUTH_PATH = os.getenv("AUTH_PATH") API_BASE_URL = os.getenv("API_BASE_URL") -API_VERSION = sorted(os.getenv("API_VERSION", "3.1.0").split(","), key=lambda x: [-int(n) for n in x.split('.')]) +API_VERSION = sorted(os.getenv("API_VERSION", "3.1.0").strip().split(","), key=lambda x: [-int(n) for n in x.split('.')]) DEFAULT_API_VERSION = API_VERSION[0] API_USER_ROLE = os.getenv("API_USER_ROLE") OIDC_PROVIDER = os.getenv("OIDC_PROVIDER") CLIENT_ID = os.getenv("CLIENT_ID") CLIENT_SECRET = os.getenv("CLIENT_SECRET") SECRET_ID = os.getenv("SECRET_ID") -SITE_URL = os.getenv("SITE_URL") SCOPES_LIST = os.getenv("SCOPES_LIST") REGION = os.getenv("AWS_DEFAULT_REGION") TOKEN_URL = os.getenv("TOKEN_URL", f"{AUTH_PATH}/oauth2/token") @@ -49,6 +48,7 @@ AUDIENCE = os.getenv("AUDIENCE") USER_ROLES_CLAIM = os.getenv("USER_ROLES_CLAIM", "cognito:groups") SSM_LOG_GROUP_NAME = os.getenv("SSM_LOG_GROUP_NAME") +ARG_VERSION="version" try: if (not USER_POOL_ID or USER_POOL_ID == "") and SECRET_ID: @@ -63,13 +63,18 @@ if not JWKS_URL: JWKS_URL = os.getenv("JWKS_URL", f"https://cognito-idp.{REGION}.amazonaws.com/{USER_POOL_ID}/" ".well-known/jwks.json") -API_BASE_URL_MAPPING = {} -if API_BASE_URL: - for url in API_BASE_URL.split(","): - if url: - pair=url.split("=") - API_BASE_URL_MAPPING[pair[0]] = pair[1] +def create_url_map(url_list): + url_map = {} + if url_list: + for url in url_list.split(","): + if url: + pair=url.split("=") + url_map[pair[0]] = pair[1] + return url_map + +API_BASE_URL_MAPPING = create_url_map(API_BASE_URL) +SITE_URL = os.getenv("SITE_URL", API_BASE_URL_MAPPING.get(DEFAULT_API_VERSION)) @@ -242,9 +247,9 @@ def ec2_action(): def get_cluster_config_text(cluster_name, region=None): url = f"/v3/clusters/{cluster_name}" if region: - info_resp = sigv4_request("GET", get_base_url(request.args.get("version")), url, params={"region": region}) + info_resp = sigv4_request("GET", get_base_url(request), url, params={"region": region}) else: - info_resp = sigv4_request("GET", get_base_url(request.args.get("version")), url) + info_resp = sigv4_request("GET", get_base_url(request), url) if info_resp.status_code != 200: abort(info_resp.status_code) @@ -493,7 +498,7 @@ def get_dcv_session(): def get_custom_image_config(): - image_info = sigv4_request("GET", get_base_url(request.args.get("version")), f"/v3/images/custom/{request.args.get('image_id')}").json() + image_info = sigv4_request("GET", get_base_url(request), f"/v3/images/custom/{request.args.get('image_id')}").json() configuration = requests.get(image_info["imageConfiguration"]["url"]) return configuration.text @@ -744,9 +749,10 @@ def _get_params(_request): params.pop("path") return params -def get_base_url(v): - if v and str(v) in API_VERSION: - return API_BASE_URL_MAPPING[str(v)] +def get_base_url(request): + version = request.args.get(ARG_VERSION) + if version and str(version) in API_VERSION: + return API_BASE_URL_MAPPING[str(version)] return API_BASE_URL_MAPPING[DEFAULT_API_VERSION] @@ -756,7 +762,7 @@ def get_base_url(v): @authenticated({'admin'}) @validated(params=PCProxyArgs) def pc_proxy_get(): - response = sigv4_request(request.method, get_base_url(request.args.get("version")), request.args.get("path"), _get_params(request)) + response = sigv4_request(request.method, get_base_url(request), request.args.get("path"), _get_params(request)) return response.json(), response.status_code @pc.route('/', methods=['POST','PUT','PATCH','DELETE'], strict_slashes=False) @@ -770,5 +776,5 @@ def pc_proxy(): except: pass - response = sigv4_request(request.method, get_base_url(request.args.get("version")), request.args.get("path"), _get_params(request), body=body) + response = sigv4_request(request.method, get_base_url(request), request.args.get("path"), _get_params(request), body=body) return response.json(), response.status_code diff --git a/api/tests/test_pcluster_api_handler.py b/api/tests/test_pcluster_api_handler.py index 75d381011..812b2c835 100644 --- a/api/tests/test_pcluster_api_handler.py +++ b/api/tests/test_pcluster_api_handler.py @@ -1,5 +1,10 @@ from unittest import mock -from api.PclusterApiHandler import login, get_base_url +from api.PclusterApiHandler import login, get_base_url, create_url_map + +class MockRequest: + cookies = {'int_value': 100} + args = {'version': '3.12.0'} + json = {'username': 'user@email.com'} @mock.patch("api.PclusterApiHandler.requests.post") @@ -33,5 +38,8 @@ def test_get_base_url(monkeypatch): monkeypatch.setattr('api.PclusterApiHandler.API_BASE_URL', '3.12.0=https://example.com,3.11.0=https://example1.com,') monkeypatch.setattr('api.PclusterApiHandler.API_BASE_URL_MAPPING', {'3.12.0': 'https://example.com', '3.11.0': 'https://example1.com'}) - assert 'https://example.com' == get_base_url('3.12.0') + assert 'https://example.com' == get_base_url(MockRequest()) + +def test_create_url_map(): + assert {'3.12.0': 'https://example.com', '3.11.0': 'https://example1.com'} == create_url_map('3.12.0=https://example.com,3.11.0=https://example1.com,') diff --git a/frontend/locales/en/strings.json b/frontend/locales/en/strings.json index 5b736d79a..1c0e6c5a1 100644 --- a/frontend/locales/en/strings.json +++ b/frontend/locales/en/strings.json @@ -496,7 +496,7 @@ "version": { "label": "Cluster Version", "title": "Version", - "placeholder": "Select your cluster version", + "placeholder": "Cluster version", "description": "Select the AWS ParallelCluster version to use for this cluster.", "help": { "main": "Choose the version of AWS ParallelCluster to use for creating and managing your cluster." From 40fd88c9c532925d28c06d51c9cc90dd349ef05d Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Tue, 13 May 2025 11:08:00 -0400 Subject: [PATCH 15/24] Add cluster version selection to custom image build. --- frontend/locales/en/strings.json | 4 ++- frontend/src/model.tsx | 3 ++- .../src/old-pages/Configure/Configure.tsx | 2 ++ .../Images/CustomImages/ImageBuildDialog.tsx | 27 ++++++++++++++----- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/frontend/locales/en/strings.json b/frontend/locales/en/strings.json index 1c0e6c5a1..f9a1250f8 100644 --- a/frontend/locales/en/strings.json +++ b/frontend/locales/en/strings.json @@ -101,7 +101,7 @@ "ariaLabel": "Info" }, "cluster": { - "editAlert": "This cluster cannot be edited as it was created using a different version of AWS ParallelCluster.", + "editAlert": "This cluster cannot be edited as it was created using an incompatible version of AWS ParallelCluster.", "tabs": { "details": "Details", "instances": "Instances", @@ -1419,6 +1419,8 @@ }, "dialogs": { "buildImage": { + "versionLabel": "Version", + "versionPlaceholder": "Select version", "closeAriaLabel": "Close", "title": "Image configuration", "cancel": "Cancel", diff --git a/frontend/src/model.tsx b/frontend/src/model.tsx index 7bb4c4311..1f7a2521e 100644 --- a/frontend/src/model.tsx +++ b/frontend/src/model.tsx @@ -448,8 +448,9 @@ function GetCustomImageConfiguration(imageId: any, callback?: Callback) { }) } -async function BuildImage(imageId: string, imageConfig: string) { +async function BuildImage(imageId: string, imageConfig: string, version: string) { var url = 'api?path=/v3/images/custom' + url += version ? `&version=${version}` : '' var body = {imageId: imageId, imageConfiguration: imageConfig} const {data} = await request('post', url, body) notify(`Successfully queued build for ${imageId}.`, 'success') diff --git a/frontend/src/old-pages/Configure/Configure.tsx b/frontend/src/old-pages/Configure/Configure.tsx index 01849fc01..81f38bba7 100644 --- a/frontend/src/old-pages/Configure/Configure.tsx +++ b/frontend/src/old-pages/Configure/Configure.tsx @@ -116,6 +116,8 @@ function Configure() { const [refreshing, setRefreshing] = React.useState(false) let navigate = useNavigate() + setState(['app', 'wizard', 'version'], null) + const clusterPath = ['clusters', 'index', clusterName] const fleetStatus: ComputeFleetStatus = useState([ ...clusterPath, diff --git a/frontend/src/old-pages/Images/CustomImages/ImageBuildDialog.tsx b/frontend/src/old-pages/Images/CustomImages/ImageBuildDialog.tsx index a9e225ca9..b036dd1fa 100644 --- a/frontend/src/old-pages/Images/CustomImages/ImageBuildDialog.tsx +++ b/frontend/src/old-pages/Images/CustomImages/ImageBuildDialog.tsx @@ -21,6 +21,7 @@ import { Header, Input, Modal, + Select, SpaceBetween, } from '@cloudscape-design/components' @@ -61,6 +62,8 @@ export default function ImageBuildDialog() { const errors = useState([...imageBuildPath, 'errors']) const imageId = useState([...imageBuildPath, 'imageId']) const pending = useState([...imageBuildPath, 'pending']) + const versions = useState(['app', 'version', 'full']) + const [selectedVersion, setSelectedVersion] = React.useState(versions[0]) let validated = useState([...buildImageErrorsPath, 'validated']) @@ -85,7 +88,7 @@ export default function ImageBuildDialog() { setState([...imageBuildPath, 'pending'], true) if (buildImageValidate(missingImageIdError)) { try { - await BuildImage(imageId, imageConfig) + await BuildImage(imageId, imageConfig, selectedVersion) handleClose() } catch (error: unknown) { if ((error as AxiosError).response) { @@ -173,14 +176,24 @@ export default function ImageBuildDialog() { /> - { + + +