Compile slurm for the os of the submitter (#182)

So that it can submit jobs to a cluster that uses a different OS.

Automate the configuration of the cluster manager and submitters in RES.

Fix errors in configuration scripts.

Add a parameter to the ParallelCluster custom resource that has a hash
of all of the config file contents so that the resource gets updated
whenever anything changes.
This current issue with this implementation is that the ParallelCluster
still doesn't change so even though the custom resource is updated,
the cluster doesn't because it's configuration is unchanged.

Fix a bug in configuring licenses.
After initial configuration the playbook was failing on subsequent updates.

Support custom munge key which must be specified in secretsmanager

Try to allow RES /home to be mounted, but currently causes a validation error.

Update EDA instance types to the latest and so that all memory sizes get built.

Restrict lambda function IAM roles and cleanup cfn_nag errors and warnings.

Resolves #177
Resolves #178
Resolves #180
Resolves #181

Author: cartalla

.gitignore

@@ -4,3 +4,4 @@ _site
 site/
 .vscode/
 source/resources/parallel-cluster/config/build-files/*/*/parallelcluster-*.yml
+security_scan/cfn_nag.log

Makefile

@@ -16,6 +16,9 @@ local-docs: .mkdocs_venv/bin/activate
 github-docs: .mkdocs_venv/bin/activate
 	source .mkdocs_venv/bin/activate; mkdocs gh-deploy --strict
 
+security_scan:
+	security_scan/security_scan.sh
+
 test:
 	pytest -x -v tests
 

security_scan/security_scan.sh

@@ -5,9 +5,9 @@
 scriptdir=$(dirname $(readlink -f $0))
 
 cd $scriptdir/..
-./install.sh --config-file ~/slurm/slurm_eda_az1.yml --cdk-cmd synth
+./install.sh --config-file ~/slurm/res-eda/res-eda-pc-3-7-2-centos7-x86-config.yml --cdk-cmd synth
 
-cfn_nag_scan --input-path $scriptdir/../source/cdk.out/slurmedaaz1.template.json --deny-list-path $scriptdir/cfn_nag-deny-list.yml --fail-on-warnings &> $scriptdir/cfn_nag.log
+cfn_nag_scan --input-path $scriptdir/../source/cdk.out/res-eda-pc-3-7-2-centos7-x86-config.template.json --deny-list-path $scriptdir/cfn_nag-deny-list.yml --fail-on-warnings &> $scriptdir/cfn_nag.log
 
 cd $scriptdir
 if [ ! -e $scriptdir/bandit-env ]; then

source/cdk/cdk_slurm_stack.py

@@ -35,8 +35,11 @@
 logger.setLevel(logging.INFO)
 
 # MIN_PARALLEL_CLUSTER_VERSION
+# 3.2.0:
+#     * Add support for memory-based job scheduling in Slurm
 # 3.3.0:
 #     * Add support for multiple instance types in a compute resource
+#     * Add new configuration section Scheduling/SlurmSettings/Database to enable accounting functionality in Slurm.
 # 3.4.0:
 #     * Add support for launching nodes across multiple availability zones to increase capacity availability
 #     * Add support for specifying multiple subnets for each queue to increase capacity availability
@@ -53,6 +56,7 @@
 # 3.7.0:
 #     * Login Nodes
 #     * Add support for configurable node weights within queue
+#     * Allow memory-based scheduling when multiple instance types are specified for a Slurm Compute Resource.
 # 3.7.1:
 #     * Fix pmix CVE
 #     * Use Slurm 23.02.5
@@ -84,11 +88,19 @@
 }
 PARALLEL_CLUSTER_SLURM_VERSIONS = {
     # This can be found on the head node at /etc/chef/local-mode-cache/cache/
-    '3.6.0': '23-02-2-1', # confirmed
-    '3.6.1': '23-02-2-1', # confirmed
-    '3.7.0': '23-02-4-1', # confirmed
-    '3.7.1': '23-02-5-1', # confirmed
-    '3.7.2': '23-02-6-1', # confirmed
+    '3.6.0':   '23.02.2', # confirmed
+    '3.6.1':   '23.02.2', # confirmed
+    '3.7.0':   '23.02.4', # confirmed
+    '3.7.1':   '23.02.5', # confirmed
+    '3.7.2':   '23.02.6', # confirmed
+}
+PARALLEL_CLUSTER_PC_SLURM_VERSIONS = {
+    # This can be found on the head node at /etc/chef/local-mode-cache/cache/
+    '3.6.0':   '23-02-2-1', # confirmed
+    '3.6.1':   '23-02-2-1', # confirmed
+    '3.7.0':   '23-02-4-1', # confirmed
+    '3.7.1':   '23-02-5-1', # confirmed
+    '3.7.2':   '23-02-6-1', # confirmed
 }
 SLURM_REST_API_VERSIONS = {
     '23-02-2-1': '0.0.39',
@@ -110,23 +122,45 @@ def get_parallel_cluster_version(config):
 
 def get_PARALLEL_CLUSTER_MUNGE_VERSION(config):
     parallel_cluster_version = get_parallel_cluster_version(config)
-    munge_version = PARALLEL_CLUSTER_MUNGE_VERSIONS[parallel_cluster_version]
-    return munge_version
+    return PARALLEL_CLUSTER_MUNGE_VERSIONS[parallel_cluster_version]
 
 def get_PARALLEL_CLUSTER_PYTHON_VERSION(config):
     parallel_cluster_version = get_parallel_cluster_version(config)
-    python_version = PARALLEL_CLUSTER_PYTHON_VERSIONS[parallel_cluster_version]
-    return python_version
+    return PARALLEL_CLUSTER_PYTHON_VERSIONS[parallel_cluster_version]
 
 def get_SLURM_VERSION(config):
     parallel_cluster_version = get_parallel_cluster_version(config)
-    slurm_version = PARALLEL_CLUSTER_SLURM_VERSIONS[parallel_cluster_version]
-    return slurm_version
+    return PARALLEL_CLUSTER_SLURM_VERSIONS[parallel_cluster_version]
+
+def get_PC_SLURM_VERSION(config):
+    parallel_cluster_version = get_parallel_cluster_version(config)
+    return PARALLEL_CLUSTER_PC_SLURM_VERSIONS[parallel_cluster_version]
 
 def get_slurm_rest_api_version(config):
-    slurm_version = get_SLURM_VERSION(config)
-    slurm_rest_api_version = SLURM_REST_API_VERSIONS.get(slurm_version, )
-    return slurm_rest_api_version
+    slurm_version = get_PC_SLURM_VERSION(config)
+    return SLURM_REST_API_VERSIONS.get(slurm_version, )
+
+# Feature support
+
+# Version 3.7.0:
+PARALLEL_CLUSTER_SUPPORTS_LOGIN_NODES_VERSION = parse_version('3.7.0')
+def PARALLEL_CLUSTER_SUPPORTS_LOGIN_NODES(parallel_cluster_version):
+    return parallel_cluster_version >= PARALLEL_CLUSTER_SUPPORTS_LOGIN_NODES_VERSION
+
+PARALLEL_CLUSTER_SUPPORTS_MULTIPLE_COMPUTE_RESOURCES_PER_QUEUE_VERSION = parse_version('3.7.0')
+def PARALLEL_CLUSTER_SUPPORTS_MULTIPLE_COMPUTE_RESOURCES_PER_QUEUE(parallel_cluster_version):
+    return parallel_cluster_version >= PARALLEL_CLUSTER_SUPPORTS_MULTIPLE_COMPUTE_RESOURCES_PER_QUEUE_VERSION
+
+PARALLEL_CLUSTER_SUPPORTS_MULTIPLE_INSTANCE_TYPES_PER_COMPUTE_RESOURCE_VERSION = parse_version('3.7.0')
+def PARALLEL_CLUSTER_SUPPORTS_MULTIPLE_INSTANCE_TYPES_PER_COMPUTE_RESOURCE(parallel_cluster_version):
+    return parallel_cluster_version >= PARALLEL_CLUSTER_SUPPORTS_MULTIPLE_INSTANCE_TYPES_PER_COMPUTE_RESOURCE_VERSION
+
+# Unsupported
+def PARALLEL_CLUSTER_SUPPORTS_CUSTOM_MUNGE_KEY(parallel_cluster_version):
+    return False
+
+def PARALLEL_CLUSTER_SUPPORTS_HOME_MOUNT(parallel_cluster_version):
+    return False
 
 # Determine all AWS regions available on the account.
 default_region = environ.get("AWS_DEFAULT_REGION", "us-east-1")
@@ -147,48 +181,88 @@ def get_slurm_rest_api_version(config):
     'AFTER_90_DAYS'
     ]
 
+# By default I've chosen to exclude *7i instance types because they have 50% of the cores as *7z instances with the same memory.
 default_eda_instance_families = [
-    #'c5',                # Mixed depending on size
-    #'c5a',               # AMD EPYC 7R32 3.3 GHz
-    #'c5ad',              # AMD EPYC 7R32 3.3 GHz
+    'c7a',               # AMD EPYC 9R14 Processor 3.7 GHz
+
+    'c7g',               # AWS Graviton3 Processor 2.6 GHz
+    # 'c7gd',              # AWS Graviton3 Processor 2.6 GHz
+    # 'c7gn',              # AWS Graviton3 Processor 2.6 GHz
+
+    # 'c7i',               # Intel Xeon Scalable (Sapphire Rapids) 3.2 GHz
+
+    #'f1',                # Intel Xeon E5-2686 v4 (Broadwell) 2.3 GHz
+
+    'm5zn',              # Intel Xeon Platinum 8252 4.5 GHz
+
+    'm7a',               # AMD EPYC 9R14 Processor 3.7 GHz
+
+    # 'm7i',               # Intel Xeon Scalable (Sapphire Rapids) 3.2 GHz
+
+    'm7g',               # AWS Graviton3 Processor 2.6 GHz
+    # 'm7gd',               # AWS Graviton3 Processor 2.6 GHz
+
+    'r7a',               # AMD EPYC 9R14 Processor 3.7 GHz
+
+    'r7g',               # AWS Graviton3 Processor 2.6 GHz
+    # 'r7gd',               # AWS Graviton3 Processor 2.6 GHz
+
+    # 'r7i',               # Intel Xeon Scalable (Sapphire Rapids) 3.2 GHz
+
+    'r7iz',              # Intel Xeon Scalable (Sapphire Rapids) 3.2 GHz
+
+    'x2gd',              # AWS Graviton2 Processor 2.5 GHz 1TB
+
+    'x2idn',             # Intel Xeon Scalable (Icelake) 3.5 GHz 2 TB
+
+    'x2iedn',            # Intel Xeon Scalable (Icelake) 3.5 GHz 4 TB
+
+    'x2iezn',            # Intel Xeon Platinum 8252 4.5 GHz 1.5 TB
+
+    #'u-6tb1',            # Intel Xeon Scalable (Skylake) 6 TB
+    #'u-9tb1',            # Intel Xeon Scalable (Skylake) 9 TB
+    #'u-12tb1',           # Intel Xeon Scalable (Skylake) 12 TB
+]
+
+old_eda_instance_families = [
+    'c5',                # Mixed depending on size
+    'c5a',               # AMD EPYC 7R32 3.3 GHz
+    'c5ad',              # AMD EPYC 7R32 3.3 GHz
     'c6a',
     'c6ad',
     'c6i',               # Intel Xeon 8375C (Ice Lake) 3.5 GHz
     'c6id',
     'c6g',               # AWS Graviton2 Processor 2.5 GHz
-    #'c6gd',              # AWS Graviton2 Processor 2.5 GHz
-    #'f1',                # Intel Xeon E5-2686 v4 (Broadwell) 2.3 GHz
-    #'m5',                # Intel Xeon Platinum 8175 (Skylake) 3.1 GHz
-    #'m5d',               # Intel Xeon Platinum 8175 (Skylake) 3.1 GHz
-    #'m5a',               # AMD EPYC 7571 2.5 GHz
-    #'m5ad',              # AMD EPYC 7571 2.5 GHz
+    'c6gd',              # AWS Graviton2 Processor 2.5 GHz
+    'f1',                # Intel Xeon E5-2686 v4 (Broadwell) 2.3 GHz
+    'm5',                # Intel Xeon Platinum 8175 (Skylake) 3.1 GHz
+    'm5d',               # Intel Xeon Platinum 8175 (Skylake) 3.1 GHz
+    'm5a',               # AMD EPYC 7571 2.5 GHz
+    'm5ad',              # AMD EPYC 7571 2.5 GHz
     'm5zn',              # Intel Xeon Platinum 8252 4.5 GHz
     'm6a',               # AMD EPYC 7R13 Processor 3.6 GHz
     'm6ad',
     'm6i',               # Intel Xeon 8375C (Ice Lake) 3.5 GHz
     'm6id',
     'm6g',               # AWS Graviton2 Processor 2.5 GHz
-    #'m6gd',              # AWS Graviton2 Processor 2.5 GHz
+    'm6gd',              # AWS Graviton2 Processor 2.5 GHz
     'r5',                # Intel Xeon Platinum 8175 (Skylake) 3.1 GHz
     'r5d',               # Intel Xeon Platinum 8175 (Skylake) 3.1 GHz
-    #'r5b',               # Intel Xeon Platinum 8259 (Cascade Lake) 3.1 GHz
+    'r5b',               # Intel Xeon Platinum 8259 (Cascade Lake) 3.1 GHz
     'r5a',               # AMD EPYC 7571 2.5 GHz
     'r5ad',              # AMD EPYC 7571 2.5 GHz
     'r6a',
     'r6i',               # Intel Xeon 8375C (Ice Lake) 3.5 GHz 1TB
     'r6id',
     'r6g',               # AWS Graviton2 Processor 2.5 GHz
-    #'r6gd',              # AWS Graviton2 Processor 2.5 GHz
-    #'x1',                # High Frequency Intel Xeon E7-8880 v3 (Haswell) 2.3 GHz 2TB
-    #'x1e',               # High Frequency Intel Xeon E7-8880 v3 (Haswell) 2.3 GHz 4TB
+    'r6gd',              # AWS Graviton2 Processor 2.5 GHz
+    'x1',                # High Frequency Intel Xeon E7-8880 v3 (Haswell) 2.3 GHz 2TB
+    'x1e',               # High Frequency Intel Xeon E7-8880 v3 (Haswell) 2.3 GHz 4TB
     'x2gd',              # AWS Graviton2 Processor 2.5 GHz 1TB
     'x2idn',             # Intel Xeon Scalable (Icelake) 3.5 GHz 2 TB
     'x2iedn',            # Intel Xeon Scalable (Icelake) 3.5 GHz 4 TB
     'x2iezn',            # Intel Xeon Platinum 8252 4.5 GHz 1.5 TB
     'z1d',               # Intel Xeon Platinum 8151 4.0 GHz
-    #'u-6tb1',            # Intel Xeon Scalable (Skylake) 6 TB
-    #'u-9tb1',            # Intel Xeon Scalable (Skylake) 9 TB
-    #'u-12tb1',           # Intel Xeon Scalable (Skylake) 12 TB
 ]
 
 default_eda_instance_types = [
@@ -219,7 +293,7 @@ def get_slurm_rest_api_version(config):
 
 default_excluded_instance_types = [
     '.+\.(micro|nano)', # Not enough memory
-    '.*\.metal'
+    '.*\.metal.*'
 ]
 
 architectures = [
@@ -259,6 +333,7 @@ def get_config_schema(config):
         # Optional, but highly recommended
         Optional('ErrorSnsTopicArn'): str,
         Optional('TimeZone', default='US/Central'): str,
+        Optional('RESEnvironmentName'): str,
         'slurm': {
             Optional('ParallelClusterConfig'): {
                 Optional('Enable', default=True): And(bool, lambda s: s == True),
@@ -328,11 +403,13 @@ def get_config_schema(config):
             #     Default to StackName-cl
             Optional('ClusterName'): And(str, lambda s: s != config['StackName']),
             #
-            # MungeKeySsmParameter:
-            #     SSM String Parameter with a base64 encoded munge key to use for the cluster.
+            # MungeKeySecret:
+            #     AWS secret with a base64 encoded munge key to use for the cluster.
+            #     For an existing secret can be the secret name or the ARN.
+            #     If the secret doesn't exist one will be created, but won't be part of the cloudformation stack
+            #     so that it won't be deleted when the stack is deleted.
             #     Required if your submitters need to use more than 1 cluster.
-            #     Will be created if it doesn't exist to save the value in Parameter Store.
-            Optional('MungeKeySsmParameter', default='/slurm/munge_key'): str,
+            Optional('MungeKeySecret'): str,
             #
             # SlurmCtl:
             #     Required, but can be an empty dict to accept all of the defaults

source/cdk/config_schema.py

@@ -0,0 +1,84 @@
+"""
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+SPDX-License-Identifier: MIT-0
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+'''
+Call /opt/slurm/{{ClusterName}}/config/bin/create_users_groups_json_configure.sh using ssm run command.
+'''
+import boto3
+import json
+import logging
+from os import environ as environ
+
+logger=logging.getLogger(__file__)
+logger_formatter = logging.Formatter('%(levelname)s: %(message)s')
+logger_streamHandler = logging.StreamHandler()
+logger_streamHandler.setFormatter(logger_formatter)
+logger.addHandler(logger_streamHandler)
+logger.setLevel(logging.INFO)
+logger.propagate = False
+
+def lambda_handler(event, context):
+    try:
+        logger.info(f"event:\n{json.dumps(event, indent=4)}")
+
+        cluster_name = environ['ClusterName']
+        cluster_region = environ['Region']
+        environment_name = environ['RESEnvironmentName']
+        logger.info(f"Update RES cluster={environment_name} manager for {cluster_name} in {cluster_region}")
+
+        ec2_client = boto3.client('ec2', region_name=cluster_region)
+
+        cluster_manager_info = ec2_client.describe_instances(
+            Filters = [
+                {'Name': 'tag:res:EnvironmentName', 'Values': [environment_name]},
+                {'Name': 'tag:res:ModuleId', 'Values': ['cluster-manager']}
+            ]
+        )['Reservations'][0]['Instances'][0]
+        cluster_manager_instance_id = cluster_manager_info['InstanceId']
+        logger.info(f"cluster manager instance id: {cluster_manager_instance_id}")
+
+        ssm_client = boto3.client('ssm', region_name=cluster_region)
+        commands = f"""
+set -ex
+
+if ! [ -e /opt/slurm/{cluster_name} ]; then
+    sudo mkdir -p /opt/slurm/{cluster_name}
+fi
+if ! mountpoint /opt/slurm/{cluster_name} ; then
+    sudo mount head_node.{cluster_name}.pcluster:/opt/slurm /opt/slurm/{cluster_name} || true
+fi
+
+script="/opt/slurm/{cluster_name}/config/bin/create_users_groups_json_configure.sh"
+if ! [ -e $script ]; then
+    echo "$script doesn't exist"
+    exit 1
+fi
+
+sudo $script
+        """
+        response = ssm_client.send_command(
+            DocumentName = 'AWS-RunShellScript',
+            InstanceIds = [cluster_manager_instance_id],
+            Parameters = {'commands': [commands]},
+            Comment = f"Configure {environment_name} cluster manager for {cluster_name}"
+        )
+        logger.info(f"Sent SSM command {response['Command']['CommandId']}")
+
+    except Exception as e:
+        logger.exception(str(e))
+        raise