(INF-2987) Create script to mass import users into CO Manage Registry from files

.gitignore

@@ -1 +1,2 @@
 .vscode/launch.json
+__pycache__/*

comanage_person_schema_utils.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+
+CO_PERSON = {
+    "co_id": None,
+    "timezone": None,
+    "dateofbirth": None,
+    "status": None
+    }
+
+IDENTIFIER = {
+    "identifier": None,
+    "type": None,
+    "login": False,
+    "status": None,
+}
+
+NAME = {
+    "honorific": None,
+    "given": "None",
+    "middle": None,
+    "family": "None",
+    "suffix": None,
+    "type": "official",
+    "language": None,
+    "primary_name": True,
+}
+
+
+GROUP = {
+    "co_group_id": None,
+    "member": True,
+    "owner": False,
+}
+
+
+COU = {
+    "co_id": None,
+    "name": None,
+    "description": None,
+    "parent_id": None,
+}
+
+
+ORG_IDENTITY = {
+    "co_id": None,
+    "title": None,
+    # Organization for OrgID
+    "o": None,
+    # Department for OrgID
+    "ou": None,
+    "valid_from": None,
+    "valid_through": None,
+    "status": "",
+    "affiliation": None,
+    "date_of_birth": None,
+    "Address": [],
+    "AdHocAttribute": [],
+    "EmailAddress": [],
+    "Identifier": [],
+    "Name": [],
+    "TelephoneNumber": [],
+    "Url": [],
+}
+
+
+ROLE = {
+    "cou_id": None,
+    "title": "",
+    "o": "",
+    "ou": "",
+    "valid_from": None,
+    "valid_through": None,
+    "status": "A",
+    "sponsor_co_person_id": None,
+    "affiliation": "member",
+    "ordr": 1,
+}
+
+
+EMAIL_ADDRESS = {
+    "mail": None,
+    "type": None,
+    "verified": None,
+}
+
+
+UNIX_CLUSTER_ACCOUNT = {
+    "sync_mode": "F",
+    "status": None,
+    "username": None,
+    "uid": None,
+    "gecos": None,
+    "login_shell": "/bin/bash",
+    "home_directory": None,
+    "primary_co_group_id": None,
+    "valid_from": None,
+    "valid_through": None,
+    "unix_cluster_id": None,
+}
+
+
+SSHKEY = {
+    "type": None,
+    "skey": None,
+    "comment": "",
+    "ssh_key_authenticator_id": None,
+}
+
+
+def co_person_schema(co_id, timezone=None, dob=None, status="Active"):
+    person_data = CO_PERSON.copy()
+    person_data["co_id"] = co_id
+    person_data["timezone"] = timezone
+    person_data["dateofbirth"] = dob
+    person_data["status"] = status
+    return person_data
+
+
+def co_person_identifier(identifier, type, login=False, status="Active"):
+    identifier_data = IDENTIFIER.copy()
+    identifier_data["identifier"] = identifier
+    identifier_data["type"] = type
+    identifier_data["login"] = login
+    identifier_data["status"] = status
+    return identifier_data
+
+
+def co_person_name(given, family=None, middle=None, type="official", primary=False):
+    name_data = NAME.copy()
+    name_data["given"] = given
+    name_data["family"] = family
+    name_data["middle"] = middle
+    name_data["type"] = type
+    name_data["primary_name"] = primary
+    return name_data
+
+
+def name_split(whole_name):
+    name_sections = str(whole_name).split()
+    parts_count = len(name_sections)
+    if parts_count == 1:
+        return co_person_name(given=whole_name)
+    elif parts_count == 2:
+        return co_person_name(given=name_sections[0], family=name_sections[1])
+    else:
+        return co_person_name(given=name_sections[0], family=name_sections[parts_count-1], middle=" ".join(name_sections[1:parts_count-1]))
+
+
+def name_unsplit(name_id):
+    if name_id["given"] is None:
+        return ""
+    elif name_id["family"] is None:
+        return name_id["given"]
+    elif name_id["middle"] is None:
+        return f'{name_id["given"]} {name_id["family"]}'
+    else:
+        return f'{name_id["given"]} {name_id["middle"]} {name_id["family"]}'
+
+
+
+def co_person_group_member(group_id, member=True, owner=False):
+    group_member = GROUP.copy()
+    group_member["co_group_id"] = group_id
+    group_member["member"] = member
+    group_member["owner"] = owner
+    return group_member
+
+
+def co_person_org_id(
+    osg_co_id, name, organization="", department="", title="", affiliation="member", id_list=[]
+):
+    #org_id = {"co_id" : osg_co_id}
+    org_id = ORG_IDENTITY.copy()
+    org_id["co_id"] = osg_co_id
+    org_id["title"] = title
+    org_id["o"] = organization
+    org_id["ou"] = department
+    org_id["affiliation"] = affiliation
+    org_id["Identifier"] = id_list
+    org_id["Name"] = name
+    return org_id
+
+
+def co_person_role(cou, title, affiliation, order):
+    role = ROLE.copy()
+    role["cou_id"] = cou
+    role["title"] = title
+    role["affiliation"] = affiliation
+    role["ordr"] = order
+    return role
+
+def co_person_email_address(mail, type="delivery", verified=False):
+    email = EMAIL_ADDRESS.copy()
+    email["mail"] = mail
+    email["type"] = type
+    email["verified"] = verified
+    return email
+
+
+def co_person_unix_cluster_acc(unix_cluster_id, username, uid, name, group_id, status="A"):
+    uca = UNIX_CLUSTER_ACCOUNT.copy()
+    uca["unix_cluster_id"] = unix_cluster_id
+    uca["username"] = username
+    uca["uid"] = uid
+    uca["status"] = status
+    uca["gecos"] = name
+    uca["home_directory"] = f"/home/{username}"
+    uca["primary_co_group_id"] = group_id
+    return uca
+
+
+def co_person_sshkey(type, skey, comment, auth_id):
+    sshkey_data = SSHKEY.copy()
+    sshkey_data["type"] = type
+    sshkey_data["skey"] = skey
+    sshkey_data["comment"] = comment
+    sshkey_data["ssh_key_authenticator_id"] = auth_id
+    return sshkey_data
+
+
+# def merge_schema(base_data, new_data, type):
+#    temp = base_data
+#    for field in type.keys():
+#        for entry in new_data[field]:
+#            if "meta" in entry:
+#
+#
+#    return temp

comanage_utils.py

@@ -30,6 +30,8 @@
 TIMEOUT_BASE = 5
 MAX_ATTEMPTS = 5
 
+# HTTP return codes we shouldn't attempt to retry
+HTTP_NO_RETRY_CODES = {401, 404, 405, 500}
 
 GET    = "GET"
 PUT    = "PUT"
@@ -47,6 +49,16 @@ class URLRequestError(Error):
     pass
 
 
+class HTTPRequestError(URLRequestError):
+    """Class for exceptions due to not being able to fulfill a HTTPRequest"""
+    def __init__(self, message, code): 
+        self.message = message
+        self.code = code
+
+    def __str__(self):
+        return self.message
+
+
 def getpw(user, passfd, passfile):
     if ":" in user:
         user, pw = user.split(":", 1)
@@ -100,6 +112,14 @@ def call_api3(method, target, data, endpoint, authstr, **kw):
         # exception catching, mainly for request timeouts, "Service Temporarily Unavailable" (Rate limiting), and DNS failures.
         except urllib.error.URLError as exception:
             req_attempts += 1
+            # If the exception was an HTTPError, with a code like 404 that won't change on a retry, fail fast
+            if issubclass(exception.__class__, urllib.error.HTTPError) and exception.code in HTTP_NO_RETRY_CODES:
+                raise HTTPRequestError(
+                    "Exception raised due to api call error status"
+                + f"Exception reason: {exception}.\n Request: {req.full_url}",
+                code=exception.code
+                )
+            # Since we think the exception *might* be transient, continue with retry logic
             if req_attempts >= MAX_ATTEMPTS:
                 raise URLRequestError(
                     "Exception raised after maximum number of retries reached after total backoff of " + 
@@ -140,6 +160,18 @@ def get_co_group(gid, endpoint, authstr):
     return grouplist[0]
 
 
+def core_api_co_person_read(identifier, coid, endpoint, authstr):
+    return call_api(f"api/co/{coid}/core/v1/people/{identifier}", endpoint, authstr)
+
+
+def core_api_co_person_create(data, coid, endpoint, authstr):
+    return call_api3(POST, f"api/co/{coid}/core/v1/people/", data, endpoint, authstr)
+
+
+def core_api_co_person_update(identifier, coid, data, endpoint, authstr):
+    return call_api3(PUT, f"api/co/{coid}/core/v1/people/{identifier}", data, endpoint, authstr)
+
+
 def get_identifier(id_, endpoint, authstr):
     resp_data = call_api("identifiers/%s.json" % id_, endpoint, authstr)
     idfs = get_datalist(resp_data, "Identifiers")
@@ -157,10 +189,37 @@ def get_unix_cluster_groups_ids(ucid, endpoint, authstr):
     return set(group["CoGroupId"] for group in unix_cluster_groups["UnixClusterGroups"])
 
 
+def update_co_person_identifier(id_, type, identifier, person_id, endpoint, authstr, provisioning_target):
+    id_data = {
+      "RequestType":"Identifiers",
+      "Version":"1.0",
+      "Identifiers":
+      [
+        {
+          "Version":"1.0",
+          "Type":type,
+          "Identifier":identifier,
+          "Login":False,
+          "Person":{"Type":"CO","Id":person_id},
+          "CoProvisioningTargetId":provisioning_target,
+          "Status":"Active"
+        }
+      ]
+    }
+    return call_api3(PUT, "/api/v2/identifiers" % id_, id_data, endpoint, authstr, )
+    #return call_api3(PUT, "identifiers/%s.json" % id_, id_data, endpoint, authstr)
+
+
 def delete_identifier(id_, endpoint, authstr):
     return call_api2(DELETE, "identifiers/%s.json" % id_, endpoint, authstr)
 
 
+def get_co_group_members_pids(gid, endpoint, authstr):
+    resp_data = get_co_group_members(gid, endpoint, authstr)
+    data = get_datalist(resp_data, "CoGroupMembers")
+    return [m["Person"]["Id"] for m in data]
+
+
 def get_datalist(data, listname):
     return data[listname] if data else []
 
@@ -183,13 +242,38 @@ def identifier_from_list(id_list, id_type):
     except ValueError:
         return None
 
+def full_identifier_from_list(id_list, id_type):
+    id_type_list = [id["Type"] for id in id_list]
+    try:
+        id_index = id_type_list.index(id_type)
+        return id_list[id_index]
+    except ValueError:
+        return None
+
 
 def identifier_matches(id_list, id_type, regex_string):
     pattern = re.compile(regex_string)
     value = identifier_from_list(id_list, id_type)
     return (value is not None) and (pattern.match(value) is not None)
 
 
+def create_co_group(groupname, description, coId, endpoint, authstr, open=False,):
+    group_info = {
+        "Version"     : "1.0",
+        "CoId"        : coId,
+        "Name"        : groupname,
+        "Description" : description,
+        "Open"        : open,
+        "Status"      : "Active",
+    }
+    data = {
+        "CoGroups"    : [group_info],
+        "RequestType" : "CoGroups",
+        "Version"     : "1.0"
+    }
+    return call_api3(POST, "co_groups/.json", data, endpoint, authstr)
+
+
 def rename_co_group(gid, group, newname, endpoint, authstr):
     # minimal edit CoGroup Request includes Name+CoId+Status+Version
     new_group_info = {