From 5b19c0aece5494e00afd6072335dd9d04f9c6b8f Mon Sep 17 00:00:00 2001 From: "michael.dsilva" Date: Fri, 23 Aug 2024 12:54:00 +1000 Subject: [PATCH 1/5] use ansible_ssh_pass rather than ansible_ssh_password to align with ansible --- .../src/test/groovy/functional/BasicIntegrationSpec.groovy | 4 ++-- .../com/rundeck/plugins/ansible/ansible/AnsibleRunner.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/functional-test/src/test/groovy/functional/BasicIntegrationSpec.groovy b/functional-test/src/test/groovy/functional/BasicIntegrationSpec.groovy index 7508cfe3..75ab9120 100644 --- a/functional-test/src/test/groovy/functional/BasicIntegrationSpec.groovy +++ b/functional-test/src/test/groovy/functional/BasicIntegrationSpec.groovy @@ -85,7 +85,7 @@ class BasicIntegrationSpec extends BaseTestConfiguration { ansibleNodeExecutionStatus.get("failed")==0 ansibleNodeExecutionStatus.get("skipped")==0 ansibleNodeExecutionStatus.get("ignored")==0 - logs.findAll {it.log.contains("encryptVariable ansible_ssh_password:")}.size() == 1 + logs.findAll {it.log.contains("encryptVariable ansible_ssh_pass:")}.size() == 1 } def "test simple inline playbook private-key with passphrase authentication"(){ @@ -301,7 +301,7 @@ class BasicIntegrationSpec extends BaseTestConfiguration { ansibleNodeExecutionStatus.get("failed")==0 ansibleNodeExecutionStatus.get("skipped")==0 ansibleNodeExecutionStatus.get("ignored")==0 - logs.findAll {it.log.contains("encryptVariable ansible_ssh_password:")}.size() == 1 + logs.findAll {it.log.contains("encryptVariable ansible_ssh_pass:")}.size() == 1 logs.findAll {it.log.contains("\"environmentTest\": \"test\"")}.size() == 1 logs.findAll {it.log.contains("\"token\": 13231232312321321321321")}.size() == 1 } diff --git a/src/main/groovy/com/rundeck/plugins/ansible/ansible/AnsibleRunner.java b/src/main/groovy/com/rundeck/plugins/ansible/ansible/AnsibleRunner.java index b8a463f3..83e6e5e3 100644 --- a/src/main/groovy/com/rundeck/plugins/ansible/ansible/AnsibleRunner.java +++ b/src/main/groovy/com/rundeck/plugins/ansible/ansible/AnsibleRunner.java @@ -447,7 +447,7 @@ public int run() throws Exception { } if (sshUsePassword) { - String extraVarsPassword = "ansible_ssh_password: " + sshPass; + String extraVarsPassword = "ansible_ssh_pass: " + sshPass; String finalextraVarsPassword = extraVarsPassword; if(useAnsibleVault){ @@ -790,4 +790,4 @@ public String encryptExtraVarsKey(String extraVars) throws Exception { return stringBuilder.toString(); } -} \ No newline at end of file +} From e62ced711d05fef6156f65e1f2fe6866ed3c539f Mon Sep 17 00:00:00 2001 From: "michael.dsilva" Date: Wed, 9 Oct 2024 12:19:31 +1100 Subject: [PATCH 2/5] increase CodePointLimit to 100MB + MaxAliasesForCollections to a big number + omit warnings from top of yaml file --- .../ansible/plugin/AnsibleResourceModelSource.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java b/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java index 0df5b78a..bfacd59b 100644 --- a/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java +++ b/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java @@ -185,7 +185,7 @@ public void configure(Properties configuration) throws ConfigurationException { gatherFacts = "true".equals(resolveProperty(AnsibleDescribable.ANSIBLE_GATHER_FACTS,null,configuration,executionDataContext)); ignoreErrors = "true".equals(resolveProperty(AnsibleDescribable.ANSIBLE_IGNORE_ERRORS,null,configuration,executionDataContext)); - yamlDataSize = resolveIntProperty(AnsibleDescribable.ANSIBLE_YAML_DATA_SIZE,10, configuration, executionDataContext); + yamlDataSize = resolveIntProperty(AnsibleDescribable.ANSIBLE_YAML_DATA_SIZE,1000, configuration, executionDataContext); limit = (String) resolveProperty(AnsibleDescribable.ANSIBLE_LIMIT,null,configuration,executionDataContext); ignoreTagPrefix = (String) resolveProperty(AnsibleDescribable.ANSIBLE_IGNORE_TAGS,null,configuration,executionDataContext); @@ -693,12 +693,16 @@ public void ansibleInventoryList(NodeSetImpl nodes, AnsibleRunner.AnsibleRunnerB int codePointLimit = yamlDataSize * 1024 * 1024; LoaderOptions snakeOptions = new LoaderOptions(); - // max inventory file size allowed to 10mb + // max inventory file size allowed to 100mb snakeOptions.setCodePointLimit(codePointLimit); + snakeOptions.setMaxAliasesForCollections(Integer.MAX_VALUE); // crazy large number for now Yaml yaml = new Yaml(new SafeConstructor(snakeOptions)); String listResp = getNodesFromInventory(runnerBuilder); + //strip everything before all: + listResp = listResp.substring(listResp.indexOf("all:")); + Map allInventory; try { allInventory = yaml.load(listResp); From 809017ad2bdd75fd7b34dcd837b4c11297745b41 Mon Sep 17 00:00:00 2001 From: "michael.dsilva" Date: Wed, 9 Oct 2024 14:26:25 +1100 Subject: [PATCH 3/5] check that the child has hosts in it and not just other children --- .../plugins/ansible/plugin/AnsibleResourceModelSource.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java b/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java index bfacd59b..b8a5687e 100644 --- a/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java +++ b/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java @@ -713,6 +713,11 @@ public void ansibleInventoryList(NodeSetImpl nodes, AnsibleRunner.AnsibleRunnerB Map all = InventoryList.getValue(allInventory, ALL); Map children = InventoryList.getValue(all, CHILDREN); + //check that the child has hosts in it and not just other children + if (hosts == null) { + continue; + } + for (Map.Entry pair : children.entrySet()) { String hostGroup = pair.getKey(); Map hostNames = InventoryList.getType(pair.getValue()); From 5a9d95b551f68e68f1401597a3f0405893a7dbfd Mon Sep 17 00:00:00 2001 From: "michael.dsilva" Date: Thu, 10 Oct 2024 08:39:25 +1100 Subject: [PATCH 4/5] fix: bad copy paste --- .../ansible/plugin/AnsibleResourceModelSource.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java b/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java index b8a5687e..c254c7b7 100644 --- a/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java +++ b/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java @@ -693,7 +693,7 @@ public void ansibleInventoryList(NodeSetImpl nodes, AnsibleRunner.AnsibleRunnerB int codePointLimit = yamlDataSize * 1024 * 1024; LoaderOptions snakeOptions = new LoaderOptions(); - // max inventory file size allowed to 100mb + // max inventory file size allowed to 1000mb snakeOptions.setCodePointLimit(codePointLimit); snakeOptions.setMaxAliasesForCollections(Integer.MAX_VALUE); // crazy large number for now Yaml yaml = new Yaml(new SafeConstructor(snakeOptions)); @@ -713,16 +713,16 @@ public void ansibleInventoryList(NodeSetImpl nodes, AnsibleRunner.AnsibleRunnerB Map all = InventoryList.getValue(allInventory, ALL); Map children = InventoryList.getValue(all, CHILDREN); - //check that the child has hosts in it and not just other children - if (hosts == null) { - continue; - } - for (Map.Entry pair : children.entrySet()) { String hostGroup = pair.getKey(); Map hostNames = InventoryList.getType(pair.getValue()); Map hosts = InventoryList.getValue(hostNames, HOSTS); + //check that the child has hosts in it and not just other children + if (hosts == null) { + continue; + } + for (Map.Entry hostNode : hosts.entrySet()) { NodeEntryImpl node = new NodeEntryImpl(); node.setTags(Set.of(hostGroup)); From 4a550f4f1b80d9c0e4cfc1026f961948e14d3236 Mon Sep 17 00:00:00 2001 From: "michael.dsilva" Date: Fri, 11 Oct 2024 14:14:31 +1100 Subject: [PATCH 5/5] Attempt to make getNodesFromInventory recursive as it was missing tags --- .../plugin/AnsibleResourceModelSource.java | 87 +++++++++++-------- 1 file changed, 50 insertions(+), 37 deletions(-) diff --git a/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java b/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java index c254c7b7..59510588 100644 --- a/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java +++ b/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java @@ -683,6 +683,55 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx } } + private NodeSetImpl ansibleInventoryListAddNodes(NodeSetImpl nodes, Map all) throws ResourceModelSourceException { + //System.out.println("[DEBUG] all: " + all); + Map children = InventoryList.getValue(all, CHILDREN); + + for (Map.Entry pair : children.entrySet()) { + String hostGroup = pair.getKey(); + Map hostNames = InventoryList.getType(pair.getValue()); + Map hosts = InventoryList.getValue(hostNames, HOSTS); + Map grandchildren = InventoryList.getValue(hostNames, CHILDREN); + + if (grandchildren != null) { + //System.out.println("[DEBUG] pair: " + pair); + //System.out.println("[DEBUG] grandchildren: " + grandchildren); + nodes = ansibleInventoryListAddNodes(nodes, + Map.ofEntries( + Map.entry("children", grandchildren) + ) + ); + } + if (hosts != null) { + for (Map.Entry hostNode : hosts.entrySet()) { + NodeEntryImpl node = new NodeEntryImpl(); + node.setTags(Set.of(hostGroup)); + String hostName = hostNode.getKey(); + node.setHostname(hostName); + node.setNodename(hostName); + Map nodeValues = InventoryList.getType(hostNode.getValue()); + + InventoryList.tagHandle(NodeTag.HOSTNAME, node, nodeValues); + InventoryList.tagHandle(NodeTag.USERNAME, node, nodeValues); + InventoryList.tagHandle(NodeTag.OS_FAMILY, node, nodeValues); + InventoryList.tagHandle(NodeTag.OS_NAME, node, nodeValues); + InventoryList.tagHandle(NodeTag.OS_ARCHITECTURE, node, nodeValues); + InventoryList.tagHandle(NodeTag.OS_VERSION, node, nodeValues); + InventoryList.tagHandle(NodeTag.DESCRIPTION, node, nodeValues); + + nodeValues.forEach((key, value) -> { + if (value != null) { + node.setAttribute(key, value.toString()); + } + }); + + nodes.putNode(node); + } + } + } + return nodes; + } + /** * Process nodes coming from Ansible to convert them to Rundeck node * @param nodes Rundeck nodes @@ -711,43 +760,7 @@ public void ansibleInventoryList(NodeSetImpl nodes, AnsibleRunner.AnsibleRunnerB } Map all = InventoryList.getValue(allInventory, ALL); - Map children = InventoryList.getValue(all, CHILDREN); - - for (Map.Entry pair : children.entrySet()) { - String hostGroup = pair.getKey(); - Map hostNames = InventoryList.getType(pair.getValue()); - Map hosts = InventoryList.getValue(hostNames, HOSTS); - - //check that the child has hosts in it and not just other children - if (hosts == null) { - continue; - } - - for (Map.Entry hostNode : hosts.entrySet()) { - NodeEntryImpl node = new NodeEntryImpl(); - node.setTags(Set.of(hostGroup)); - String hostName = hostNode.getKey(); - node.setHostname(hostName); - node.setNodename(hostName); - Map nodeValues = InventoryList.getType(hostNode.getValue()); - - InventoryList.tagHandle(NodeTag.HOSTNAME, node, nodeValues); - InventoryList.tagHandle(NodeTag.USERNAME, node, nodeValues); - InventoryList.tagHandle(NodeTag.OS_FAMILY, node, nodeValues); - InventoryList.tagHandle(NodeTag.OS_NAME, node, nodeValues); - InventoryList.tagHandle(NodeTag.OS_ARCHITECTURE, node, nodeValues); - InventoryList.tagHandle(NodeTag.OS_VERSION, node, nodeValues); - InventoryList.tagHandle(NodeTag.DESCRIPTION, node, nodeValues); - - nodeValues.forEach((key, value) -> { - if (value != null) { - node.setAttribute(key, value.toString()); - } - }); - - nodes.putNode(node); - } - } + nodes = ansibleInventoryListAddNodes(nodes, all); } /**