Skip to content

Commit 5a833f2

Browse files
authored
pants: Add system_user detection to pants-plugins/uses_services (+ mongo detection improvements) (#6244)
2 parents d985b0d + 3dfee62 commit 5a833f2

13 files changed

Lines changed: 360 additions & 8 deletions

File tree

CHANGELOG.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ Added
3636
* Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)
3737
working on StackStorm, improve our security posture, and improve CI reliability thanks in part
3838
to pants' use of PEX lockfiles. This is not a user-facing addition.
39-
#6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241
39+
#6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244
4040
Contributed by @cognifloyd
4141
* Build of ST2 EL9 packages #6153
4242
Contributed by @amanda11

contrib/runners/orquesta_runner/tests/unit/BUILD

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,18 @@ __defaults__(
55

66
python_tests(
77
name="tests",
8+
overrides={
9+
(
10+
"test_basic.py",
11+
"test_cancel.py",
12+
"test_context.py",
13+
"test_error_handling.py",
14+
"test_pause_and_resume.py",
15+
"test_with_items.py",
16+
): dict(
17+
uses=["system_user"],
18+
),
19+
},
820
)
921

1022
python_test_utils(

pants-plugins/uses_services/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,6 @@ python_tests(
1717
# "mongo_rules_test.py": dict(uses=["mongo"]),
1818
# "rabbitmq_rules_test.py": dict(uses=["rabbitmq"]),
1919
# "redis_rules_test.py": dict(uses=["redis"]),
20+
# "system_user_test.py": dict(uses=["system_user"]),
2021
# },
2122
)

pants-plugins/uses_services/mongo_rules.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14-
import os
1514

1615
from dataclasses import dataclass
1716
from textwrap import dedent
@@ -20,6 +19,7 @@
2019
PytestPluginSetupRequest,
2120
PytestPluginSetup,
2221
)
22+
from pants.backend.python.subsystems.pytest import PyTest
2323
from pants.backend.python.util_rules.pex import (
2424
PexRequest,
2525
PexRequirements,
@@ -64,9 +64,12 @@ class UsesMongoRequest:
6464
db_host: str = "127.0.0.1" # localhost in test_db.DbConnectionTestCase
6565
db_port: int = 27017
6666
# db_name is "st2" in test_db.DbConnectionTestCase
67-
db_name: str = f"st2-test{os.environ.get('ST2TESTS_PARALLEL_SLOT', '')}"
67+
db_name: str = "st2-test{}" # {} will be replaced by test slot (a format string)
68+
6869
db_connection_timeout: int = 3000
6970

71+
execution_slot_var: str = "ST2TESTS_PARALLEL_SLOT"
72+
7073

7174
@dataclass(frozen=True)
7275
class MongoIsRunning:
@@ -87,7 +90,7 @@ def is_applicable(cls, target: Target) -> bool:
8790
level=LogLevel.DEBUG,
8891
)
8992
async def mongo_is_running_for_pytest(
90-
request: PytestUsesMongoRequest,
93+
request: PytestUsesMongoRequest, pytest: PyTest
9194
) -> PytestPluginSetup:
9295
# TODO: delete these comments once the Makefile becomes irrelevant.
9396
# the comments explore how the Makefile prepares to run and runs tests
@@ -104,7 +107,10 @@ async def mongo_is_running_for_pytest(
104107
# nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) $$component/tests/unit
105108

106109
# this will raise an error if mongo is not running
107-
_ = await Get(MongoIsRunning, UsesMongoRequest())
110+
_ = await Get(
111+
MongoIsRunning,
112+
UsesMongoRequest(execution_slot_var=pytest.execution_slot_var or ""),
113+
)
108114

109115
return PytestPluginSetup()
110116

@@ -145,8 +151,10 @@ async def mongo_is_running(
145151
str(request.db_port),
146152
request.db_name,
147153
str(request.db_connection_timeout),
154+
request.execution_slot_var,
148155
),
149156
input_digest=script_digest,
157+
execution_slot_variable=request.execution_slot_var,
150158
description="Checking to see if Mongo is up and accessible.",
151159
# this can change from run to run, so don't cache results.
152160
cache_scope=ProcessCacheScope.PER_SESSION,

pants-plugins/uses_services/register.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,13 @@
1616
PythonTestsGeneratorTarget,
1717
)
1818

19-
from uses_services import mongo_rules, platform_rules, rabbitmq_rules, redis_rules
19+
from uses_services import (
20+
mongo_rules,
21+
platform_rules,
22+
rabbitmq_rules,
23+
redis_rules,
24+
system_user_rules,
25+
)
2026
from uses_services.target_types import UsesServicesField
2127

2228

@@ -28,4 +34,5 @@ def rules():
2834
*mongo_rules.rules(),
2935
*rabbitmq_rules.rules(),
3036
*redis_rules.rules(),
37+
*system_user_rules.rules(),
3138
]
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Copyright 2024 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from __future__ import annotations
15+
16+
import os
17+
import pwd
18+
import sys
19+
20+
21+
def _has_system_user(system_user: str) -> bool:
22+
"""Make sure the system_user exists.
23+
24+
This should not import the st2 code as it should be self-contained.
25+
"""
26+
try:
27+
pwd.getpwnam(system_user)
28+
except KeyError:
29+
# put current user (for use in error msg instructions)
30+
print(pwd.getpwuid(os.getuid()).pw_name)
31+
return False
32+
print(system_user)
33+
return True
34+
35+
36+
if __name__ == "__main__":
37+
args = dict((k, v) for k, v in enumerate(sys.argv))
38+
39+
system_user = args.get(1, "stanley")
40+
41+
is_running = _has_system_user(system_user)
42+
exit_code = 0 if is_running else 1
43+
sys.exit(exit_code)

pants-plugins/uses_services/scripts/is_mongo_running.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14+
import os
1415
import sys
1516

1617

@@ -48,9 +49,12 @@ def _is_mongo_running(
4849
args = dict((k, v) for k, v in enumerate(sys.argv))
4950
db_host = args.get(1, "127.0.0.1")
5051
db_port = args.get(2, 27017)
51-
db_name = args.get(3, "st2-test")
52+
db_name = args.get(3, "st2-test{}")
5253
connection_timeout_ms = args.get(4, 3000)
5354

55+
slot_var = args.get(5, "ST2TESTS_PARALLEL_SLOT")
56+
db_name = db_name.format(os.environ.get(slot_var) or "")
57+
5458
is_running = _is_mongo_running(
5559
db_host, int(db_port), db_name, int(connection_timeout_ms)
5660
)
Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
# Copyright 2024 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from __future__ import annotations
15+
16+
from dataclasses import dataclass
17+
from textwrap import dedent
18+
19+
from pants.backend.python.goals.pytest_runner import (
20+
PytestPluginSetupRequest,
21+
PytestPluginSetup,
22+
)
23+
from pants.backend.python.target_types import Executable
24+
from pants.backend.python.util_rules.pex import (
25+
PexRequest,
26+
VenvPex,
27+
VenvPexProcess,
28+
rules as pex_rules,
29+
)
30+
from pants.core.goals.test import TestExtraEnv
31+
from pants.engine.fs import CreateDigest, Digest, FileContent
32+
from pants.engine.rules import collect_rules, Get, rule
33+
from pants.engine.process import FallibleProcessResult, ProcessCacheScope
34+
from pants.engine.target import Target
35+
from pants.engine.unions import UnionRule
36+
from pants.util.logging import LogLevel
37+
38+
from uses_services.exceptions import ServiceMissingError
39+
from uses_services.platform_rules import Platform
40+
from uses_services.scripts.has_system_user import (
41+
__file__ as has_system_user_full_path,
42+
)
43+
from uses_services.target_types import UsesServicesField
44+
45+
46+
@dataclass(frozen=True)
47+
class UsesSystemUserRequest:
48+
"""One or more targets need the system_user (like stanley) using these settings.
49+
50+
The system_user attributes represent the system_user.user settings from st2.conf.
51+
In st2 code, they come from:
52+
oslo_config.cfg.CONF.system_user.user
53+
"""
54+
55+
system_user: str = "stanley"
56+
57+
58+
@dataclass(frozen=True)
59+
class HasSystemUser:
60+
pass
61+
62+
63+
class PytestUsesSystemUserRequest(PytestPluginSetupRequest):
64+
@classmethod
65+
def is_applicable(cls, target: Target) -> bool:
66+
if not target.has_field(UsesServicesField):
67+
return False
68+
uses = target.get(UsesServicesField).value
69+
return uses is not None and "system_user" in uses
70+
71+
72+
@rule(
73+
desc="Ensure system_user is present before running tests.",
74+
level=LogLevel.DEBUG,
75+
)
76+
async def has_system_user_for_pytest(
77+
request: PytestUsesSystemUserRequest,
78+
test_extra_env: TestExtraEnv,
79+
) -> PytestPluginSetup:
80+
system_user = test_extra_env.env.get("ST2TESTS_SYSTEM_USER", "stanley")
81+
82+
# this will raise an error if system_user is not present
83+
_ = await Get(HasSystemUser, UsesSystemUserRequest(system_user=system_user))
84+
85+
return PytestPluginSetup()
86+
87+
88+
@rule(
89+
desc="Test to see if system_user is present.",
90+
level=LogLevel.DEBUG,
91+
)
92+
async def has_system_user(
93+
request: UsesSystemUserRequest, platform: Platform
94+
) -> HasSystemUser:
95+
script_path = "./has_system_user.py"
96+
97+
# pants is already watching this directory as it is under a source root.
98+
# So, we don't need to double watch with PathGlobs, just open it.
99+
with open(has_system_user_full_path, "rb") as script_file:
100+
script_contents = script_file.read()
101+
102+
script_digest = await Get(
103+
Digest, CreateDigest([FileContent(script_path, script_contents)])
104+
)
105+
script_pex = await Get(
106+
VenvPex,
107+
PexRequest(
108+
output_filename="script.pex",
109+
internal_only=True,
110+
sources=script_digest,
111+
main=Executable(script_path),
112+
),
113+
)
114+
115+
result = await Get(
116+
FallibleProcessResult,
117+
VenvPexProcess(
118+
script_pex,
119+
argv=(request.system_user,),
120+
description="Checking to see if system_user is present.",
121+
# this can change from run to run, so don't cache results.
122+
cache_scope=ProcessCacheScope.PER_SESSION,
123+
level=LogLevel.DEBUG,
124+
),
125+
)
126+
has_user = result.exit_code == 0
127+
128+
if has_user:
129+
return HasSystemUser()
130+
131+
current_user = result.stdout.decode().strip()
132+
133+
# system_user is not present, so raise an error with instructions.
134+
raise ServiceMissingError(
135+
service="system_user",
136+
platform=platform,
137+
msg=dedent(
138+
f"""\
139+
The system_user ({request.system_user}) does not seem to be present!
140+
141+
Please export the ST2TESTS_SYSTEM_USER env var to specify which user
142+
tests should use as the system_user. This user must be present on
143+
your system.
144+
145+
To use your current user ({current_user}) as the system_user, run:
146+
147+
export ST2TESTS_SYSTEM_USER=$(id -un)
148+
"""
149+
),
150+
)
151+
152+
153+
def rules():
154+
return [
155+
*collect_rules(),
156+
UnionRule(PytestPluginSetupRequest, PytestUsesSystemUserRequest),
157+
*pex_rules(),
158+
]

0 commit comments

Comments
 (0)