Skip to content

Commit ffc1b18

Browse files
authored
fix: run invocation lifecycle on handler worker
Run Start, handler and serialization, resource cleanup, and End on the actual handler worker. Remove the optional handler_context dispatch, preserve failed-Start isolation and token ownership, and propagate fresh contexts to SDK-managed map/parallel branches. Include unit, public handler, installed-version compatibility, and replay coverage. Fixes #428. Related PRs: #758 and aws/aws-durable-execution-conformance-tests#131.
1 parent ef68359 commit ffc1b18

27 files changed

Lines changed: 2142 additions & 184 deletions
Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
"""Install and verify built artifacts without silently using editable sources."""
2+
3+
from __future__ import annotations
4+
5+
import argparse
6+
import ast
7+
import hashlib
8+
import importlib.metadata
9+
import json
10+
import subprocess
11+
import sys
12+
import zipfile
13+
from pathlib import Path
14+
15+
16+
ROOT = Path(__file__).resolve().parents[2]
17+
CORE = "aws-durable-execution-sdk-python"
18+
OTEL = CORE + "-otel"
19+
20+
21+
def built_wheel(package: str) -> Path:
22+
directory = ROOT / "packages" / package
23+
module = package.replace("-", "_")
24+
about = ast.parse((directory / "src" / module / "__about__.py").read_text())
25+
version = next(
26+
ast.literal_eval(node.value)
27+
for node in about.body
28+
if isinstance(node, ast.Assign)
29+
and any(
30+
isinstance(target, ast.Name) and target.id == "__version__"
31+
for target in node.targets
32+
)
33+
)
34+
wheels = list((directory / "dist").glob(f"{module}-{version}-*.whl"))
35+
if len(wheels) != 1:
36+
raise ValueError(f"Build exactly one {package} {version} wheel first: {wheels}")
37+
return wheels[0]
38+
39+
40+
def verify(wheel: Path, package: str) -> None:
41+
installed = importlib.metadata.distribution(package)
42+
direct = json.loads(installed.read_text("direct_url.json") or "{}")
43+
assert not direct.get("dir_info", {}).get("editable"), direct
44+
digest = hashlib.sha256(wheel.read_bytes()).hexdigest()
45+
assert direct["archive_info"]["hashes"]["sha256"] == digest, direct
46+
module = package.replace("-", "_")
47+
with zipfile.ZipFile(wheel) as archive:
48+
sources = [
49+
name
50+
for name in archive.namelist()
51+
if name.startswith(module + "/") and name.endswith(".py")
52+
]
53+
assert sources
54+
for name in sources:
55+
path = Path(installed.locate_file(name)).resolve()
56+
assert "site-packages" in path.parts, path
57+
assert path.read_bytes() == archive.read(name), path
58+
print(
59+
json.dumps(
60+
{
61+
"package": package,
62+
"version": installed.version,
63+
"wheel": str(wheel),
64+
"sha256": digest,
65+
"verified_sources": len(sources),
66+
}
67+
)
68+
)
69+
70+
71+
def main() -> None:
72+
parser = argparse.ArgumentParser(description=__doc__)
73+
parser.add_argument("--legacy-plugin", action="store_true")
74+
args = parser.parse_args()
75+
packages = [CORE] if args.legacy_plugin else [CORE, OTEL]
76+
wheels = [built_wheel(package) for package in packages]
77+
subprocess.run(
78+
[
79+
sys.executable,
80+
"-m",
81+
"pip",
82+
"install",
83+
"--no-index",
84+
"--no-deps",
85+
"--force-reinstall",
86+
*map(str, wheels),
87+
],
88+
check=True,
89+
)
90+
for wheel, package in zip(wheels, packages, strict=True):
91+
verify(wheel, package)
92+
if args.legacy_plugin:
93+
assert importlib.metadata.version(OTEL) == "1.0.0"
94+
import aws_durable_execution_sdk_python_otel as otel
95+
96+
assert "site-packages" in Path(otel.__file__).resolve().parts
97+
subprocess.run([sys.executable, "-m", "pip", "check"], check=True)
98+
99+
100+
if __name__ == "__main__":
101+
main()
Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
"""Exercise real installed version pairs through the public durable handler."""
2+
3+
from __future__ import annotations
4+
5+
import contextvars
6+
from datetime import UTC, datetime
7+
from importlib.metadata import version
8+
import os
9+
from pathlib import Path
10+
import threading
11+
from types import SimpleNamespace
12+
13+
import pytest
14+
from aws_durable_execution_sdk_python import durable_execution
15+
from aws_durable_execution_sdk_python import execution as core_execution
16+
from aws_durable_execution_sdk_python.lambda_service import (
17+
ExecutionDetails,
18+
Operation,
19+
OperationStatus,
20+
OperationType,
21+
)
22+
from aws_durable_execution_sdk_python.plugin import DurableInstrumentationPlugin
23+
from aws_durable_execution_sdk_python_otel.execution_plugin import ExecutionOtelPlugin
24+
from aws_durable_execution_sdk_python_otel.invocation_plugin import InvocationOtelPlugin
25+
from aws_durable_execution_sdk_python_otel.otel_plugin_config import OtelPluginConfig
26+
from opentelemetry import baggage, context, trace
27+
from opentelemetry.sdk.trace import TracerProvider
28+
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
29+
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
30+
from packaging.version import Version
31+
32+
33+
class NoNetworkClient:
34+
def __getattr__(self, name: str):
35+
raise AssertionError(f"Unexpected service access: {name}")
36+
37+
38+
@pytest.mark.parametrize("view", [InvocationOtelPlugin, ExecutionOtelPlugin])
39+
@pytest.mark.parametrize("order", ["alone", "baggage-first", "baggage-last"])
40+
@pytest.mark.parametrize("ambient_present", [False, True])
41+
def test_installed_pair_preserves_context_and_documents_legacy_fallback(
42+
view, order, ambient_present
43+
):
44+
legacy = os.environ.get("OTEL_COMPAT_LEGACY") == "1"
45+
assert Version(version("aws-durable-execution-sdk-python")) >= Version("2.1.0")
46+
assert version("aws-durable-execution-sdk-python-otel") == (
47+
"1.0.0" if legacy else "1.1.0"
48+
)
49+
assert "site-packages" in Path(core_execution.__file__).resolve().parts
50+
import aws_durable_execution_sdk_python_otel as installed_otel
51+
52+
assert "site-packages" in Path(installed_otel.__file__).resolve().parts
53+
54+
def run() -> None:
55+
provider = TracerProvider()
56+
exporter = InMemorySpanExporter()
57+
provider.add_span_processor(SimpleSpanProcessor(exporter))
58+
tracer = provider.get_tracer("installed-lifecycle")
59+
phases = []
60+
worker_threads = []
61+
caller_thread = threading.get_ident()
62+
63+
class BaggagePlugin(DurableInstrumentationPlugin):
64+
def on_invocation_start(self, info):
65+
self.token = context.attach(baggage.set_baggage("customer", "present"))
66+
phases.append("baggage-start")
67+
worker_threads.append(threading.get_ident())
68+
69+
def on_invocation_end(self, info):
70+
context.detach(self.token)
71+
phases.append("baggage-end")
72+
worker_threads.append(threading.get_ident())
73+
74+
plugin = view(
75+
OtelPluginConfig(
76+
tracer_provider=provider,
77+
enrich_logger=False,
78+
context_extractor=lambda _: None,
79+
)
80+
)
81+
plugins = {
82+
"alone": [plugin],
83+
"baggage-first": [BaggagePlugin(), plugin],
84+
"baggage-last": [plugin, BaggagePlugin()],
85+
}[order]
86+
body_parents = []
87+
88+
@durable_execution(plugins=plugins, boto3_client=NoNetworkClient())
89+
def handler(event, durable_context):
90+
phases.append("body")
91+
worker_threads.append(threading.get_ident())
92+
assert baggage.get_baggage("incoming") == "keep"
93+
assert baggage.get_baggage("customer") == (
94+
None if order == "alone" else "present"
95+
)
96+
parent = trace.get_current_span().get_span_context()
97+
body_parents.append(parent)
98+
with tracer.start_as_current_span("customer-span"):
99+
pass
100+
return "ok"
101+
102+
operation = Operation(
103+
operation_id="installed",
104+
operation_type=OperationType.EXECUTION,
105+
status=OperationStatus.STARTED,
106+
start_timestamp=datetime(2026, 10, 8, tzinfo=UTC),
107+
execution_details=ExecutionDetails(input_payload="{}"),
108+
)
109+
event = {
110+
"DurableExecutionArn": "test-arn/installed",
111+
"CheckpointToken": "token",
112+
"InitialExecutionState": {
113+
"Operations": [operation.to_json_dict()],
114+
"NextMarker": "",
115+
},
116+
}
117+
lambda_context = SimpleNamespace(
118+
aws_request_id="installed",
119+
client_context=None,
120+
identity=None,
121+
_epoch_deadline_time_in_ms=0,
122+
invoked_function_arn="test-arn",
123+
tenant_id=None,
124+
)
125+
ambient = tracer.start_span("host") if ambient_present else trace.INVALID_SPAN
126+
host = baggage.set_baggage(
127+
"incoming", "keep", trace.set_span_in_context(ambient)
128+
)
129+
token = context.attach(host)
130+
try:
131+
for _ in range(2):
132+
assert handler(event, lambda_context)["Status"] == "SUCCEEDED"
133+
assert context.get_current() is host
134+
assert caller_thread not in worker_threads
135+
assert phases == (
136+
["body"] * 2
137+
if order == "alone"
138+
else ["baggage-start", "body", "baggage-end"] * 2
139+
)
140+
spans = exporter.get_finished_spans()
141+
if legacy and view is InvocationOtelPlugin:
142+
# The released plugin does not attach an Invocation fallback.
143+
assert body_parents == [ambient.get_span_context()] * 2
144+
else:
145+
name = "Invocation" if view is InvocationOtelPlugin else "Workflow"
146+
contexts = [span.context for span in spans if span.name == name]
147+
assert contexts
148+
assert all(
149+
parent.is_valid and parent in contexts for parent in body_parents
150+
)
151+
users = [span for span in spans if span.name == "customer-span"]
152+
assert len(users) == 2
153+
assert [span.parent for span in users] == [
154+
parent if parent.is_valid else None for parent in body_parents
155+
]
156+
assert plugin._context_tokens == {}
157+
finally:
158+
context.detach(token)
159+
ambient.end()
160+
provider.shutdown()
161+
162+
contextvars.Context().run(run)

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,6 @@ jobs:
8484
run: hatch run types:check
8585
- name: Run tests + coverage
8686
run: hatch run test:cov
87-
- name: Verify supported legacy core compatibility
88-
run: hatch run test-pypi-otel-legacy:test
8987
- name: Build distribution
9088
run: |
9189
for pkg in packages/*/; do
@@ -96,6 +94,10 @@ jobs:
9694
cd "$GITHUB_WORKSPACE"
9795
fi
9896
done
97+
- name: Test installed core and OTel wheels
98+
run: hatch run test-wheel-otel:test
99+
- name: Test released OTel with the new core wheel
100+
run: hatch run test-wheel-otel-legacy:test
99101
- name: Verify OTel wheel dependency contract
100102
run: |
101103
OTEL_WHEEL=$(find packages/aws-durable-execution-sdk-python-otel/dist \

‎.github/workflows/cloud-tests.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,9 @@ jobs:
133133
echo "Could not resolve the latest ADOT Python layer for $AWS_REGION"
134134
exit 1
135135
fi
136-
aws lambda get-layer-version-by-arn \
136+
# Parallel jobs can throttle this read; keep retries local to the lookup.
137+
AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=8 \
138+
aws lambda get-layer-version-by-arn \
137139
--arn "$ADOT_LAYER_ARN" \
138140
--region "$AWS_REGION" \
139141
--query LayerVersionArn \

‎.github/workflows/opentelemetry-conformance-tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ jobs:
6666
actions: write
6767
contents: read
6868
id-token: write
69-
uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@a66037abbbfa55fde97f714e30f0bc262edefd63
69+
uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@f18bd0b5f28c5c90e288d0fb8bca08a849b51863
7070
with:
7171
language: python
7272
runs_on: codebuild-github-actions-runner-${{ github.run_id }}-${{ github.run_attempt }}

‎CONTRIBUTING.md‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -70,15 +70,25 @@ hatch run dev-otel:typecheck # type check otel only
7070
hatch run dev-examples:test # run examples tests only
7171
```
7272

73-
### PyPI release testing
73+
### Installed package compatibility testing
7474

75-
To verify packages work against the published PyPI version of the core SDK (rather than the local workspace):
75+
Build the core and OTel distributions with `hatch build` in each package, then run
76+
these commands from the repository root:
7677

7778
```bash
78-
hatch run test-pypi-otel:test # test new OTel capabilities against capable installed core
79-
hatch run test-pypi-otel-legacy:test # valid registrations/lifecycles on supported core 2.0.x
80-
hatch run test-pypi-examples:test # test examples against PyPI core SDK
81-
```
79+
hatch run test-wheel-otel:test # full OTel suite on the two built wheels
80+
hatch run test-wheel-otel-legacy:test # released OTel 1.0.0 with the built core
81+
hatch run test-pypi-examples:test # examples against the published core
82+
```
83+
84+
The wheel lanes have no editable workspace members. They verify installed source
85+
bytes and artifact hashes before exercising public handlers. OTel 1.1 requires
86+
the redesigned core 2.1.0 lifecycle; it no longer claims compatibility with core
87+
2.0.x. Publish core first. Building both wheels lets CI verify the intended pair
88+
before that minimum is available on PyPI. The legacy-plugin lane documents the
89+
actual core-only upgrade: host isolation is provided by the new core, while old
90+
Invocation OTel does not gain the new fallback. Workspace tests continue to cover
91+
the complete current implementation with `hatch run dev-otel:test`.
8292

8393
### Package-level commands
8494

‎packages/aws-durable-execution-sdk-python-conformance-tests/handlers/plugin/plugin_wait_replay_flag.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
"""
1919

2020
import json
21+
from threading import Lock
2122
from typing import Any
2223

2324
from aws_durable_execution_sdk_python.config import Duration, ParallelConfig
@@ -31,13 +32,19 @@
3132
)
3233

3334

35+
_log_lock = Lock()
36+
37+
3438
def _emit(record: dict[str, Any], execution_arn: str | None) -> None:
3539
# Prefix every plugin record with the execution ARN as a top-level field so
3640
# the conformance runner's CloudWatch JSON filter can scope logs to a single
3741
# execution. Omit the field when the ARN is unset (never invent a value).
3842
if execution_arn:
3943
record = {"durableExecutionArn": execution_arn, **record}
40-
print(json.dumps(record), flush=True)
44+
# Start and end hooks can run on different threads. Keep print's separate
45+
# body/newline writes together so the runner receives one JSON per line.
46+
with _log_lock:
47+
print(json.dumps(record), flush=True)
4148

4249

4350
class WaitReplayFlagPlugin(DurableInstrumentationPlugin):

0 commit comments

Comments
 (0)