PR - [Feature] Step files with param

.gitignore

@@ -24,4 +24,5 @@ package.properties
 .settings
 
 */target
-.vscode
+.vscode
+/jira/

core/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.jsmart</groupId>
         <artifactId>zerocode-tdd-parent</artifactId>
-        <version>1.4.4</version>
+        <version>1.4.5</version>
     </parent>
 
     <artifactId>zerocode-tdd</artifactId>

core/src/main/java/org/jsmart/zerocode/core/domain/Step.java

@@ -2,6 +2,7 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -33,10 +34,16 @@ public class Step {
 
     @JsonFormat(with = JsonFormat.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
     private List<JsonNode> stepFiles;
+    private JsonNode parameters;
     private List<Object> parameterized;
     private List<String> parameterizedCsv;
     private String customLog;
 
+    // Not part of the DSL:
+    // set only on steps read from a stepFile/stepFiles, to show where they came from
+    @JsonIgnore
+    private String stepFileParent;
+
     public Integer getLoop() {
         return loop;
     }
@@ -109,6 +116,14 @@ public void setStepFile(JsonNode stepFile) {
         this.stepFile = stepFile;
     }
 
+    public JsonNode getParameters() {
+        return parameters;
+    }
+
+    public void setParameters(JsonNode parameters) {
+        this.parameters = parameters;
+    }
+
 
     public List<Object> getParameterized() {
         return parameterized;
@@ -128,6 +143,18 @@ public void setParameterizedCsv(List<String> parameterizedCsv) {
 
     public String getCustomLog(){ return customLog; }
 
+    /**
+     * @return the "name (id: ...)" of the step that referenced the stepFile/stepFiles this step came from,
+     * or null when this step is written directly in the scenario
+     */
+    public String getStepFileParent() {
+        return stepFileParent;
+    }
+
+    public void setStepFileParent(String stepFileParent) {
+        this.stepFileParent = stepFileParent;
+    }
+
     public void setCustomLog(String customLog) { this.customLog = customLog; }
 
     public boolean getIgnoreStep() {
@@ -181,6 +208,7 @@ public String toString() {
                 ", id='" + id + '\'' +
                 ", stepFile=" + stepFile +
                 ", stepFiles=" + stepFiles +
+                ", parameters=" + parameters +
                 ", parameterized=" + parameterized +
                 ", customLog=" + customLog +
                 '}';

core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/StepFileParametersResolver.java

@@ -0,0 +1,172 @@
+package org.jsmart.zerocode.core.engine.preprocessor;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import org.jsmart.zerocode.core.domain.Step;
+
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static java.util.stream.Collectors.joining;
+import static org.apache.commons.lang3.StringUtils.removeStart;
+import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.JSON_PAYLOAD_FILE;
+import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.STEP_FILE_PARAM;
+import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.YAML_PAYLOAD_FILE;
+import static org.jsmart.zerocode.core.utils.SmartUtils.getJsonFilePhToken;
+
+/**
+ * Replaces ${PARAM.name} (or {{PARAM.name}}) tokens inside an inlined "stepFile"/"stepFiles" with the
+ * values from the "parameters" of the referencing step.
+ * -
+ * Runs before the usual token resolution, so a parameter value may itself carry tokens such as
+ * ${RANDOM.NUMBER} or ${$.earlier_step.response.body.id}, which are then resolved as normal.
+ * -
+ * The parameters apply only to the steps of that one reference and do not leak into the scenario.
+ */
+public class StepFileParametersResolver {
+    private static final String QUOTED_PARAM = Pattern.quote(STEP_FILE_PARAM);
+    private static final Pattern PARAM_TOKEN_PATTERN = Pattern.compile(
+            "\\$\\{" + QUOTED_PARAM + "([^}]+)\\}|\\{\\{" + QUOTED_PARAM + "([^}]+)\\}\\}");
+    private static final String STEP_FILE = "stepFile";
+    private static final String STEP_FILES = "stepFiles";
+    private static final String INLINE_SOURCE = "<inline>";
+
+    private final ObjectMapper objectMapper;
+
+    public StepFileParametersResolver(ObjectMapper objectMapper) {
+        this.objectMapper = objectMapper;
+    }
+
+    /**
+     * Fills in the ${PARAM.x} values of the step file content.
+     *
+     * @param rawStep     the step before the file was read, used to name the file in error messages
+     * @param inlinedStep the step after the file was read
+     * @return the step with its parameters filled in, or inlinedStep as-is if it has no step file
+     */
+    public Step resolve(Step rawStep, Step inlinedStep) {
+        final JsonNode parameters = inlinedStep.getParameters();
+        final boolean hasParameters = parameters != null && !parameters.isNull();
+
+        if (!hasStepFileContent(inlinedStep)) {
+            // If this is regular DSL step, then "parameters" not allowed, throw an exception.
+            failIfParametersPresent(hasParameters, inlinedStep);
+            return inlinedStep;
+        }
+        if (hasParameters && !parameters.isObject()) {
+            // "parameters" should be of shape JSON object
+            throw new RuntimeException(describeSource(rawStep, inlinedStep.getId())
+                    + " has \"parameters\" that is not a JSON object: " + parameters);
+        }
+
+        final ParameterContext context = new ParameterContext(
+                hasParameters ? parameters : JsonNodeFactory.instance.objectNode(),
+                describeSource(rawStep, inlinedStep.getId()));
+
+        final ObjectNode stepNode = objectMapper.valueToTree(inlinedStep);
+        final String stepFileField = inlinedStep.getStepFile() != null ? STEP_FILE : STEP_FILES;
+        stepNode.set(stepFileField, substitute(stepNode.get(stepFileField), context));
+
+        return toStep(stepNode, inlinedStep.getId());
+    }
+
+    private boolean hasStepFileContent(Step step) {
+        final List<JsonNode> stepFiles = step.getStepFiles();
+        return step.getStepFile() != null || (stepFiles != null && !stepFiles.isEmpty());
+    }
+
+    private void failIfParametersPresent(boolean hasParameters, Step step) {
+        if (!hasParameters) {
+            return;
+        }
+        throw new RuntimeException("Step (name: " + step.getName() + ", id: " + step.getId()
+                + ") has \"parameters\", but \"parameters\" is only supported together with \"stepFile\" or \"stepFiles\"");
+    }
+
+    private JsonNode substitute(JsonNode node, ParameterContext context) {
+        if (node.isTextual()) {
+            return TextNode.valueOf(replaceParamTokens(node.asText(), context));
+        }
+        if (node.isArray()) {
+            final ArrayNode resolvedArray = JsonNodeFactory.instance.arrayNode();
+            node.forEach(element -> resolvedArray.add(substitute(element, context)));
+            return resolvedArray;
+        }
+        if (node.isObject()) {
+            final ObjectNode resolvedObject = JsonNodeFactory.instance.objectNode();
+            node.fields().forEachRemaining(field -> resolvedObject.set(field.getKey(), substitute(field.getValue(), context)));
+            return resolvedObject;
+        }
+        return node;
+    }
+
+    /**
+     * Replaces every ${PARAM.x} and {{PARAM.x}} in one text value. Other tokens are left for later resolution.
+     * e.g.
+     * with "parameters": { "username": "alice", "role": "ADMIN", "retries": 3, "tag": "t_${RANDOM.NUMBER}" }
+     * -  "/api/v1/users/${PARAM.username}"                  -> "/api/v1/users/alice"
+     * -  "{{PARAM.role}}:${PARAM.retries}:${PARAM.tag}"     -> "ADMIN:3:t_${RANDOM.NUMBER}"
+     */
+    private String replaceParamTokens(String text, ParameterContext context) {
+        final Matcher matcher = PARAM_TOKEN_PATTERN.matcher(text);
+        final StringBuffer resolved = new StringBuffer();
+        while (matcher.find()) {
+            final String paramName = matcher.group(1) != null ? matcher.group(1) : matcher.group(2);
+            final String paramValue = context.valueOf(paramName, matcher.group(0));
+            matcher.appendReplacement(resolved, Matcher.quoteReplacement(paramValue));
+        }
+        matcher.appendTail(resolved);
+        return resolved.toString();
+    }
+
+    private Step toStep(ObjectNode stepNode, String stepId) {
+        try {
+            return objectMapper.treeToValue(stepNode, Step.class);
+        } catch (JsonProcessingException exx) {
+            throw new RuntimeException("Failed to apply \"parameters\" to step file content of step id: " + stepId, exx);
+        }
+    }
+
+    private String describeSource(Step rawStep, String stepId) {
+        final String sourceFiles = rawStep.getStepFile() != null
+                ? STEP_FILE + " '" + fileNameOf(rawStep.getStepFile()) + "'"
+                : STEP_FILES + " '" + rawStep.getStepFiles().stream().map(this::fileNameOf).collect(joining(", ")) + "'";
+        return sourceFiles + " (id: " + stepId + ")";
+    }
+
+    private String fileNameOf(JsonNode fileReference) {
+        if (!fileReference.isTextual()) {
+            return INLINE_SOURCE;
+        }
+        final String token = getJsonFilePhToken(fileReference.asText());
+        if (token == null) {
+            return fileReference.asText();
+        }
+        return removeStart(removeStart(token, JSON_PAYLOAD_FILE), YAML_PAYLOAD_FILE);
+    }
+
+    private static class ParameterContext {
+        private final JsonNode parameters;
+        private final String sourceDescription;
+
+        ParameterContext(JsonNode parameters, String sourceDescription) {
+            this.parameters = parameters;
+            this.sourceDescription = sourceDescription;
+        }
+
+        String valueOf(String paramName, String paramToken) {
+            final JsonNode value = parameters.get(paramName);
+            if (value == null) {
+                throw new RuntimeException(sourceDescription + " uses " + paramToken
+                        + ", but '" + paramName + "' is not in \"parameters\"");
+            }
+            return value.isValueNode() ? value.asText() : value.toString();
+        }
+    }
+}

core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeExternalFileProcessor.java

@@ -8,5 +8,14 @@ public interface ZeroCodeExternalFileProcessor {
 
     Step resolveExtJsonFile(Step thisStep);
 
+    /**
+     * Replaces the ${PARAM.name} tokens of an inlined stepFile/stepFiles with the step's "parameters".
+     *
+     * @param rawStep     the step as written in the scenario, before the external file was inlined
+     * @param inlinedStep the step after {@link #resolveExtJsonFile(Step)}
+     * @return the step with its step file parameters resolved
+     */
+    Step resolveStepFileParameters(Step rawStep, Step inlinedStep);
+
     List<Step> createFromStepFile(Step thisStep, String stepId);
 }

core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeExternalFileProcessorImpl.java

@@ -46,16 +46,20 @@
 @Singleton
 public class ZeroCodeExternalFileProcessorImpl implements ZeroCodeExternalFileProcessor {
     private static final Logger LOGGER = getLogger(ZeroCodeExternalFileProcessorImpl.class);
+    private static final String UNNAMED_STEP_FILE_PARENT = "(stepFile)";
 
     private final ObjectMapper objectMapper;
 
+    private final StepFileParametersResolver stepFileParametersResolver;
+
     @Inject
     @Named("YamlMapper")
     private ObjectMapper yamlMapper;
 
     @Inject
     public ZeroCodeExternalFileProcessorImpl(ObjectMapper objectMapper) {
         this.objectMapper = objectMapper;
+        this.stepFileParametersResolver = new StepFileParametersResolver(objectMapper);
     }
 
     /**
@@ -97,6 +101,11 @@ public Step resolveExtJsonFile(Step thisStep) {
 
     }
 
+    @Override
+    public Step resolveStepFileParameters(Step rawStep, Step inlinedStep) {
+        return stepFileParametersResolver.resolve(rawStep, inlinedStep);
+    }
+
     @Override
     public List<Step> createFromStepFile(Step thisStep, String stepId) {
         List<Step> thisSteps = new ArrayList<>();
@@ -118,9 +127,26 @@ public List<Step> createFromStepFile(Step thisStep, String stepId) {
                 throw new RuntimeException(e);
             }
         }
+        final String stepFileParent = describeStepFileParent(thisStep);
+        thisSteps.forEach(innerStep -> innerStep.setStepFileParent(stepFileParent));
         return thisSteps;
     }
 
+    private String describeStepFileParent(Step parentStep) {
+        final String name = parentStep.getName();
+        final String id = parentStep.getId();
+        if (name == null && id == null) {
+            return UNNAMED_STEP_FILE_PARENT;
+        }
+        if (name == null) {
+            return "(id: " + id + ")";
+        }
+        if (id == null) {
+            return name;
+        }
+        return name + " (id: " + id + ")";
+    }
+
     /**
      * Digs deep into the nested map and looks for external file reference,if found, replaces the place holder with
      * the file content. This is handy when the engineers wants to drive the common contents from a central place.

core/src/main/java/org/jsmart/zerocode/core/engine/tokens/ZeroCodeValueTokens.java

@@ -34,6 +34,7 @@ public class ZeroCodeValueTokens {
     public static final String ABS_PATH = "ABS.PATH:";
     public static final String JSON_CONTENT = "JSON.CONTENT:";
     public static final String URL_ENCODED = "URLENCODED:";
+    public static final String STEP_FILE_PARAM = "PARAM.";
 
     public static Map<String, Object> globalTokenCache = new HashMap<>();
 

core/src/main/java/org/jsmart/zerocode/core/runner/ZeroCodeMultiStepsScenarioRunnerImpl.java

@@ -182,7 +182,9 @@ private Boolean executeRetryWithSteps(RunNotifier notifier,
                                           Description description,
                                           ScenarioExecutionState scenarioExecutionState,
                                           ScenarioSpec scenario, Step thisStep) {
+        final Step rawStep = thisStep;
         thisStep = extFileProcessor.resolveExtJsonFile(thisStep);
+        thisStep = extFileProcessor.resolveStepFileParameters(rawStep, thisStep);
         thisStep = zeroCodeAssertionsProcessor.resolveJsonContent(thisStep, scenarioExecutionState);
 
         List<Step> thisSteps = extFileProcessor.createFromStepFile(thisStep, thisStep.getId());
@@ -233,12 +235,12 @@ private Boolean executeRetry(RunNotifier notifier,
             retryTillSuccess = true;
         }
 
-        String thisStepName = thisStep.getName();
+        final String thisStepName = displayNameOf(thisStep);
 
         for (int retryCounter = 0; retryCounter < retryMaxTimes; retryCounter++) {
             if(retryCounter > 0){
                 LOGGER.warn("\n\n------------>Retrying...[step][attempt-{}][executions-{}]:'{}' -> '{}'",
-                        retryCounter, (retryCounter+1), scenario.getScenarioName(), thisStep.getName());
+                        retryCounter, (retryCounter+1), scenario.getScenarioName(), thisStepName);
             }
             try {
                 if (retryCounter > 0 && !isEmpty(thisStep.getRetry().getWithSteps())) {
@@ -406,6 +408,20 @@ private Boolean executeRetry(RunNotifier notifier,
         return null;
     }
 
+    /**
+     * Name to show in logs and reports. A step from a stepFile/stepFiles is shown with the step
+     * that used it,
+     * e.g.
+     * "create_admin_reuse (id: create_admin) -> get_user".
+     * The step's own name stays the same, so ${$.get_user...} still works.
+     */
+    private String displayNameOf(Step thisStep) {
+        if (thisStep.getStepFileParent() == null) {
+            return thisStep.getName();
+        }
+        return thisStep.getStepFileParent() + " -> " + thisStep.getName();
+    }
+
     private String expectedValidatorsAsJson(Step thisStep) throws JsonProcessingException {
         if(thisStep.getValidators() == null){
             return "No validators were found for this step";
@@ -421,7 +437,7 @@ private String executeApi(String logPrefixRelationshipId,
         String url = thisStep.getUrl();
         String operationName = thisStep.getOperation();
         String stepId = thisStep.getId();
-        String thisStepName = thisStep.getName();
+        String thisStepName = displayNameOf(thisStep);
 
         // --------------------------------
         // Resolve the URL patterns if any