diff --git a/.gitignore b/.gitignore index 28de45b4..0af7ccdf 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ applications/cli/src/main/resources/static/wallet/ /applications/cli/chainstate/ /e2e-tests/evolution-sdk/node_modules/ /chainstate/ +*.swp +*.swo diff --git a/Earthfile b/Earthfile index 2e17de16..e0c2ffe6 100644 --- a/Earthfile +++ b/Earthfile @@ -5,9 +5,9 @@ ARG --global tag="dev" ARG --global local="true" ARG --global REGISTRY_ORG = "bloxbean" ARG --global build_type="native" -ARG --global STORE_NATIVE_BRANCH="release/2.0.1_devkit" -ARG --global YANO_TAG="v0.1.0-pre6" -ARG --global YANO_VERSION="0.1.0-pre6" +ARG --global STORE_NATIVE_BRANCH="v2.0.4" +ARG --global YANO_TAG="v0.1.0-pre17" +ARG --global YANO_VERSION="0.1.0-pre17" build: LOCALLY diff --git a/README.md b/README.md index 0eae0e28..8889098b 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ For most development scenarios, Yaci DevKit runs a single Cardano node: │ API Layer │ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ │ │ Yaci Store │ │ Ogmios │ │ Kupo │ │ -│ │ (Indexer) │ │ (Optional) │ │ (Optional) │ │ +│ │ (Indexer) │ │ (Tx Eval) │ │ (Optional) │ │ │ └─────────────┘ └─────────────┘ └─────────────┘ │ └─────────────────────────────────────────────────────────────┘ │ @@ -90,8 +90,8 @@ For PV11 hard fork compatible local devnets, use the PV11 beta release from this | **[Yaci Store](https://github.com/bloxbean/yaci-store)** | Lightweight indexer with Blockfrost-compatible APIs | 8080 | | **[Yaci Viewer](./applications/viewer)** | Web-based blockchain explorer for developers | 5173 | | **Cardano Node** | Official Cardano node (supports both amd64/arm64) | 3001 (n2n), 3333 (n2c through socat) | -| **[Ogmios](https://ogmios.dev/)** | WebSocket API for Cardano (optional) | 1337 | -| **[Kupo](https://cardanosolutions.github.io/kupo/)** | Chain indexer (optional) | 1442 | +| **[Ogmios](https://ogmios.dev/) v7.0.0** | WebSocket API for Cardano (enabled by default; used as Yaci Store's tx evaluator) | 1337 | +| **[Kupo](https://cardanosolutions.github.io/kupo/) v2.12.0** | Chain indexer (optional) | 1442 | | **MCP Server** | AI coding assistant integration | 10000 | ## 🎯 Quick Start @@ -173,7 +173,9 @@ yaci-cli> create-node -o --start (Can be used in a Java app with Cardano Client Lib's Blockfrost backend or [Javascript app with MeshJS + Blockfrost provider](https://github.com/MeshJS/examples/blob/main/mesh/yaci-send-lovelace.ts) as it exposes required BF compatible minimum apis for tx building and submission) - **Yaci Store Swagger UI**: http://localhost:8080/swagger-ui/index.html -- **Ogmios Url (Optional)**: http://localhost:1337 +- **Ogmios Url**: http://localhost:1337 + + (Enabled by default and used by Yaci Store to evaluate Plutus script costs. Set `ogmios_enabled=false` in `config/env` to fall back to the embedded Scalus evaluator.) **(Optional) If you enabled Kupo** - **Kupo Url (Optional)**: http://localhost:1442 diff --git a/applications/cli/Earthfile b/applications/cli/Earthfile index 6e1d3b30..2319dda6 100644 --- a/applications/cli/Earthfile +++ b/applications/cli/Earthfile @@ -67,9 +67,9 @@ docker-build: ARG APP_VERSION ARG REGISTRY_ORG ARG COMMIT_ID - ARG STORE_NATIVE_BRANCH="release/2.0.1_devkit" - ARG YANO_TAG="v0.1.0-pre6" - ARG YANO_VERSION="0.1.0-pre6" + ARG STORE_NATIVE_BRANCH="v2.0.4" + ARG YANO_TAG="v0.1.0-pre17" + ARG YANO_VERSION="0.1.0-pre17" COPY --dir +java-setup/openjdk /opt/java/ ENV PATH="${JAVA_HOME}/bin:${PATH}" @@ -91,11 +91,6 @@ docker-build: COPY docker/download-$TARGETARCH.sh . RUN sh download-$TARGETARCH.sh - # PV11 Ogmios/Kupo IntersectMBO artifacts are Linux x86_64-only for now. - # The arm64 image bundles those x86_64 binaries as a temporary Apple Silicon - # Docker Desktop workaround; native Linux arm64 hosts may fail at runtime - # unless x86_64 binfmt/qemu emulation is available. - #Ogmios download COPY docker/download-ogmios.sh . RUN bash download-ogmios.sh $TARGETARCH diff --git a/applications/cli/config/application.properties b/applications/cli/config/application.properties index 5081b08e..cbb220f5 100644 --- a/applications/cli/config/application.properties +++ b/applications/cli/config/application.properties @@ -28,6 +28,13 @@ bp.create.enabled=true #yano.server.port=14447 #yano.http.port=6060 +## Spacing for Yano's past-time-travel backfill, the catch-up from the shifted genesis to wall clock. +## auto (default): Yano derives the spacing from genesis, placing only the blocks a Haskell relay needs +## to validate the history - gaps stay inside the forecast window (3k/f) and the epoch-boundary blocks +## required for nonces are kept. Create time then stops growing with epoch length. +## dense: one block per slot, Yano's own default. An integer forces the interval in slots. +#yano.backfill.block.interval.slots=auto + ## Whether Yano's past-time-travel backfill runs Praos slot-leader checks, placing blocks only in ## VRF-eligible slots. auto (default): on when a Haskell node validates the chain (companion, yano-primary) ## and activeSlotsCoeff is below 1 (block time longer than slot length), and for local multi-node devnets. diff --git a/applications/cli/config/download.properties b/applications/cli/config/download.properties index ab51f963..84461a8f 100644 --- a/applications/cli/config/download.properties +++ b/applications/cli/config/download.properties @@ -1,19 +1,20 @@ #Please specify either the version or the full url for the following components node.version=11.0.1 -ogmios.version=6.14.0.2 -kupo.version=2.11.0.1 +ogmios.version=7.0.0 +kupo.version=2.12.0 -yaci.store.tag=rel-native-2.0.1-devkit -yaci.store.version=2.0.1-devkit -yaci.store.jar.version=2.0.1 +yaci.store.tag=rel-native-2.0.4 +yaci.store.version=2.0.4 +yaci.store.jar.version=2.0.4 -yano.tag=v0.1.0-pre6 -yano.version=0.1.0-pre6 +yano.tag=v0.1.0-pre17 +yano.version=0.1.0-pre17 #node.url= -# IntersectMBO currently publishes the PV11-compatible Ogmios/Kupo builds only -# for Linux x86_64. These default versions resolve on Linux x86_64; on other -# platforms, set a compatible custom URL here when one is available. +# Ogmios/Kupo are published by CardanoSolutions for x86_64-linux, aarch64-linux +# and aarch64-macos. The Linux builds are statically linked; the macOS builds link +# only against system libraries. macOS x86_64 and Windows are not published : on +# those set a compatible custom URL here or use the DevKit Docker image. #ogmios.url= #kupo.url= #yaci.store.url= diff --git a/applications/cli/config/node.properties b/applications/cli/config/node.properties index 896c9bb2..20596a66 100644 --- a/applications/cli/config/node.properties +++ b/applications/cli/config/node.properties @@ -4,7 +4,7 @@ nodeMode=companion #protocolMagic=42 -#maxKESEvolutions=60 +#maxKESEvolutions=62 #securityParam=80 ## Fraction of the epoch covered by the stability window (3k/f) when securityParam is derived (securityParam=0). @@ -12,12 +12,17 @@ nodeMode=companion ## gap in block production shorter than 3k/f slots, so this factor times the epoch's wall-clock length is the ## idle tolerance of the devnet. Values up to about 0.7 are sound (4k/f must stay below the epoch length). #stabilityWindowFactor=0.5 -#slotsPerKESPeriod=129600 + +## A KES period long enough that a past-time-travel devnet never leaves period 0. The bootstrap shift +## moves genesis back by whole epochs before catching up to wall clock, and the node derives the current +## KES period as slot / slotsPerKESPeriod, so at the mainnet 129600 a long-epoch devnet burns a large +## share of its evolutions before producing a block. Devnets do not exercise KES rotation. +#slotsPerKESPeriod=10000000 #updateQuorum=1 #peerSharing=true -## Node log tracing of the chain-sync client. On by default so a bootstrap chain the node rejects -## (for example VRFLeaderValueTooBig at activeSlotsCoeff below 1) shows up in the node log. +## Node log tracing of the chain-sync client. Off by default. Turn it on to see why the node rejects +## a bootstrap chain (for example VRFLeaderValueTooBig at activeSlotsCoeff below 1). #traceChainSyncClient=true ## Shelley Genesis diff --git a/applications/cli/docker/download-kupo.sh b/applications/cli/docker/download-kupo.sh index f4631115..898a38b4 100644 --- a/applications/cli/docker/download-kupo.sh +++ b/applications/cli/docker/download-kupo.sh @@ -1,21 +1,13 @@ #!/bin/bash set -euo pipefail -# PV11 workaround: IntersectMBO currently publishes the cardano-node 11.0.1 -# compatible Kupo release only as a Linux x86_64 tarball. We bundle it for -# arm64 Docker images too so Apple Silicon Docker Desktop can run it through -# emulation. This may fail at runtime on native Linux arm64 hosts without -# x86_64 binfmt/qemu support. Keep the original CardanoSolutions downloader in -# download-kupo.sh.original for reverting once upstream publishes the needed -# assets again. +# Assign the architecture based on user input and determine the correct suffix case "${1:-}" in amd64) ARCH="x86_64" ;; arm64) - echo "Warning: IntersectMBO Kupo v2.11.0.1 has no Linux arm64 asset." - echo "Bundling the Linux x86_64 artifact for arm64 Docker image compatibility on Docker Desktop." - ARCH="x86_64" + ARCH="aarch64" ;; *) echo "Error: Invalid architecture specified. Use 'amd64' or 'arm64'." @@ -23,15 +15,17 @@ case "${1:-}" in ;; esac -version=v2.11.0.1 -file=kupo-${version}-${ARCH}-linux.tar.gz -url=https://github.com/IntersectMBO/kupo/releases/download/${version}/${file} +# Kupo release tags are major.minor (v2.12) while the asset carries the full version (v2.12.0) +version=v2.12.0 +tag=v2.12 +file=kupo-${version}-${ARCH}-linux.zip +url=https://github.com/CardanoSolutions/kupo/releases/download/${tag}/${file} wget "${url}" rm -rf /app/kupo mkdir -p /app/kupo -tar -xzf "${file}" -C /app/kupo +unzip "${file}" -d /app/kupo chmod +x /app/kupo/bin/kupo diff --git a/applications/cli/docker/download-kupo.sh.original b/applications/cli/docker/download-kupo.sh.original deleted file mode 100644 index 5db3c8ff..00000000 --- a/applications/cli/docker/download-kupo.sh.original +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Assign the architecture based on user input and determine the correct suffix -case $1 in - amd64) - ARCH="x86_64" - ;; - arm64) - ARCH="aarch64" - ;; - *) - echo "Error: Invalid architecture specified. Use 'amd64' or 'arm64'." - exit 1 - ;; -esac - -file=kupo-v2.11.0-$ARCH-linux.zip -wget https://github.com/CardanoSolutions/kupo/releases/download/v2.11/$file - -mkdir /app/kupo -unzip $file -d /app/kupo - -chmod +x /app/kupo/bin/kupo - -rm $file diff --git a/applications/cli/docker/download-ogmios.sh b/applications/cli/docker/download-ogmios.sh index 3a56223d..d323a425 100644 --- a/applications/cli/docker/download-ogmios.sh +++ b/applications/cli/docker/download-ogmios.sh @@ -1,21 +1,13 @@ #!/bin/bash set -euo pipefail -# PV11 workaround: IntersectMBO currently publishes the cardano-node 11.0.1 -# compatible Ogmios release only as a Linux x86_64 tarball. We bundle it for -# arm64 Docker images too so Apple Silicon Docker Desktop can run it through -# emulation. This may fail at runtime on native Linux arm64 hosts without -# x86_64 binfmt/qemu support. Keep the original CardanoSolutions downloader in -# download-ogmios.sh.original for reverting once upstream publishes the needed -# assets again. +# Assign the architecture based on user input and determine the correct suffix case "${1:-}" in amd64) ARCH="x86_64" ;; arm64) - echo "Warning: IntersectMBO Ogmios v6.14.0.2 has no Linux arm64 asset." - echo "Bundling the Linux x86_64 artifact for arm64 Docker image compatibility on Docker Desktop." - ARCH="x86_64" + ARCH="aarch64" ;; *) echo "Error: Invalid architecture specified. Use 'amd64' or 'arm64'." @@ -23,15 +15,15 @@ case "${1:-}" in ;; esac -version=v6.14.0.2 -file=ogmios-${version}-${ARCH}-linux.tar.gz -url=https://github.com/IntersectMBO/ogmios/releases/download/${version}/${file} +version=v7.0.0 +file=ogmios-${version}-${ARCH}-linux.zip +url=https://github.com/CardanoSolutions/ogmios/releases/download/${version}/${file} wget "${url}" rm -rf /app/ogmios mkdir -p /app/ogmios -tar -xzf "${file}" -C /app/ogmios +unzip "${file}" -d /app/ogmios chmod +x /app/ogmios/bin/ogmios diff --git a/applications/cli/docker/download-ogmios.sh.original b/applications/cli/docker/download-ogmios.sh.original deleted file mode 100644 index 93f34bd9..00000000 --- a/applications/cli/docker/download-ogmios.sh.original +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# Assign the architecture based on user input and determine the correct suffix -case $1 in - amd64) - SUFFIX="x86_64" - ;; - arm64) - SUFFIX="aarch64" - ;; - *) - echo "Error: Invalid architecture specified. Use 'amd64' or 'arm64'." - exit 1 - ;; -esac - - -version=v6.14.0 -file=ogmios-${version}-${SUFFIX}-linux.zip -wget https://github.com/CardanoSolutions/ogmios/releases/download/${version}/$file - -mkdir /app/ogmios -unzip $file -d /app/ogmios - -chmod +x /app/ogmios/bin/ogmios - -rm $file diff --git a/applications/cli/docker/download-yano.sh b/applications/cli/docker/download-yano.sh index 2955988c..39bfb1f1 100644 --- a/applications/cli/docker/download-yano.sh +++ b/applications/cli/docker/download-yano.sh @@ -14,7 +14,7 @@ case "${1:-}" in ;; esac -VERSION="${2:-0.1.0-pre6}" +VERSION="${2:-0.1.0-pre17}" TAG="${3:-v${VERSION}}" file="yano-native-${VERSION}-linux-${ARCH}.zip" diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/common/DownloadService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/common/DownloadService.java index e5d50179..2ae290ad 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/common/DownloadService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/common/DownloadService.java @@ -32,10 +32,6 @@ public class DownloadService { private final static String YACI_STORE_DOWNLOAD_URL = "https://github.com/bloxbean/yaci-store/releases/download"; private final static String OGMIOS_DOWNLOAD_URL = "https://github.com/CardanoSolutions/ogmios/releases/download"; private final static String KUPO_DOWNLOAD_URL = "https://github.com/CardanoSolutions/kupo/releases/download"; - private final static String INTERSECT_OGMIOS_DOWNLOAD_URL = "https://github.com/IntersectMBO/ogmios/releases/download"; - private final static String INTERSECT_KUPO_DOWNLOAD_URL = "https://github.com/IntersectMBO/kupo/releases/download"; - private final static String INTERSECT_OGMIOS_VERSION = "6.14.0.2"; - private final static String INTERSECT_KUPO_VERSION = "2.11.0.1"; private final static String YANO_DOWNLOAD_URL = "https://github.com/bloxbean/yano/releases/download"; private final ClusterConfig clusterConfig; @@ -315,8 +311,8 @@ public boolean downloadJre(boolean overwrite) { public boolean downloadOgmios(boolean overwrite) { String downloadPath = resolveOgmiosDownloadPath(); - if ( downloadPath == null) { - writeLn(error("Download URL for ogmios is not set. Please set the download URL in download.properties")); + if (downloadPath == null) { + //resolveOgmiosDownloadPath() has already reported the specific reason return false; } @@ -337,7 +333,14 @@ public boolean downloadOgmios(boolean overwrite) { if (downloadedFile != null) { try { extractArchive(downloadedFile.toFile().getAbsolutePath(), clusterConfig.getOgmiosHome(), downloadPath); + flattenNestedArchiveLayout(clusterConfig.getOgmiosHome(), "ogmios"); + if (!ogmiosExec.toFile().exists()) { + writeLn(error("ogmios binary was not found at %s after extraction. The downloaded archive may be invalid.", + ogmiosExec.toFile().getAbsolutePath())); + return false; + } setExecutablePermission(ogmiosExec.toFile().getAbsolutePath()); + Files.deleteIfExists(downloadedFile); return true; } catch (IOException e) { writeLn(error("Error extracting ogmios" + e.getMessage())); @@ -352,8 +355,8 @@ public boolean downloadOgmios(boolean overwrite) { public boolean downloadKupo(boolean overwrite) { String downloadPath = resolveKupoDownloadPath(); - if ( downloadPath == null) { - writeLn(error("Download URL for Kupo is not set. Please set the download URL in download.properties")); + if (downloadPath == null) { + //resolveKupoDownloadPath() has already reported the specific reason return false; } @@ -374,7 +377,14 @@ public boolean downloadKupo(boolean overwrite) { if (downloadedFile != null) { try { extractArchive(downloadedFile.toFile().getAbsolutePath(), clusterConfig.getKupoHome(), downloadPath); + flattenNestedArchiveLayout(clusterConfig.getKupoHome(), "kupo"); + if (!kupoExec.toFile().exists()) { + writeLn(error("kupo binary was not found at %s after extraction. The downloaded archive may be invalid.", + kupoExec.toFile().getAbsolutePath())); + return false; + } setExecutablePermission(kupoExec.toFile().getAbsolutePath()); + Files.deleteIfExists(downloadedFile); return true; } catch (IOException e) { writeLn(error("Error extracting kupo" + e.getMessage())); @@ -558,6 +568,44 @@ private String resolveArchiveFileName(String component, String downloadPath) { return isTarGz(downloadPath) ? component + ".tar.gz" : component + ".zip"; } + /** + * CardanoSolutions release archives are not consistent across platforms : the Ogmios macOS build + * nests everything under a single version directory (v7.0.0/bin/ogmios) while the Linux builds are + * flat (bin/ogmios). Flatten the nested layout so the executable is always at + * <targetDir>/bin/<binaryName>, which is what the runtime services expect. + */ + private void flattenNestedArchiveLayout(String targetDir, String binaryName) throws IOException { + Path target = Path.of(targetDir); + if (Files.exists(target.resolve("bin").resolve(binaryName))) + return; + + Path nestedRoot = null; + try (var children = Files.list(target)) { + var candidates = children.filter(Files::isDirectory) + .filter(dir -> Files.exists(dir.resolve("bin").resolve(binaryName))) + .toList(); + if (candidates.size() == 1) + nestedRoot = candidates.get(0); + } + + if (nestedRoot == null) + return; + + writeLn(info("Flattening nested archive layout : %s", nestedRoot.getFileName().toString())); + + //Move files individually rather than whole directories : moving a directory onto an existing + //non-empty directory fails with DirectoryNotEmptyException even with REPLACE_EXISTING. + try (var files = Files.walk(nestedRoot)) { + for (Path source : files.filter(Files::isRegularFile).toList()) { + Path destination = target.resolve(nestedRoot.relativize(source).toString()); + Files.createDirectories(destination.getParent()); + Files.move(source, destination, StandardCopyOption.REPLACE_EXISTING); + } + } + + FileUtils.deleteDirectory(nestedRoot.toFile()); + } + private boolean isTarGz(String downloadPath) { String normalized = downloadPath.toLowerCase(Locale.ROOT); return normalized.contains(".tar.gz") || normalized.contains(".tgz"); @@ -702,36 +750,11 @@ private String resolveOgmiosDownloadPath() { return null; } - if (INTERSECT_OGMIOS_VERSION.equals(ogmiosVersion)) { - if (!isLinuxX86_64()) { - writeLn(error("Ogmios %s is currently available from IntersectMBO only for Linux x86_64. Set ogmios.url to a compatible build or use the DevKit Docker image.", ogmiosVersion)); - return null; - } - - return INTERSECT_OGMIOS_DOWNLOAD_URL + "/v" + ogmiosVersion + "/ogmios-v" + ogmiosVersion + "-x86_64-linux.tar.gz"; - } - - String osPrefix = null; - if (SystemUtils.IS_OS_MAC) { - osPrefix = "macos"; - } else if (SystemUtils.IS_OS_LINUX) { - osPrefix = "linux"; - } else { - writeLn(error("Unsupported OS : " + System.getProperty("os.name"))); - } - - if (osPrefix == null) + String platformSuffix = resolveOgmiosKupoPlatformSuffix("Ogmios", "ogmios.url"); + if (platformSuffix == null) return null; - String arch = System.getProperty("os.arch"); - String cpuArch = null; - if (arch.startsWith("aarch") || arch.startsWith("arm")) { - cpuArch = "aarch64"; - } else{ - cpuArch = "x86_64"; - } - - String url = OGMIOS_DOWNLOAD_URL + "/v" + ogmiosVersion + "/ogmios-v" + ogmiosVersion + "-" + cpuArch + "-" + osPrefix + ".zip"; + String url = OGMIOS_DOWNLOAD_URL + "/v" + ogmiosVersion + "/ogmios-v" + ogmiosVersion + "-" + platformSuffix + ".zip"; return url; } @@ -745,47 +768,51 @@ private String resolveKupoDownloadPath() { return null; } - if (INTERSECT_KUPO_VERSION.equals(kupoVersion)) { - if (!isLinuxX86_64()) { - writeLn(error("Kupo %s is currently available from IntersectMBO only for Linux x86_64. Set kupo.url to a compatible build or use the DevKit Docker image.", kupoVersion)); - return null; - } + String platformSuffix = resolveOgmiosKupoPlatformSuffix("Kupo", "kupo.url"); + if (platformSuffix == null) + return null; - return INTERSECT_KUPO_DOWNLOAD_URL + "/v" + kupoVersion + "/kupo-v" + kupoVersion + "-x86_64-linux.tar.gz"; - } + //Kupo release tags are major.minor (e.g. v2.12) while the asset name carries the + //full version (e.g. kupo-v2.12.0-x86_64-linux.zip) + String versionParts[] = kupoVersion.split("\\."); + String trimmedVersionPath = kupoVersion; + if (versionParts.length == 3) + trimmedVersionPath = versionParts[0] + "." + versionParts[1]; - String osPrefix = null; + String url = KUPO_DOWNLOAD_URL + "/v" + trimmedVersionPath + "/kupo-v" + kupoVersion + "-" + platformSuffix + ".zip"; + return url; + } + + /** + * Resolves the "<cpuArch>-<os>" suffix used by CardanoSolutions Ogmios/Kupo release assets. + * Only x86_64-linux, aarch64-linux and aarch64-macos builds are published, so any other OS/arch + * combination fails fast with an actionable message instead of a 404 download. + */ + private String resolveOgmiosKupoPlatformSuffix(String component, String urlProperty) { + String osPrefix; if (SystemUtils.IS_OS_MAC) { osPrefix = "macos"; } else if (SystemUtils.IS_OS_LINUX) { osPrefix = "linux"; } else { writeLn(error("Unsupported OS : " + System.getProperty("os.name"))); - } - - if (osPrefix == null) return null; + } String arch = System.getProperty("os.arch"); - String cpuArch = null; + String cpuArch; if (arch.startsWith("aarch") || arch.startsWith("arm")) { cpuArch = "aarch64"; - } else{ + } else { cpuArch = "x86_64"; } - String versionParts[] = kupoVersion.split("\\."); - String trimmedVersionPath = kupoVersion; - if (versionParts.length == 3) - trimmedVersionPath = versionParts[0] + "." + versionParts[1]; - - String url = KUPO_DOWNLOAD_URL + "/v" + trimmedVersionPath + "/kupo-v" + kupoVersion + "-" + cpuArch + "-" + osPrefix + ".zip"; - return url; - } + if (SystemUtils.IS_OS_MAC && "x86_64".equals(cpuArch)) { + writeLn(error("%s is not published for macOS x86_64. Set %s to a compatible build or use the DevKit Docker image.", component, urlProperty)); + return null; + } - private boolean isLinuxX86_64() { - String arch = System.getProperty("os.arch"); - return SystemUtils.IS_OS_LINUX && !arch.startsWith("aarch") && !arch.startsWith("arm"); + return cpuArch + "-" + osPrefix; } private String resolveYanoDownloadPath() { diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/general/DownloadCommand.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/general/DownloadCommand.java index 177b69a3..3b7f7f0e 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/general/DownloadCommand.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/general/DownloadCommand.java @@ -112,8 +112,12 @@ public void downloadAndStart( if (enableYaciStore) { applicationConfig.setYaciStoreEnabled(true); - applicationConfig.setOgmiosEnabled(false); //TODO -- Temporarily till Ogmios is ready - } else if (enableKupomios){ + //Ogmios is the default tx evaluator for Yaci Store. If Ogmios fails to start, + //Yaci Store falls back to the embedded Scalus evaluator. + applicationConfig.setOgmiosEnabled(true); + } + + if (enableKupomios) { applicationConfig.setOgmiosEnabled(true); applicationConfig.setKupoEnabled(true); } @@ -124,7 +128,9 @@ public void downloadAndStart( if (!componentList.contains("ogmios")) { componentList.add("ogmios"); } - } else if (enableKupomios) { + } + + if (enableKupomios) { if (!componentList.contains("ogmios")) componentList.add("ogmios"); if (!componentList.contains("kupo")) diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/ClusterAdminController.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/ClusterAdminController.java index f11ee3bf..1d8144be 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/ClusterAdminController.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/ClusterAdminController.java @@ -240,7 +240,11 @@ public boolean create(@RequestBody DevNetCreateRequest request) { boolean originalEnableKupo = applicationConfig.isKupoEnabled(); applicationConfig.setYaciStoreEnabled(request.enableYaciStore()); - applicationConfig.setOgmiosEnabled(request.enableOgmios()); + //enableOgmios is optional : when the caller omits it, keep the configured default + //(ogmios_enabled in config/env) instead of forcing Ogmios off. + applicationConfig.setOgmiosEnabled(request.enableOgmios() != null + ? request.enableOgmios() + : originalEnableOgmios); if (request.enableKupomios()) { applicationConfig.setOgmiosEnabled(true); @@ -307,8 +311,8 @@ record DevNetCreateRequest(Map genesisProperties, int multiNodeStakeRatioFactor, @Schema(description = "Enable Yaci Store", defaultValue = "false") boolean enableYaciStore, - @Schema(description = "Enable Ogmios", defaultValue = "false") - boolean enableOgmios, + @Schema(description = "Enable Ogmios. If omitted, the configured default (ogmios_enabled) is used.") + Boolean enableOgmios, @Schema(description = "Enable Ogmios and Kupo", defaultValue = "false") boolean enableKupomios) { } diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/config/GenesisConfig.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/config/GenesisConfig.java index 54f0e648..73218e4c 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/config/GenesisConfig.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/config/GenesisConfig.java @@ -16,15 +16,21 @@ public class GenesisConfig { private String networkId = "Testnet"; private long protocolMagic = 42; - private int maxKESEvolutions = 60; + private int maxKESEvolutions = 62; private double stabilityWindowFactor = 0.5; //This is used to automatically derive the security parameter from epoch length private int securityParam = 0; - private long slotsPerKESPeriod = 129600; + //A KES period long enough that a past-time-travel devnet never leaves period 0. The bootstrap + //shift shifts genesis back by whole epochs before catching up to wall clock, and the node derives + //the current KES period as slot / slotsPerKESPeriod : at the mainnet 129600 a long-epoch devnet + //burns a large share of its evolutions before producing a single block. Devnets do not exercise + //KES rotation, so trade that realism for a devnet that does not expire. + private long slotsPerKESPeriod = 10000000; private int updateQuorum = 1; private boolean peerSharing = true; - // Node config TraceChainSyncClient. On by default so a bootstrap chain the node rejects - // (for example VRFLeaderValueTooBig at activeSlotsCoeff below 1) shows up in the node log. - private boolean traceChainSyncClient = true; + // Node config TraceChainSyncClient. Off by default: it logs every header received from a peer, which + // adds up on multi-node devnets. Turn it on to see why a node rejects a bootstrap chain + // (for example VRFLeaderValueTooBig at activeSlotsCoeff below 1). + private boolean traceChainSyncClient = false; private String genesisUtxoSupply = "30000000000000000"; //In byron genesis private int nGenesisKeys = 3; //For new priv network diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ogmios/OgmiosService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ogmios/OgmiosService.java index 6b968242..09005908 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ogmios/OgmiosService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ogmios/OgmiosService.java @@ -19,8 +19,13 @@ import java.io.File; import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.*; import java.util.concurrent.*; import java.util.function.Consumer; @@ -41,6 +46,10 @@ public class OgmiosService { private final TemplateEngine templateEngine; private final ProcessUtil processUtil; + private final HttpClient httpClient = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(2)) + .build(); + private List processes = new ArrayList<>(); private Process ogmiosProcess; @@ -52,8 +61,14 @@ public class OgmiosService { public void handleClusterStarted(ClusterStarted clusterStarted) { String clusterName = clusterStarted.getClusterName(); - start(clusterName, msg -> writeLn(msg)); - + //Ogmios is optional. This listener runs before YaciStoreService's (@Order(1)), so an exception here + //would abort the rest of the startup chain and prevent Yaci Store from starting at all. + try { + start(clusterName, msg -> writeLn(msg)); + } catch (Exception e) { + log.error("Error starting Ogmios/Kupo", e); + writeLn(error("Could not start Ogmios/Kupo : " + e.getMessage())); + } } public boolean start(String clusterName, Consumer writer) { @@ -78,6 +93,13 @@ public boolean start(String clusterName, Consumer writer) { throw new IllegalStateException("Cluster info not found for cluster: " + clusterName + ". Please check if the cluster is created."); + //Ogmios/Kupo talk to the Haskell node over its n2c socket, which doesn't exist in yano-only mode. + NodeMode nodeMode = clusterInfo.getNodeMode() != null ? clusterInfo.getNodeMode() : NodeMode.HASKELL_ONLY; + if (nodeMode == NodeMode.YANO_ONLY) { + writer.accept(info("Ogmios/Kupo require a Haskell cardano-node and are not supported in yano-only mode. Skipping.")); + return false; + } + if (appConfig.isOgmiosEnabled()) { if (!ogmiosPortAvailabilityCheck(clusterInfo, writer)) return false; @@ -108,6 +130,49 @@ public boolean isOgmiosRunning() { return ogmiosProcess != null && ogmiosProcess.isAlive(); } + /** + * Waits until Ogmios actually answers on its HTTP health endpoint. + *

+ * {@link #isOgmiosRunning()} only reports that the process was still alive a second after spawn, which is + * not enough to hand Yaci Store the "ogmios" tx evaluator : if Ogmios dies or never connects to the node + * socket shortly after start, the evaluator mode is already baked into the Store process and every Plutus + * script evaluation would fail for the lifetime of the devnet. Callers use this to fall back to Scalus. + */ + public boolean waitForOgmiosReady(int ogmiosPort, Consumer writer) { + String healthUrl = "http://localhost:" + ogmiosPort + "/health"; + int maxAttempts = 15; + + for (int i = 0; i < maxAttempts; i++) { + if (!isOgmiosRunning()) { + writer.accept(warn("Ogmios process is no longer running.")); + return false; + } + + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(healthUrl)) + .timeout(Duration.ofSeconds(2)) + .GET() + .build(); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() == 200) + return true; + } catch (Exception e) { + //Not ready yet + } + + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return false; + } + } + + writer.accept(warn("Ogmios did not become ready within timeout.")); + return false; + } + private static boolean ogmiosPortAvailabilityCheck(ClusterInfo clusterInfo, Consumer writer) { boolean ogmiosPortAvailable = PortUtil.isPortAvailable(clusterInfo.getOgmiosPort()); if (!ogmiosPortAvailable) { diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yacistore/YaciStoreService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yacistore/YaciStoreService.java index 6e0d2962..bd8a61ac 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yacistore/YaciStoreService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yacistore/YaciStoreService.java @@ -380,7 +380,7 @@ private StoreStartResult startStoreApp(ClusterInfo clusterInfo, Era era, Consume builder.directory(new File(clusterConfig.getYaciStoreBinPath())); boolean yanoOnly = NodeMode.YANO_ONLY == clusterInfo.getNodeMode(); - String txEvaluatorMode = resolveTxEvaluatorMode(writer); + String txEvaluatorMode = resolveTxEvaluatorMode(clusterInfo, writer); builder.environment().put("STORE_SUBMIT_TX_EVALUATOR_MODE", txEvaluatorMode); writer.accept(info("Yaci Store tx evaluator mode: " + txEvaluatorMode)); @@ -522,15 +522,25 @@ private StoreStartResult startStoreApp(ClusterInfo clusterInfo, Era era, Consume return new StoreStartResult(process, started.get()); } - private String resolveTxEvaluatorMode(Consumer writer) { - if (appConfig.isOgmiosEnabled() && ogmiosService.isOgmiosRunning()) - return TX_EVALUATOR_MODE_OGMIOS; + private String resolveTxEvaluatorMode(ClusterInfo clusterInfo, Consumer writer) { + if (!appConfig.isOgmiosEnabled()) + return TX_EVALUATOR_MODE_SCALUS; - if (appConfig.isOgmiosEnabled()) { + if (!ogmiosService.isOgmiosRunning()) { writer.accept(warn("Ogmios is enabled but not running. Using Scalus tx evaluator for Yaci Store.")); + return TX_EVALUATOR_MODE_SCALUS; } - return TX_EVALUATOR_MODE_SCALUS; + //The mode is baked into the Store process at startup and never re-evaluated, so confirm Ogmios is + //actually serving before committing to it. Otherwise a short-lived Ogmios would leave the Store + //unable to evaluate any Plutus script for the lifetime of the devnet. + if (!ogmiosService.waitForOgmiosReady(clusterInfo.getOgmiosPort(), writer)) { + writer.accept(warn("Ogmios is not responding on port " + clusterInfo.getOgmiosPort() + + ". Using Scalus tx evaluator for Yaci Store.")); + return TX_EVALUATOR_MODE_SCALUS; + } + + return TX_EVALUATOR_MODE_OGMIOS; } private Process startViewerApp(String cluster) throws IOException, InterruptedException, ExecutionException, TimeoutException { diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoConfigBuilder.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoConfigBuilder.java index 415b239b..a0de3c72 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoConfigBuilder.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoConfigBuilder.java @@ -34,13 +34,15 @@ public class YanoConfigBuilder { * @param clusterInfo cluster configuration * @param yanoConfigDir resolved path to Yano's devnet config (genesis files, keys) * @param yanoDataDir resolved path for Yano's chainstate storage + * @param yanoHistoryDir resolved path for Yano's history archive * @param pastTimeTravelMode whether past-time-travel mode is enabled * @param slotLeaderTimeTravel whether the past-time-travel backfill runs Praos slot-leader checks * (see {@link YanoService#slotLeaderTimeTravelEnabled(ClusterInfo)}) + * @param backfillBlockIntervalSlots slots between blocks in the past-time-travel backfill; 0 = automatic * @return true if config was written successfully */ - public boolean build(ClusterInfo clusterInfo, Path yanoConfigDir, Path yanoDataDir, - boolean pastTimeTravelMode, boolean slotLeaderTimeTravel) { + public boolean build(ClusterInfo clusterInfo, Path yanoConfigDir, Path yanoDataDir, Path yanoHistoryDir, + boolean pastTimeTravelMode, boolean slotLeaderTimeTravel, int backfillBlockIntervalSlots) { Map props = new LinkedHashMap<>(); // Quarkus profile @@ -66,12 +68,26 @@ public boolean build(ClusterInfo clusterInfo, Path yanoConfigDir, Path yanoDataD // Storage — inside the node folder so it gets cleaned up with create-node -o props.put("yano.storage.path", yanoDataDir.toAbsolutePath().toString()); + // History archive — Yano's bundled devnet profile enables the DuckLake projection + // archive by default. DevKit reads nothing from Yano's history api, so it's turned off + // here. The dir is pinned inside the node folder anyway: it defaults to ./history, + // relative to the working directory, which is the shared yanoHome — an archive written + // there outlives create-node -o and then fails the startup identity/coverage guards. + props.put("yano.history.projection.enabled", "false"); + props.put("yano.history.dir", yanoHistoryDir.toAbsolutePath().toString()); + // Past-time-travel mode if (pastTimeTravelMode) { props.put("yano.block-producer.past-time-travel-mode", "true"); if (slotLeaderTimeTravel) { props.put("yano.block-producer.past-time-travel-slot-leader-mode", "true"); } + + //Spacing for the catch-up from the shifted genesis to wall clock. 0 = automatic : Yano derives + //the widest spacing that still fits inside the forecast window a Haskell relay can validate, + //and keeps the epoch-boundary blocks needed for nonces. 1 is one block per slot (Yano's default), + //which makes create time grow with epoch length. + props.put("yano.block-producer.backfill-block-interval-slots", String.valueOf(backfillBlockIntervalSlots)); } // Write to {yanoHome}/config/application.properties diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java index 936c24b8..b7de1251 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java @@ -49,6 +49,14 @@ public class YanoService { private List executors = new ArrayList<>(); private Queue logs = EvictingQueue.create(300); + // Spacing for Yano's past-time-travel backfill, the catch-up from the shifted genesis to wall clock. + // auto (0): Yano derives the spacing from genesis, keeping every gap inside the forecast window a + // Haskell relay can validate and still placing the epoch-boundary blocks it needs for nonces. + // dense (1): one block per slot, Yano's own default, which makes create time grow with epoch length. + // An integer forces the interval. + @Value("${yano.backfill.block.interval.slots:auto}") + private String backfillBlockIntervalSlots = "auto"; + // Whether Yano's past-time-travel backfill runs Praos slot-leader checks (VRF-eligible slots only). // auto: on when a Haskell node will validate the chain and activeSlotsCoeff is below 1, or for local // multi-node devnets. true / false force it. The Haskell relay rejects a dense backfill at f < 1 with @@ -242,6 +250,32 @@ private void alignCostModelFromArray(ObjectMapper mapper, ObjectNode costModels, costModels.set(language, aligned); } + /** + * Slots between blocks in Yano's past-time-travel backfill, from + * {@code yano.backfill.block.interval.slots} : {@code auto} (the default) hands the decision to Yano + * as 0, {@code dense} is one block per slot, and an integer forces the interval. An unusable value + * falls back to automatic rather than to the dense default, which is the slow one. + */ + int backfillBlockIntervalSlots() { + String setting = backfillBlockIntervalSlots == null ? "auto" : backfillBlockIntervalSlots.trim().toLowerCase(); + if (setting.isEmpty() || setting.equals("auto")) + return 0; + if (setting.equals("dense")) + return 1; + try { + int interval = Integer.parseInt(setting); + if (interval < 0) { + log.warn("yano.backfill.block.interval.slots must be non-negative, got {}. Using automatic.", + backfillBlockIntervalSlots); + return 0; + } + return interval; + } catch (NumberFormatException e) { + log.warn("Unknown yano.backfill.block.interval.slots '{}', using automatic", backfillBlockIntervalSlots); + return 0; + } + } + /** * Decide whether Yano's past-time-travel backfill should run Praos slot-leader checks. *

@@ -281,10 +315,16 @@ private Process startYanoProcess(ClusterInfo clusterInfo, Path clusterFolder, Pa // Store Yano data inside node folder so it gets cleaned up with create-node -o Path yanoDataDir = clusterFolder.resolve("node").resolve("yano"); Files.createDirectories(yanoDataDir); + Path yanoHistoryDir = clusterFolder.resolve("node").resolve("yano-history"); // Write application.properties for Yano (persists config on disk for debugging) + int backfillInterval = backfillBlockIntervalSlots(); boolean slotLeaderTimeTravel = pastTimeTravelMode && slotLeaderTimeTravelEnabled(clusterInfo); - yanoConfigBuilder.build(clusterInfo, yanoConfigDir, yanoDataDir, pastTimeTravelMode, slotLeaderTimeTravel); + yanoConfigBuilder.build(clusterInfo, yanoConfigDir, yanoDataDir, yanoHistoryDir, pastTimeTravelMode, + slotLeaderTimeTravel, backfillInterval); + if (pastTimeTravelMode) + writer.accept(info("Yano backfill block interval : %s", + backfillInterval == 0 ? "automatic (derived from genesis)" : backfillInterval + " slots")); ProcessBuilder builder = new ProcessBuilder(); builder.directory(new File(clusterConfig.getYanoHome())); @@ -370,6 +410,7 @@ public void handleClusterStopped(ClusterStopped clusterStopped) { public void handleClusterDeleted(ClusterDeleted clusterDeleted) { Path clusterFolder = Path.of(clusterConfig.getClusterHome(), clusterDeleted.getClusterName()); deleteIfPresent(clusterFolder.resolve("node").resolve("yano"), "Yano data"); + deleteIfPresent(clusterFolder.resolve("node").resolve("yano-history"), "Yano history"); deleteIfPresent(clusterFolder.resolve("yano-config"), "Yano config"); deleteBootstrapMarker(clusterFolder); } diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessStream.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessStream.java index cf95095b..c17991c3 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessStream.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessStream.java @@ -5,6 +5,9 @@ import java.io.InputStreamReader; import java.util.function.Consumer; +import lombok.extern.slf4j.Slf4j; + +@Slf4j public class ProcessStream implements Runnable { private InputStream inputStream; private Consumer consumer; @@ -22,7 +25,13 @@ public void run() { if(stop) return; - consumer.accept(line); + //A failing consumer must not end the read loop : nothing else drains the pipe, so the + //process would then block or drop its output once the pipe buffer is full + try { + consumer.accept(line); + } catch (RuntimeException e) { + log.warn("Could not process output line: {}", e.getMessage()); + } }); } diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessUtil.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessUtil.java index ba5eaf89..b77a69cf 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessUtil.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessUtil.java @@ -68,7 +68,8 @@ public Process startLongRunningProcess(String processName, ProcessBuilder builde Process process = builder.start(); ProcessStream processStream = new ProcessStream(process.getInputStream(), line -> { - logs.add(String.format("[%s] " + line, processName)); + //The line is data, not a format string : node output such as "Progress: 95.65%" would throw + logs.add("[" + processName + "] " + line); }); ProcessStream errorStream = diff --git a/applications/cli/src/main/resources/localcluster/templates/devnet/genesis-templates/spec/config.json b/applications/cli/src/main/resources/localcluster/templates/devnet/genesis-templates/spec/config.json index 36b4ccde..70ec386b 100644 --- a/applications/cli/src/main/resources/localcluster/templates/devnet/genesis-templates/spec/config.json +++ b/applications/cli/src/main/resources/localcluster/templates/devnet/genesis-templates/spec/config.json @@ -33,7 +33,7 @@ "TraceBlockFetchServer": false, "TraceChainDb": true, "TraceChainSyncBlockServer": false, - "TraceChainSyncClient": true, + "TraceChainSyncClient": false, "TraceChainSyncHeaderServer": false, "TraceChainSyncProtocol": false, "TraceConnectionManager": true, diff --git a/applications/cli/src/main/resources/localcluster/templates/devnet/node-2/node.sh b/applications/cli/src/main/resources/localcluster/templates/devnet/node-2/node.sh index 925be3a0..988b3ca7 100755 --- a/applications/cli/src/main/resources/localcluster/templates/devnet/node-2/node.sh +++ b/applications/cli/src/main/resources/localcluster/templates/devnet/node-2/node.sh @@ -12,5 +12,4 @@ ${BIN_FOLDER}/cardano-node run \ --byron-delegation-certificate ./pool-keys/byron-delegation.cert \ --byron-signing-key ./pool-keys/byron-delegate.key \ --shelley-operational-certificate ./pool-keys/opcert.cert \ ---port ${port} \ -| tee -a 'node.log' +--port ${port} diff --git a/applications/cli/src/main/resources/localcluster/templates/devnet/node-3/node.sh b/applications/cli/src/main/resources/localcluster/templates/devnet/node-3/node.sh index 925be3a0..988b3ca7 100755 --- a/applications/cli/src/main/resources/localcluster/templates/devnet/node-3/node.sh +++ b/applications/cli/src/main/resources/localcluster/templates/devnet/node-3/node.sh @@ -12,5 +12,4 @@ ${BIN_FOLDER}/cardano-node run \ --byron-delegation-certificate ./pool-keys/byron-delegation.cert \ --byron-signing-key ./pool-keys/byron-delegate.key \ --shelley-operational-certificate ./pool-keys/opcert.cert \ ---port ${port} \ -| tee -a 'node.log' +--port ${port} diff --git a/applications/cli/src/main/resources/localcluster/templates/devnet/node/node.sh b/applications/cli/src/main/resources/localcluster/templates/devnet/node/node.sh index 925be3a0..988b3ca7 100755 --- a/applications/cli/src/main/resources/localcluster/templates/devnet/node/node.sh +++ b/applications/cli/src/main/resources/localcluster/templates/devnet/node/node.sh @@ -12,5 +12,4 @@ ${BIN_FOLDER}/cardano-node run \ --byron-delegation-certificate ./pool-keys/byron-delegation.cert \ --byron-signing-key ./pool-keys/byron-delegate.key \ --shelley-operational-certificate ./pool-keys/opcert.cert \ ---port ${port} \ -| tee -a 'node.log' +--port ${port} diff --git a/applications/cli/src/main/resources/localcluster/templates/devnet/templates/configuration.json b/applications/cli/src/main/resources/localcluster/templates/devnet/templates/configuration.json index 523c8322..cfbd5483 100644 --- a/applications/cli/src/main/resources/localcluster/templates/devnet/templates/configuration.json +++ b/applications/cli/src/main/resources/localcluster/templates/devnet/templates/configuration.json @@ -82,12 +82,16 @@ [ "StdoutSK", "stdout" + ], + [ + "FileSK", + "node.log" ] ], "hasEKG": 12788, "hasPrometheus": [ "127.0.0.1", - 12798 + {{prometheusPort}} ], "minSeverity": "Info", "options": { @@ -119,6 +123,12 @@ "scKind": "StdoutSK", "scName": "stdout", "scRotation": null + }, + { + "scFormat": "ScText", + "scKind": "FileSK", + "scName": "node.log", + "scRotation": null } ] } diff --git a/applications/cli/src/main/resources/localcluster/templates/devnet/templates/configuration.yaml b/applications/cli/src/main/resources/localcluster/templates/devnet/templates/configuration.yaml index a46afab8..2b03942c 100644 --- a/applications/cli/src/main/resources/localcluster/templates/devnet/templates/configuration.yaml +++ b/applications/cli/src/main/resources/localcluster/templates/devnet/templates/configuration.yaml @@ -189,7 +189,7 @@ TraceBlockchainTime: False TraceChainDb: True # Trace ChainSync client. -TraceChainSyncClient: True +TraceChainSyncClient: False # Trace ChainSync server (blocks). TraceChainSyncBlockServer: False diff --git a/applications/cli/src/main/resources/localcluster/templates/devnet/templates/node-bp.sh b/applications/cli/src/main/resources/localcluster/templates/devnet/templates/node-bp.sh index 83add892..bb7e38f7 100755 --- a/applications/cli/src/main/resources/localcluster/templates/devnet/templates/node-bp.sh +++ b/applications/cli/src/main/resources/localcluster/templates/devnet/templates/node-bp.sh @@ -6,5 +6,4 @@ --shelley-kes-key ../../../pool-keys/{{NODE_NAME}}/kes.skey \ --shelley-vrf-key ../../../pool-keys/{{NODE_NAME}}/vrf.skey \ --shelley-operational-certificate ../../../pool-keys/{{NODE_NAME}}/opcert.cert \ ---port {{port}} \ -| tee -a 'node.log' +--port {{port}} diff --git a/applications/cli/src/main/resources/localcluster/templates/devnet/templates/node-relay.sh b/applications/cli/src/main/resources/localcluster/templates/devnet/templates/node-relay.sh index e86aad2f..b4b90beb 100755 --- a/applications/cli/src/main/resources/localcluster/templates/devnet/templates/node-relay.sh +++ b/applications/cli/src/main/resources/localcluster/templates/devnet/templates/node-relay.sh @@ -3,5 +3,4 @@ --topology topology.json \ --database-path db \ --socket-path node.sock \ ---port {{port}} \ -| tee -a 'node.log' +--port {{port}} diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoServiceBackfillIntervalTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoServiceBackfillIntervalTest.java new file mode 100644 index 00000000..28170365 --- /dev/null +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoServiceBackfillIntervalTest.java @@ -0,0 +1,42 @@ +package com.bloxbean.cardano.yacicli.localcluster.yano; + +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +class YanoServiceBackfillIntervalTest { + + private static YanoService service(String setting) { + YanoService service = new YanoService(null, null, null); + ReflectionTestUtils.setField(service, "backfillBlockIntervalSlots", setting); + return service; + } + + @Test + void autoHandsTheDecisionToYanoAsZero() { + assertThat(service("auto").backfillBlockIntervalSlots()).isZero(); + assertThat(service("AUTO").backfillBlockIntervalSlots()).isZero(); + assertThat(service(" auto ").backfillBlockIntervalSlots()).isZero(); + } + + @Test + void denseMeansOneBlockPerSlot() { + assertThat(service("dense").backfillBlockIntervalSlots()).isEqualTo(1); + } + + @Test + void explicitIntegerForcesTheInterval() { + assertThat(service("500").backfillBlockIntervalSlots()).isEqualTo(500); + assertThat(service("1").backfillBlockIntervalSlots()).isEqualTo(1); + } + + @Test + void unusableValuesFallBackToAutomatic() { + // automatic, not dense : falling back to the slow default would hide the misconfiguration + assertThat(service("sparse-ish").backfillBlockIntervalSlots()).isZero(); + assertThat(service("-5").backfillBlockIntervalSlots()).isZero(); + assertThat(service("").backfillBlockIntervalSlots()).isZero(); + assertThat(service(null).backfillBlockIntervalSlots()).isZero(); + } +} diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/util/ProcessStreamTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/util/ProcessStreamTest.java new file mode 100644 index 00000000..9543eac8 --- /dev/null +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/util/ProcessStreamTest.java @@ -0,0 +1,33 @@ +package com.bloxbean.cardano.yacicli.util; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class ProcessStreamTest { + + private static ByteArrayInputStream input(String... lines) { + return new ByteArrayInputStream((String.join("\n", lines) + "\n").getBytes(StandardCharsets.UTF_8)); + } + + @Test + void keepsReadingAfterAConsumerFailure() { + // A consumer that throws used to end the read loop, after which nothing drained the process + // pipe (seen with cardano-node's "Progress: 95.65%" lines when they were used as a format string). + List seen = new ArrayList<>(); + ProcessStream stream = new ProcessStream(input("first", "bad", "last"), line -> { + if (line.equals("bad")) + throw new IllegalStateException("boom"); + seen.add(line); + }); + + stream.run(); + + assertThat(seen).containsExactly("first", "last"); + } +} diff --git a/config/env b/config/env index 5a3b53f6..caf55888 100644 --- a/config/env +++ b/config/env @@ -1,5 +1,7 @@ yaci_store_enabled=true -ogmios_enabled=false +#Ogmios is enabled by default and used as the Yaci Store tx evaluator. +#Set to false to fall back to the embedded Scalus evaluator. +ogmios_enabled=true kupo_enabled=false node=node1 diff --git a/config/node.properties b/config/node.properties index 1e490431..790f3856 100644 --- a/config/node.properties +++ b/config/node.properties @@ -10,7 +10,7 @@ nodeMode=companion ## To create a custom network with different protocolMagic and networkId, you can use Yaci CLI binary directly. ################################################################################################################# -#maxKESEvolutions=60 +#maxKESEvolutions=62 #securityParam=80 ## Fraction of the epoch covered by the stability window (3k/f) when securityParam is derived (securityParam=0). @@ -18,12 +18,17 @@ nodeMode=companion ## gap in block production shorter than 3k/f slots, so this factor times the epoch's wall-clock length is the ## idle tolerance of the devnet. Values up to about 0.7 are sound (4k/f must stay below the epoch length). #stabilityWindowFactor=0.5 -#slotsPerKESPeriod=129600 + +## A KES period long enough that a past-time-travel devnet never leaves period 0. The bootstrap shift +## moves genesis back by whole epochs before catching up to wall clock, and the node derives the current +## KES period as slot / slotsPerKESPeriod, so at the mainnet 129600 a long-epoch devnet burns a large +## share of its evolutions before producing a block. Devnets do not exercise KES rotation. +#slotsPerKESPeriod=10000000 #updateQuorum=1 #peerSharing=true -## Node log tracing of the chain-sync client. On by default so a bootstrap chain the node rejects -## (for example VRFLeaderValueTooBig at activeSlotsCoeff below 1) shows up in the node log. +## Node log tracing of the chain-sync client. Off by default. Turn it on to see why the node rejects +## a bootstrap chain (for example VRFLeaderValueTooBig at activeSlotsCoeff below 1). #traceChainSyncClient=true ## Shelley Genesis diff --git a/docs/pages/getting-started/docker.mdx b/docs/pages/getting-started/docker.mdx index a652c4e0..98b68fa0 100644 --- a/docs/pages/getting-started/docker.mdx +++ b/docs/pages/getting-started/docker.mdx @@ -25,11 +25,11 @@ The Docker distribution includes: For Ogmios and Kupo, ensure ports 1337 and 1442 are also available if you plan to use these services. -Starting with Yaci DevKit `v0.12.0-beta5`, the default Docker setup uses Yaci Store's `scalus` transaction evaluator. Ogmios is optional and is not started just for transaction evaluation. +The default Docker setup starts Ogmios and uses it as Yaci Store's transaction evaluator. If Ogmios is unavailable, Yaci Store falls back to the embedded `scalus` evaluator. Kupo remains optional. - -For Yaci DevKit `v0.12.0-beta5`, PV11-compatible Ogmios/Kupo binaries are currently available only as Linux x86_64 builds from IntersectMBO. Docker images can run these on Apple Silicon Docker Desktop through emulation, but native Linux arm64 hosts may need x86_64 emulation or the amd64 image. See [Known Issues](/known-issues#pv11-ogmioskupo-docker-arm64-compatibility). + +Ogmios `v7.0.0` and Kupo `v2.12.0` are bundled as **native** binaries in both the `linux/amd64` and `linux/arm64` images — no x86_64 emulation is required on Apple Silicon or native Linux arm64 hosts. ## Installation @@ -194,13 +194,15 @@ topup_addresses=addr_test1...address1:20000,addr_test1...address2:10000 ### Optional Services -#### Enable Ogmios and Kupo -To activate Ogmios and Kupo support, either: +#### Ogmios and Kupo +Ogmios is **enabled by default** (`ogmios_enabled=true` in `config/env`) and is used as Yaci Store's transaction evaluator. Set `ogmios_enabled=false` to disable it and fall back to the embedded `scalus` evaluator. + +Kupo is optional and disabled by default. To activate it, either: -1. **Via env file**: Set `ogmios_enabled=true` and `kupo_enabled=true` in `config/env` +1. **Via env file**: Set `kupo_enabled=true` in `config/env` 2. **Via CLI**: Use the `enable-kupomios` command in Yaci CLI -In `v0.12.0-beta5+`, if Ogmios is enabled and running when Yaci Store starts, Yaci Store uses `ogmios` transaction evaluation. Otherwise it uses `scalus`. +When Yaci Store starts, DevKit uses the `ogmios` evaluator if Ogmios is enabled and responding on its health endpoint; otherwise it logs a warning and uses `scalus`. #### Service URLs - **CLI/Admin Swagger UI**: http://localhost:10000/swagger-ui.html diff --git a/docs/pages/known-issues.mdx b/docs/pages/known-issues.mdx index 0ae7f6aa..305bf2e7 100644 --- a/docs/pages/known-issues.mdx +++ b/docs/pages/known-issues.mdx @@ -6,41 +6,44 @@ This page documents known issues, workarounds, and frequently asked questions ab ## Known Issues -### PV11 Ogmios/Kupo Docker arm64 Compatibility +### Ogmios / Kupo Platform Availability - -**Issue**: PV11-compatible (Cardano node 11.0.1) Ogmios `v6.14.0.2` and Kupo `v2.11.0.1` are currently published by IntersectMBO only as Linux x86_64 binaries. + +Ogmios `v7.0.0` and Kupo `v2.12.0` are published for Linux (x86_64, arm64) and macOS arm64 only. #### Current Behavior -The DevKit Docker image bundles those x86_64 Ogmios/Kupo binaries in both the `linux/amd64` and `linux/arm64` images as a temporary workaround. - -- **Linux x86_64 Docker hosts**: Supported. -- **Apple Silicon Docker Desktop**: Expected to work through Docker Desktop's x86_64 emulation. -- **Native Linux arm64 Docker hosts**: May fail at runtime unless x86_64 binfmt/qemu emulation is configured on the host. -- **Native CLI on macOS arm64**: The default PV11 Ogmios/Kupo Linux x86_64 binaries cannot run directly. Use Docker, or set custom `ogmios.url` / `kupo.url` values when compatible native binaries are available. +- **Docker (linux/amd64 and linux/arm64)**: Fully supported. Both images bundle **native**, statically linked binaries — no emulation required. +- **Native CLI on Linux (x86_64 and arm64)**: Both supported. +- **Native CLI on macOS arm64**: Both supported. The macOS builds link only against system libraries and run standalone. +- **Native CLI on macOS x86_64 / Windows**: Neither component is published. `download -c ogmios` fails fast with an explanatory message instead of attempting a 404 download. #### Workaround -On native Linux arm64 hosts, run the DevKit Docker image as `linux/amd64` or configure x86_64 emulation. For native CLI installs, use a compatible Ogmios/Kupo build when one becomes available by setting direct URLs in `config/download.properties`. +On an unsupported platform, use the DevKit Docker image, or set a custom `ogmios.url` / `kupo.url` in `config/download.properties`. + +When Ogmios is unavailable, Yaci Store automatically falls back to the embedded `scalus` transaction evaluator, so transaction evaluation keeps working. --- -### Yaci Store Tx Evaluator Mode (v0.12.0-beta5+) +### Yaci Store Tx Evaluator Mode -Starting with Yaci DevKit `v0.12.0-beta5`, Yaci Store uses the `scalus` transaction evaluator by default. +Yaci Store uses the `ogmios` transaction evaluator by default in the Docker distribution, and falls back to the embedded `scalus` evaluator when Ogmios is unavailable. -Default Yaci Store transaction evaluation no longer requires Ogmios. If Ogmios is enabled and running when Yaci Store starts, DevKit automatically switches the evaluator mode to `ogmios`. +DevKit selects the evaluator at devnet start: + +- If Ogmios is enabled **and** responds on its health endpoint, the mode is `ogmios`. +- Otherwise DevKit logs a warning and uses `scalus`. No transaction evaluation is lost either way. -This means the temporary PV11 Ogmios/Kupo binary limitations do not block the default Yaci Store transaction evaluation path. +To force the Scalus evaluator, set `ogmios_enabled=false` in `config/env` (Docker) or `ogmios.enabled=false` in `config/application.properties` (zip distribution). Check the active mode in the Yaci CLI startup output: ```text -Yaci Store tx evaluator mode: scalus +Yaci Store tx evaluator mode: ogmios ``` --- diff --git a/docs/pages/node-configuration.mdx b/docs/pages/node-configuration.mdx index 301723b6..2e4fd1cd 100644 --- a/docs/pages/node-configuration.mdx +++ b/docs/pages/node-configuration.mdx @@ -50,6 +50,24 @@ then you can set `securityParam` to a higher values like `86400` for 1 sec block For a real/multi-node network setup, please refer to this known issue/workaround: https://github.com/bloxbean/yaci-devkit/issues/65#issuecomment-2318155838 +### maxKESEvolutions and slotsPerKESPeriod + +Defaults are `62` and `10000000`. The node derives the current KES period as `slot / slotsPerKESPeriod`, and +an operational certificate expires once the periods elapsed since it was issued exceed `maxKESEvolutions`. + +In companion mode the bootstrap shifts genesis back by whole epochs and then catches up to wall clock, so the +chain starts at a high slot number. At the mainnet `slotsPerKESPeriod` of `129600` that shift alone consumes a +large share of the evolutions before the devnet produces its first real block, and the share grows with epoch +length. The large period keeps a devnet inside KES period 0 for its whole life. + +The trade-off is that a DevKit devnet does not exercise KES rotation. Set both to mainnet-like values +(`62` and `129600`) if that is what you want to test: + +```shell +maxKESEvolutions=62 +slotsPerKESPeriod=129600 +``` + ### stabilityWindowFactor When `securityParam` is left at `0`, DevKit derives it from the epoch length so that the stability window @@ -73,21 +91,13 @@ the rewards calculation needs the stability window inside the epoch. Setting `se this derivation. ### traceChainSyncClient -Whether the node logs its chain-sync client (`TraceChainSyncClient`). Default `true`, so a bootstrap chain the -node rejects during a Yano handover shows up in the node log instead of failing silently. Set to `false` for a -quieter log. +Whether the node logs its chain-sync client (`TraceChainSyncClient`). Default `false`. Set to `true` to see why a +node rejects a bootstrap chain during a Yano handover. It logs every header received from a peer, so expect a +busier log on multi-node devnets. ### peerSharing To enable or disable peer sharing. Default value is `true`. -### slotsPerKESPeriod - -To set the number of slots per KES period. Default value is `129600`. - -### maxKESEvolutions - -To set the maximum KES evolutions. Default value is `60`. - **Note:** More configurations can be found in `node.properties` file. diff --git a/docs/pages/services.mdx b/docs/pages/services.mdx index b9326539..de913a6c 100644 --- a/docs/pages/services.mdx +++ b/docs/pages/services.mdx @@ -73,7 +73,9 @@ For Docker, services are configured in `config/env` with underscore notation. ```shell yaci_store_enabled=true -ogmios_enabled=false +# Ogmios is enabled by default and used as the Yaci Store tx evaluator. +# Set to false to fall back to the embedded Scalus evaluator. +ogmios_enabled=true kupo_enabled=false ``` diff --git a/docs/pages/yano-node-modes.mdx b/docs/pages/yano-node-modes.mdx index 84863488..ffacd5d9 100644 --- a/docs/pages/yano-node-modes.mdx +++ b/docs/pages/yano-node-modes.mdx @@ -38,6 +38,28 @@ This avoids requiring users to wait for two epoch transitions before updated pro Use companion mode when you want normal Haskell-node behavior, including node socket access, but want DevKit to prepare the chain quickly for PV11 protocol parameters. +### Backfill block interval + +Catching up from the shifted genesis to wall clock used to place one empty block in every slot, so devnet +create time grew with epoch length and the Haskell relay had to copy the whole chain. + +DevKit now asks Yano to derive the spacing itself, through +`yano.block-producer.backfill-block-interval-slots=0`. Yano places only the blocks a validating node +actually needs: gaps stay inside the forecast window (`3k/f` slots) so the relay can validate every header, +the epoch-boundary blocks needed for nonces are kept, and a final block lands at the catch-up target so the +producer resumes at wall clock. Everything in between was empty filler. + +Override it in `config/application.properties`: + +```properties +# auto (default) hands the decision to Yano; dense is one block per slot; an integer forces the interval +yano.backfill.block.interval.slots=auto +``` + +Automatic spacing needs a Yano build that understands the property (`0.1.0-pre15` or later). Yano validates +the resulting interval against its own forecast window and fails with an explicit message rather than +producing a chain the relay would reject. + ### Block time longer than slot length With `--block-time` larger than `--slot-length`, `activeSlotsCoeff` is below 1 and only VRF-eligible slots may carry @@ -53,8 +75,8 @@ choice can be forced in `config/application.properties`: yano.past.time.travel.slot.leader.mode=auto ``` -The node configuration sets `TraceChainSyncClient` to `true` by default, so a rejected bootstrap chain shows up in -the node log instead of failing silently. Set `traceChainSyncClient=false` in `node.properties` to turn it off. +If the relay still does not sync, set `traceChainSyncClient=true` in `node.properties` to see the rejection in the +node log. ### Relay sync wait during handover diff --git a/npm/yaci-devkit/README.md b/npm/yaci-devkit/README.md index e60b523d..0e5d8936 100644 --- a/npm/yaci-devkit/README.md +++ b/npm/yaci-devkit/README.md @@ -35,4 +35,6 @@ npm i -g @bloxbean/yaci-devkit ``` - Downloads and starts the Cardano node, Ogmios, and Kupo. +> **Note on platforms**: Ogmios and Kupo are published for Linux (x86_64, arm64) and macOS arm64. On macOS x86_64 and Windows they are unavailable; there Yaci Store falls back to the embedded `scalus` script cost evaluator, or you can use the Docker distribution. + For a full list of Yaci CLI commands, check the Yaci DevKit [documentation](https://devkit.yaci.xyz/commands). diff --git a/scripts/info.sh b/scripts/info.sh index 54a46764..74e2ceb5 100755 --- a/scripts/info.sh +++ b/scripts/info.sh @@ -23,7 +23,7 @@ echo "--------------------------------------------" echo "Other Urls" echo "--------------------------------------------" -echo "Ogmios Url (Optional) : ws://localhost:${HOST_OGMIOS_PORT}" +echo "Ogmios Url : ws://localhost:${HOST_OGMIOS_PORT}" echo "Kupo Url (Optional) : http://localhost:${HOST_KUPO_PORT}" echo ""