diff --git a/.github/actions/swift-packages/action.yml b/.github/actions/swift-packages/action.yml new file mode 100644 index 000000000..a4c91b195 --- /dev/null +++ b/.github/actions/swift-packages/action.yml @@ -0,0 +1,53 @@ +name: Swift packages +description: > + Resolves the Swift packages into $SOURCE_PACKAGES and caches them. Pass + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES" -disableAutomaticPackageResolution + -skipPackageUpdates to every later xcodebuild call, or it resolves and clones everything again. + +# SwiftPM keeps a full bare clone of every package and looks each one up on every resolve, so +# the clones have to be cached too. bitkit-core's history alone is over 13 GB, past the +# repository cache limit, so on a miss the clones are cut down to the pinned commits first. +runs: + using: composite + steps: + - name: Locate Swift packages + id: locate + shell: bash + run: | + echo "SOURCE_PACKAGES=$GITHUB_WORKSPACE/SourcePackages" >> "$GITHUB_ENV" + echo "xcode=$(xcodebuild -version | head -1 | cut -d' ' -f2)" >> "$GITHUB_OUTPUT" + + - name: Restore Swift packages + id: cache + uses: actions/cache/restore@v6 + with: + path: ${{ github.workspace }}/SourcePackages + key: ${{ runner.os }}-xcode-${{ steps.locate.outputs.xcode }}-source-packages-v2-${{ hashFiles('Bitkit.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} + + - name: Resolve Swift packages + shell: bash + run: | + echo "⏱️ Starting dependency resolution at $(date)" + set -o pipefail && xcodebuild -resolvePackageDependencies \ + -scheme Bitkit \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES" \ + -disablePackageRepositoryCache \ + -onlyUsePackageVersionsFromResolvedFile \ + -skipPackageUpdates \ + | xcbeautify + echo "✅ Dependencies resolved at $(date)" + + - name: Shrink package clones + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + run: | + "$GITHUB_WORKSPACE/scripts/shallow-source-packages.sh" "$SOURCE_PACKAGES" + du -sh "$SOURCE_PACKAGES" + + # Saved right after resolving so a failing job still warms the cache. + - name: Save Swift packages + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ${{ github.workspace }}/SourcePackages + key: ${{ steps.cache.outputs.cache-primary-key }} diff --git a/.github/workflows/e2e-staging.yml b/.github/workflows/e2e-staging.yml index dddbbfa21..af24b7b29 100644 --- a/.github/workflows/e2e-staging.yml +++ b/.github/workflows/e2e-staging.yml @@ -52,20 +52,8 @@ jobs: run: | brew install xcbeautify - - name: Cache Swift Package Manager - uses: actions/cache@v6 - with: - path: | - ~/Library/Caches/org.swift.swiftpm - ~/Library/org.swift.swiftpm - Bitkit.xcodeproj/project.xcworkspace/xcshareddata/swiftpm - key: ${{ runner.os }}-spm-${{ hashFiles('**/Package.resolved') }} - - - name: Install dependencies - run: | - echo "⏱️ Starting dependency resolution at $(date)" - xcodebuild -resolvePackageDependencies -onlyUsePackageVersionsFromResolvedFile | xcbeautify - echo "✅ Dependencies resolved at $(date)" + - name: Swift packages + uses: ./.github/actions/swift-packages - name: Pre-start simulator run: | @@ -96,6 +84,10 @@ jobs: -configuration Debug \ -destination "platform=iOS Simulator,name=${{ env.SIMULATOR_NAME }},OS=${{ env.IOS_VERSION }}" \ -derivedDataPath DerivedData \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES" \ + -disablePackageRepositoryCache \ + -disableAutomaticPackageResolution \ + -skipPackageUpdates \ SWIFT_ACTIVE_COMPILATION_CONDITIONS='DEBUG E2E_BUILD' \ -allowProvisioningUpdates \ build diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 3c59e1d9d..b0f739e28 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -77,20 +77,8 @@ jobs: run: | brew install xcbeautify - - name: Cache Swift Package Manager - uses: actions/cache@v6 - with: - path: | - ~/Library/Caches/org.swift.swiftpm - ~/Library/org.swift.swiftpm - Bitkit.xcodeproj/project.xcworkspace/xcshareddata/swiftpm - key: ${{ runner.os }}-spm-${{ hashFiles('**/Package.resolved') }} - - - name: Install dependencies - run: | - echo "⏱️ Starting dependency resolution at $(date)" - xcodebuild -resolvePackageDependencies -onlyUsePackageVersionsFromResolvedFile | xcbeautify - echo "✅ Dependencies resolved at $(date)" + - name: Swift packages + uses: ./.github/actions/swift-packages - name: Pre-start simulator run: | @@ -122,6 +110,10 @@ jobs: -configuration Debug \ -destination "platform=iOS Simulator,name=$SIMULATOR_NAME,OS=$OS_VERSION" \ -derivedDataPath DerivedData \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES" \ + -disablePackageRepositoryCache \ + -disableAutomaticPackageResolution \ + -skipPackageUpdates \ SWIFT_ACTIVE_COMPILATION_CONDITIONS='DEBUG E2E_BUILD' \ TREZOR_BRIDGE=true \ TREZOR_BRIDGE_URL=http://127.0.0.1:21325 \ diff --git a/.github/workflows/e2e_migration.yml b/.github/workflows/e2e_migration.yml index 94d9e9e40..b115803e5 100644 --- a/.github/workflows/e2e_migration.yml +++ b/.github/workflows/e2e_migration.yml @@ -63,20 +63,8 @@ jobs: run: | brew install xcbeautify - - name: Cache Swift Package Manager - uses: actions/cache@v6 - with: - path: | - ~/Library/Caches/org.swift.swiftpm - ~/Library/org.swift.swiftpm - Bitkit.xcodeproj/project.xcworkspace/xcshareddata/swiftpm - key: ${{ runner.os }}-spm-${{ hashFiles('**/Package.resolved') }} - - - name: Install dependencies - run: | - echo "⏱️ Starting dependency resolution at $(date)" - xcodebuild -resolvePackageDependencies -onlyUsePackageVersionsFromResolvedFile | xcbeautify - echo "✅ Dependencies resolved at $(date)" + - name: Swift packages + uses: ./.github/actions/swift-packages - name: Pre-start simulator run: | @@ -108,6 +96,10 @@ jobs: -configuration Debug \ -destination "platform=iOS Simulator,name=${{ env.SIMULATOR_NAME }},OS=${{ env.IOS_VERSION }}" \ -derivedDataPath DerivedData \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES" \ + -disablePackageRepositoryCache \ + -disableAutomaticPackageResolution \ + -skipPackageUpdates \ SWIFT_ACTIVE_COMPILATION_CONDITIONS='$(inherited) E2E_BUILD' \ -allowProvisioningUpdates \ build diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 6867a0637..927863b3a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -22,6 +22,9 @@ jobs: name: Run Integration Tests runs-on: macos-15 timeout-minutes: 60 + env: + DERIVED_DATA: ${{ github.workspace }}/DerivedData + RESULT_BUNDLE: ${{ github.workspace }}/IntegrationTests.xcresult steps: - name: Checkout repository @@ -32,81 +35,85 @@ jobs: with: xcode-version: '16.4' + # A fresh simulator takes over 10 minutes to boot on the runner. Booting it now lets that + # overlap with resolving and building instead of stalling the test step. + - name: Boot simulator + run: | + udid=$(xcrun simctl list devices available -j | python3 -c ' + import json, sys + runtimes = json.load(sys.stdin)["devices"] + version = lambda runtime: [int(part) for part in runtime.rsplit(".", 1)[-1].split("-")[1:]] + ios = sorted((r for r in runtimes if ".iOS-" in r and any(d["name"] == "iPhone 16" for d in runtimes[r])), key=version) + print(next(d["udid"] for d in runtimes[ios[-1]] if d["name"] == "iPhone 16")) + ') + echo "SIMULATOR_UDID=$udid" >> "$GITHUB_ENV" + echo "DESTINATION=platform=iOS Simulator,id=$udid" >> "$GITHUB_ENV" + xcrun simctl boot "$udid" + - name: Install xcbeautify run: | brew install xcbeautify - - name: Cache Swift Package Manager - uses: actions/cache@v6 - with: - path: | - ~/Library/Caches/org.swift.swiftpm - ~/Library/org.swift.swiftpm - Bitkit.xcodeproj/project.xcworkspace/xcshareddata/swiftpm - key: ${{ runner.os }}-spm-${{ hashFiles('**/Package.resolved') }} - - - name: Install dependencies - run: | - echo "⏱️ Starting dependency resolution at $(date)" - xcodebuild -resolvePackageDependencies -onlyUsePackageVersionsFromResolvedFile | xcbeautify - echo "✅ Dependencies resolved at $(date)" + - name: Swift packages + uses: ./.github/actions/swift-packages - - name: Clean build + - name: Build for testing run: | - echo "⏱️ Cleaning build at $(date)" - xcodebuild clean -scheme Bitkit | xcbeautify - echo "✅ Build cleaned at $(date)" + echo "⏱️ Starting build at $(date)" + set -o pipefail && xcodebuild build-for-testing \ + -scheme Bitkit \ + -destination "$DESTINATION" \ + -derivedDataPath "$DERIVED_DATA" \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES" \ + -disablePackageRepositoryCache \ + -disableAutomaticPackageResolution \ + -skipPackageUpdates \ + -enableCodeCoverage NO \ + | xcbeautify + echo "✅ Build finished at $(date)" - name: Run integration tests + timeout-minutes: 25 run: | + xcrun simctl bootstatus "$SIMULATOR_UDID" -b echo "⏱️ Starting integration tests at $(date)" - run_tests() { - # RNMigrationCleanupTests deletes the real ~/Documents/mmkv and ~/Documents/ldk, so it - # skips itself unless this is set. This lane runs on a simulator it erases between - # attempts; the unit lane does not, and must not run it. - # - # The TEST_RUNNER_ prefix is required: xcodebuild does not pass the invoking shell's - # environment to the simulator-hosted runner, only variables named TEST_RUNNER_, - # which it forwards with the prefix stripped. Without it the guard never sees the - # variable, every test in the suite skips, and the lane still reports green. - set -o pipefail && TEST_RUNNER_BITKIT_DESTRUCTIVE_TESTS=1 xcodebuild test \ - -scheme Bitkit \ - -destination 'platform=iOS Simulator,name=iPhone 16,OS=latest' \ - -enableCodeCoverage NO \ - -parallel-testing-enabled NO \ - -only-testing:BitkitTests/UtxoSelectionTests \ - -only-testing:BitkitTests/BlocktankTests \ - -only-testing:BitkitTests/PaymentFlowTests \ - -only-testing:BitkitTests/BlocktankRefundAddressLiveIntegrationTests \ - -only-testing:BitkitTests/AddressTypeIntegrationTests \ - -only-testing:BitkitTests/RNMigrationCleanupTests \ - | xcbeautify --report junit - } - - # Try up to 3 times with simulator reset between failures - for attempt in 1 2 3; do - echo "🔄 Test attempt $attempt of 3" - if run_tests; then - echo "✅ Integration tests completed at $(date)" - exit 0 - fi - - if [ $attempt -lt 3 ]; then - echo "⚠️ Test attempt $attempt failed, resetting simulator..." - xcrun simctl shutdown all || true - xcrun simctl erase "iPhone 16" || true - xcrun simctl boot "iPhone 16" - sleep 5 - fi - done - - echo "❌ All test attempts failed" - exit 1 + # RNMigrationCleanupTests deletes the real ~/Documents/mmkv and ~/Documents/ldk, so it + # skips itself unless this is set. This lane runs on a throwaway CI simulator; the unit + # lane does not, and must not run it. + # + # The TEST_RUNNER_ prefix is required: xcodebuild does not pass the invoking shell's + # environment to the simulator-hosted runner, only variables named TEST_RUNNER_, + # which it forwards with the prefix stripped. Without it the guard never sees the + # variable, every test in the suite skips, and the lane still reports green. + # + # The live regtest suites are flaky on CI runners, so a failed test is retried once on + # its own instead of rebuilding and rerunning the whole suite. + set -o pipefail && TEST_RUNNER_BITKIT_DESTRUCTIVE_TESTS=1 xcodebuild test-without-building \ + -scheme Bitkit \ + -destination "$DESTINATION" \ + -derivedDataPath "$DERIVED_DATA" \ + -resultBundlePath "$RESULT_BUNDLE" \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES" \ + -disablePackageRepositoryCache \ + -disableAutomaticPackageResolution \ + -skipPackageUpdates \ + -enableCodeCoverage NO \ + -parallel-testing-enabled NO \ + -retry-tests-on-failure \ + -test-iterations 2 \ + -only-testing:BitkitTests/UtxoSelectionTests \ + -only-testing:BitkitTests/BlocktankTests \ + -only-testing:BitkitTests/PaymentFlowTests \ + -only-testing:BitkitTests/BlocktankRefundAddressLiveIntegrationTests \ + -only-testing:BitkitTests/AddressTypeIntegrationTests \ + -only-testing:BitkitTests/RNMigrationCleanupTests \ + | xcbeautify --report junit + echo "✅ Integration tests completed at $(date)" - name: Upload test results uses: actions/upload-artifact@v7 if: success() || failure() with: name: integration-test-results - path: ~/Library/Developer/Xcode/DerivedData/**/Logs/Test/*.xcresult + path: ${{ env.RESULT_BUNDLE }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e3b85ff8a..0a83d0d07 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -35,20 +35,8 @@ jobs: run: | brew install xcbeautify - - name: Cache Swift Package Manager - uses: actions/cache@v6 - with: - path: | - ~/Library/Caches/org.swift.swiftpm - ~/Library/org.swift.swiftpm - Bitkit.xcodeproj/project.xcworkspace/xcshareddata/swiftpm - key: ${{ runner.os }}-spm-${{ hashFiles('**/Package.resolved') }} - - - name: Install dependencies - run: | - echo "⏱️ Starting dependency resolution at $(date)" - xcodebuild -resolvePackageDependencies -onlyUsePackageVersionsFromResolvedFile | xcbeautify - echo "✅ Dependencies resolved at $(date)" + - name: Swift packages + uses: ./.github/actions/swift-packages - name: Pre-start simulator run: | @@ -56,18 +44,16 @@ jobs: xcrun simctl boot "iPhone 16" || true echo "✅ Simulator started at $(date)" - - name: Clean build - run: | - echo "⏱️ Cleaning build at $(date)" - xcodebuild clean -scheme Bitkit | xcbeautify - echo "✅ Build cleaned at $(date)" - - name: Run unit tests run: | echo "⏱️ Starting unit tests at $(date)" set -o pipefail && xcodebuild test \ -scheme Bitkit \ -destination 'platform=iOS Simulator,name=iPhone 16,OS=latest' \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES" \ + -disablePackageRepositoryCache \ + -disableAutomaticPackageResolution \ + -skipPackageUpdates \ -enableCodeCoverage NO \ -parallel-testing-enabled NO \ -skip-testing:BitkitTests/UtxoSelectionTests \ diff --git a/BitkitTests/UtxoSelectionTests.swift b/BitkitTests/UtxoSelectionTests.swift index f05058b6f..4e5e0e7d3 100644 --- a/BitkitTests/UtxoSelectionTests.swift +++ b/BitkitTests/UtxoSelectionTests.swift @@ -8,6 +8,42 @@ final class UtxoSelectionTests: XCTestCase { let blocktank = CoreService.shared.blocktank let lightning = LightningService.shared + private static let pollIntervalNanoseconds: UInt64 = 2_000_000_000 + private static let pollTimeout: TimeInterval = 120 + + /// Thrown rather than recorded, so a test whose funding never confirmed stops before it spends. + private struct PollTimeout: Error, CustomStringConvertible { + let description: String + } + + /// Syncs and lists spendable outputs until `isReady` holds. Regtest confirmations reach the + /// node at varying speed on CI, and a sync can fail with `NodeError.WalletOperationTimeout`, + /// so a fixed sleep then a single sync sees partial state. + private func waitForSpendableOutputs( + _ description: String, + until isReady: ([SpendableUtxo]) -> Bool + ) async throws -> [SpendableUtxo] { + let deadline = Date().addingTimeInterval(Self.pollTimeout) + var outputs: [SpendableUtxo] = [] + + repeat { + do { + try await lightning.sync() + outputs = try await lightning.listSpendableOutputs() + if isReady(outputs) { + return outputs + } + Logger.test("Waiting for \(description), have \(outputs.count) spendable outputs", context: "UtxoSelectionTests") + } catch { + Logger.test("Spendable output poll failed, retrying: \(error)", context: "UtxoSelectionTests") + } + + try await Task.sleep(nanoseconds: Self.pollIntervalNanoseconds) + } while Date() < deadline + + throw PollTimeout(description: "Timed out after \(Int(Self.pollTimeout))s waiting for \(description), last saw \(outputs.count) outputs") + } + override func setUp() async throws { try await super.setUp() // `StartupHandler.createNewWallet` resets `selectedAddressType` and `addressTypesToMonitor` for a @@ -95,15 +131,9 @@ final class UtxoSelectionTests: XCTestCase { try await blocktank.regtestMineBlocks(6) Logger.test("Blocks mined successfully", context: "UtxoSelectionTests") - // Sleep 15 seconds to ensure all blocks are processed - Logger.test("Waiting 15 seconds for blocks to be processed", context: "UtxoSelectionTests") - try await Task.sleep(nanoseconds: 15_000_000_000) - Logger.test("Wait completed", context: "UtxoSelectionTests") - - // Sync the wallet to see the new balance - Logger.test("Syncing wallet to update balances", context: "UtxoSelectionTests") - try await lightning.sync() - Logger.test("Wallet sync complete", context: "UtxoSelectionTests") + let outputs = try await waitForSpendableOutputs("\(depositAmounts.count) confirmed deposits") { + $0.count >= depositAmounts.count + } // Verify updated balances after funding let updatedBalances = await MainActor.run { lightning.balances } @@ -117,9 +147,7 @@ final class UtxoSelectionTests: XCTestCase { XCTAssertEqual(finalTotal, totalExpectedAmount, "Final balance should equal the sum of all deposits") XCTAssertGreaterThan(finalTotal, 0, "Final balance should be greater than 0") - // List utxos and make sure we have the right amount - Logger.test("Listing UTXOs to verify amounts", context: "UtxoSelectionTests") - let outputs = try await lightning.listSpendableOutputs() + // Make sure we have the right UTXOs Logger.test("Found \(outputs.count) spendable outputs", context: "UtxoSelectionTests") XCTAssertEqual(outputs.count, depositAmounts.count, "Number of UTXOs should match number of deposits") @@ -184,19 +212,10 @@ final class UtxoSelectionTests: XCTestCase { try await blocktank.regtestMineBlocks(1) Logger.test("Block mined successfully", context: "UtxoSelectionTests") - // Wait for the block to be processed - Logger.test("Waiting 10 seconds for block to be processed", context: "UtxoSelectionTests") - try await Task.sleep(nanoseconds: 10_000_000_000) - Logger.test("Wait completed", context: "UtxoSelectionTests") - - // Sync the wallet to update UTXOs - Logger.test("Syncing wallet to update UTXO set", context: "UtxoSelectionTests") - try await lightning.sync() - Logger.test("Wallet sync complete", context: "UtxoSelectionTests") - - // List UTXOs again and verify the spent ones are missing - Logger.test("Listing UTXOs after spending to verify the spent ones are missing", context: "UtxoSelectionTests") - let remainingOutputs = try await lightning.listSpendableOutputs() + // Verify the spent ones are missing + let remainingOutputs = try await waitForSpendableOutputs("the selected UTXOs to be spent") { outputs in + !outputs.contains { output in selectedUtxoIds.contains("\(output.outpoint.txid):\(output.outpoint.vout)") } + } Logger.test("Found \(remainingOutputs.count) remaining spendable outputs", context: "UtxoSelectionTests") // Verify the specific UTXOs we spent are no longer in the list @@ -271,19 +290,9 @@ final class UtxoSelectionTests: XCTestCase { try await blocktank.regtestMineBlocks(6) Logger.test("Blocks mined successfully", context: "UtxoSelectionTests") - // Wait for blocks to be processed - Logger.test("Waiting 15 seconds for blocks to be processed", context: "UtxoSelectionTests") - try await Task.sleep(nanoseconds: 15_000_000_000) - Logger.test("Wait completed", context: "UtxoSelectionTests") - - // Sync the wallet to see the new balance - Logger.test("Syncing wallet to update balances", context: "UtxoSelectionTests") - try await lightning.sync() - Logger.test("Wallet sync complete", context: "UtxoSelectionTests") - - // Get all available UTXOs - Logger.test("Listing all available UTXOs", context: "UtxoSelectionTests") - let allUtxos = try await lightning.listSpendableOutputs() + let allUtxos = try await waitForSpendableOutputs("\(depositAmounts.count) confirmed deposits") { + $0.count >= depositAmounts.count + } Logger.test("Found \(allUtxos.count) spendable outputs", context: "UtxoSelectionTests") XCTAssertEqual(allUtxos.count, depositAmounts.count, "Number of UTXOs should match number of deposits") diff --git a/scripts/shallow-source-packages.sh b/scripts/shallow-source-packages.sh new file mode 100755 index 000000000..596ca86e8 --- /dev/null +++ b/scripts/shallow-source-packages.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Shrinks SwiftPM's package clones to the pinned commits so CI can cache them. +# +# SwiftPM keeps a full bare clone of every package under /repositories, and +# looks each one up again on every resolve (bitkit-core's history alone is over 13 GB, past +# the repository cache limit). This keeps, for each clone, only the commit its checkout is +# on plus the tags pointing at it, and drops the checkouts' own refs to commits that are gone. +# +# Usage: scripts/shallow-source-packages.sh +set -euo pipefail + +source_packages="${1:?usage: $0 }" + +for checkout in "$source_packages"/checkouts/*/; do + checkout="${checkout%/}" + alternates="$checkout/.git/objects/info/alternates" + [ -f "$alternates" ] || continue + repository="$(dirname "$(cat "$alternates")")" + revision="$(git -C "$checkout" rev-parse HEAD)" + + shallow="$repository.shallow" + rm -rf "$shallow" + git init -q --bare "$shallow" + cp "$repository/config" "$shallow/config" + git -C "$repository" config uploadpack.allowAnySHA1InWant true + + refspecs=("$revision:refs/heads/pinned") + while read -r tag; do + [ -n "$tag" ] && refspecs+=("refs/tags/$tag:refs/tags/$tag") + done < <(git -C "$repository" tag --points-at "$revision") + git -C "$shallow" fetch -q --depth 1 --no-tags "file://$repository" "${refspecs[@]}" + + rm -rf "$repository" + mv "$shallow" "$repository" + + cp "$repository/shallow" "$checkout/.git/shallow" + git -C "$checkout" update-ref --no-deref HEAD "$revision" + git -C "$checkout" for-each-ref --format='%(refname)' | while read -r ref; do + git -C "$checkout" update-ref -d --no-deref "$ref" + done + rm -rf "$checkout/.git/logs" + + echo "$(basename "$checkout"): $(du -sh "$repository" | cut -f1) at ${revision:0:7}" +done