Working on testing aarch64-linux noci

Author: thomasjm

.github/workflows/ci.yml

@@ -203,3 +203,63 @@ jobs:
             "$TEST_ROOT/" \
             "$DEST"
         fi
+
+  # aarch64-linux test using QEMU binfmt emulation on the x86_64-linux runner
+  # Requires: boot.binfmt.emulatedSystems = [ "aarch64-linux" ]; on the NixOS runner
+  test-aarch64-linux:
+    name: "bash (aarch64-linux via emulation)"
+    runs-on:
+    - self-hosted
+    - Linux
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Build tests
+      if: "!contains(github.event.head_commit.message, 'noci')"
+      run: |
+        cd tests
+
+        export NIX_PATH=nixpkgs=$(nix run .#nixpkgsPath)
+
+        export PATH=$(nix build --no-link .#stack --json | jq -r '.[0].outputs.out')/bin:$PATH
+        export PATH=$(nix build --no-link .#direnv --json | jq -r '.[0].outputs.out')/bin:$PATH
+        echo "Got path: $PATH"
+
+        direnv allow
+        direnv exec . stack build --nix --no-nix-pure
+
+    - name: Run tests (aarch64-linux)
+      if: "!contains(github.event.head_commit.message, 'noci')"
+      run: |
+        cd tests
+
+        export TEST_ROOT=$(pwd)/test_root_aarch64
+        echo "Got TEST_ROOT: $TEST_ROOT"
+        echo TEST_ROOT="$TEST_ROOT" >> $GITHUB_ENV
+
+        export NIX_PATH=nixpkgs=$(nix run .#nixpkgsPath)
+        export PATH=$(nix build --no-link .#stack --json | jq -r '.[0].outputs.out')/bin:$PATH
+
+        stack run --nix --no-nix-pure -- \
+          --target-system aarch64-linux \
+          --print \
+          --fixed-root "$TEST_ROOT" \
+          --markdown-summary "$GITHUB_STEP_SUMMARY" \
+          --bash
+
+    - name: Upload test artifacts
+      if: always()
+      continue-on-error: true
+      shell: bash
+      run: |
+        DEST="tester@desktop1:/home/tester/all_test_runs/languages_${{github.workflow}}_${{github.run_number}}/bash-aarch64-linux"
+        echo "DEST: $DEST"
+        if [[ -n "$TEST_ROOT" ]]; then
+          rsync \
+            -e "ssh -F /secrets/ci_ssh_config" \
+            -azW --mkpath \
+            "$TEST_ROOT/" \
+            "$DEST"
+        fi

tests/app/Spec/Tests.hs

@@ -25,6 +25,7 @@ tests :: forall context. (
   , Typeable context
   ) => SpecFree context IO ()
 tests =
+  introduceTargetSystem $
   introduceJupyterRunner $
   introduceJustBubblewrap $
   introduceBootstrapNixpkgs $

tests/app/Spec/Tests/Haskell/Diagnostics.hs

@@ -87,7 +87,8 @@ etaExpandCode = [__i|module Foo where
                     |]
 
 main :: IO ()
-main = runSandwichWithCommandLineArgs Sandwich.defaultOptions $
+main = runSandwichWithCommandLineArgs' Sandwich.defaultOptions specialOptions $
+  introduceTargetSystem $
   introduceNixEnvironment [kernelSpec ghcPackage] [] "Haskell" $
     introduceJustBubblewrap $
       tests ghcPackage HaskellCommon.lsName

tests/app/Spec/Tests/Searchers.hs

@@ -13,6 +13,7 @@ tests = describe "Searchers" $ do
   it "searcher has some results" $ testSearcherHasNonemptyResults "packageSearch"
 
 main :: IO ()
-main = runSandwichWithCommandLineArgs Sandwich.defaultOptions $
+main = runSandwichWithCommandLineArgs' Sandwich.defaultOptions specialOptions $
+  introduceTargetSystem $
   introduceBootstrapNixpkgs $
   tests

tests/app/Spec/Tests/SettingsSchemas.hs

@@ -47,6 +47,7 @@ itemsLackingField byPackage field = catMaybes [
       _ -> Just ([i|#{packageName}.#{targetName}|] :: Text)
 
 main :: IO ()
-main = runSandwichWithCommandLineArgs Sandwich.defaultOptions $
+main = runSandwichWithCommandLineArgs' Sandwich.defaultOptions specialOptions $
+  introduceTargetSystem $
   introduceBootstrapNixpkgs $
   tests

tests/app/Spec/Tests/Shells/Zsh.hs

@@ -5,6 +5,7 @@ module Spec.Tests.Shells.Zsh (tests) where
 import Data.String.Interpolate
 import Data.Text
 import Test.Sandwich as Sandwich
+import TestLib.JupyterRunnerContext (introduceTargetSystem)
 import TestLib.NixEnvironmentContext
 import TestLib.Types
 
@@ -14,7 +15,7 @@ otherConfig = [
   "shells.zsh.enable = true;"
   ]
 
-tests :: TopSpec
+tests :: NixEnvSpec
 tests = describe "ZSH" $ introduceNixEnvironment [] otherConfig "ZSH environment" $ do
   it "can run a command" $ do
     nixEnv <- getContext nixEnvironment
@@ -23,4 +24,5 @@ tests = describe "ZSH" $ introduceNixEnvironment [] otherConfig "ZSH environment
     pending
 
 main :: IO ()
-main = runSandwichWithCommandLineArgs Sandwich.defaultOptions tests
+main = runSandwichWithCommandLineArgs' Sandwich.defaultOptions specialOptions $
+  introduceTargetSystem tests

tests/app/Spec/Tests/Spellchecker.hs

@@ -19,6 +19,7 @@ import Test.Sandwich.Waits (waitUntil)
 import TestLib.JupyterRunnerContext
 import TestLib.LSP
 import TestLib.NixEnvironmentContext
+import TestLib.Types
 import UnliftIO.Directory
 
 
@@ -27,7 +28,7 @@ otherConfig = [
   "language-servers.spellchecker.enable = true;"
   ]
 
-tests :: TopSpec
+tests :: NixEnvSpec
 tests = describe "Spellchecker" $ introduceNixEnvironment [] otherConfig "Spellchecker env" $ introduceJustBubblewrap $ do
   it "Gets diagnostics and a working code action" $ do
     doSession'' "test.md" "spellchecker" [i|\# This is mispelled|] [] $ \(Helpers.LspSessionInfo {..}) -> do
@@ -66,4 +67,5 @@ getTitle (InL x) = x ^. title
 getTitle (InR x) = x ^. title
 
 main :: IO ()
-main = runSandwichWithCommandLineArgs Sandwich.defaultOptions tests
+main = runSandwichWithCommandLineArgs' Sandwich.defaultOptions specialOptions $
+  introduceTargetSystem tests

tests/src/TestLib/JupyterRunnerContext.hs

@@ -31,6 +31,14 @@ import UnliftIO.Directory
 import UnliftIO.Process
 
 
+introduceTargetSystem :: (
+  HasBaseContext context, HasCommandLineOptions context SpecialOptions, MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
+  ) => SpecFree (LabelValue "targetSystem" (Maybe String) :> context) m () -> SpecFree context m ()
+introduceTargetSystem = introduceWith [i|Target system|] targetSystem $ \action -> do
+  opts <- getCommandLineOptions
+  let ts = optTargetSystem (optUserOptions opts)
+  void $ action ts
+
 introduceJupyterRunner :: (
   HasBaseContext context, MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
   ) => SpecFree (LabelValue "jupyterRunner" FilePath :> context) m () -> SpecFree context m ()
@@ -53,6 +61,26 @@ introduceBootstrapNixpkgs = introduceWith [i|Jupyter runner|] bootstrapNixpkgs $
   out <- readCreateProcessWithLogging ((proc "nix" ["run", ".#nixpkgsPath"]) { cwd = Just (rootDir </> "tests") }) ""
   void $ action (T.unpack $ T.strip $ T.pack out)
 
+-- | Use bubblewrap unless we're cross-compiling (target system differs from host)
+introduceBubblewrap :: (
+  HasBaseContext context, HasTargetSystem context, MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
+  ) => SpecFree (LabelValue "maybeBubblewrap" (Maybe FilePath) :> context) m () -> SpecFree context m ()
+introduceBubblewrap = introduceWith [i|bwrap|] maybeBubblewrap $ \action -> do
+  maybeTargetSys <- getContext targetSystem
+  case maybeTargetSys of
+    Just _ -> do
+      -- Skip bubblewrap for cross-architecture tests to avoid closure issues
+      info [i|Skipping bubblewrap due to cross-architecture target system|]
+      void $ action Nothing
+    Nothing -> do
+#ifdef darwin_HOST_OS
+      void $ action Nothing
+#else
+      liftIO (findExecutable "bwrap") >>= \case
+        Nothing -> expectationFailure [i|The tests currently require bubblewrap to be present.|]
+        Just path -> void $ action (Just path)
+#endif
+
 -- | TODO: pipe through a command-line argument to control whether bwrap is used?
 introduceJustBubblewrap :: (
   HasBaseContext context, MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
@@ -352,7 +380,8 @@ notebookWithCode kernel code = A.object [
 
 jupyterMain :: LanguageSpec -> IO ()
 jupyterMain tests = runSandwichWithCommandLineArgs' Sandwich.defaultOptions specialOptions $
+  introduceTargetSystem $
   introduceJupyterRunner $
-  introduceJustBubblewrap $
+  introduceBubblewrap $
   introduceBootstrapNixpkgs
   tests

tests/src/TestLib/NixEnvironmentContext.hs

@@ -28,14 +28,15 @@ import UnliftIO.Temporary
 
 
 introduceNixEnvironment :: (
-  HasBaseContext context, MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
+  HasBaseContext context, HasTargetSystem context, MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
   )
   => [NixKernelSpec]
   -> [Text]
   -> Text
   -> SpecFree (LabelValue "nixEnvironment" FilePath :> context) m ()
   -> SpecFree context m ()
 introduceNixEnvironment kernels otherConfig label = introduceWith' (defaultNodeOptions {nodeOptionsVisibilityThreshold = 60}) [i|#{label}|] nixEnvironment $ \action -> do
+  maybeTargetSys <- getContext targetSystem
   rootDir <- findFirstParentMatching (\x -> doesPathExist (x </> ".git"))
 
   metadata :: A.Object <- bracket (openFile "/dev/null" WriteMode) hClose $ \devNullHandle -> do
@@ -74,6 +75,7 @@ introduceNixEnvironment kernels otherConfig label = introduceWith' (defaultNodeO
     x -> expectationFailure [i|Unhandled src spec type: #{x}|]
 
   built <- withTempDirectory dir "test-nix-build" $ \((</> "link") -> linkPath) -> do
+    let systemArgs = maybe [] (\sys -> ["--system", sys]) maybeTargetSys
     let args = ["build"
                , "--quiet"
                , "--no-link"
@@ -84,6 +86,7 @@ introduceNixEnvironment kernels otherConfig label = introduceWith' (defaultNodeO
                , "--expr", T.unpack rendered
                , "-o", linkPath
                ]
+               ++ systemArgs
     info [i|nix #{L.unwords args}|]
     createProcessWithLogging (proc "nix" args)
       >>= waitForProcess >>= (`shouldBe` ExitSuccess)

tests/src/TestLib/Types.hs

@@ -18,11 +18,13 @@ import TestLib.NixTypes
 
 data SpecialOptions = SpecialOptions {
   optTestParallelism :: Int
+  , optTargetSystem :: Maybe String
   }
 
 specialOptions :: Parser SpecialOptions
 specialOptions = SpecialOptions
   <$> option auto (long "test-parallelism" <> short 'n' <> showDefault <> help "Test parallelism" <> value 4 <> metavar "INT")
+  <*> optional (strOption (long "target-system" <> help "Target system for nix builds (e.g., aarch64-linux)" <> metavar "SYSTEM"))
 
 -- * Nix
 
@@ -53,6 +55,10 @@ lockedToNixSrcSpec name (LockedGithub {..}) = NixSrcFetchFromGithub {
 
 -- * Labels
 
+targetSystem :: Label "targetSystem" (Maybe String)
+targetSystem = Label
+type HasTargetSystem context = HasLabel context "targetSystem" (Maybe String)
+
 nixEnvironment :: Label "nixEnvironment" FilePath
 nixEnvironment = Label
 type HasNixEnvironment context = HasLabel context "nixEnvironment" FilePath
@@ -76,8 +82,12 @@ type SomeLanguageSpec context = (
   , HasJupyterRunner context
   , HasMaybeBubblewrap context
   , HasBootstrapNixpkgs context
+  , HasTargetSystem context
   )
 
 type LanguageSpec = forall context. SomeLanguageSpec context => SpecFree context IO ()
 
-type SimpleSpec = forall context. (HasBaseContext context, HasBootstrapNixpkgs context) => SpecFree context IO ()
+type SimpleSpec = forall context. (HasBaseContext context, HasBootstrapNixpkgs context, HasTargetSystem context) => SpecFree context IO ()
+
+-- | Spec type for tests that only need nix environment building capability
+type NixEnvSpec = forall context. (HasBaseContext context, HasTargetSystem context) => SpecFree context IO ()