Static binaries for Linux

[f-f]

We should publish statically-linked binaries for our Linux releases. The reason for doing this is that various Linux distros ship different versions of some native libraries we link to, and dynamically linking means that the binary doesn't work out of the box on some of them, and people have to install dependencies with their package manager, etc etc.

It would be nice if we could instead build and publish a statically linked binary. This has been tried in the past (e.g. see #437), but failed because we were still linking to glibc, while to properly link statically we should link to musl.
The easiest way to compile Haskell statically with no hassle is probably with static-haskell-nix.

So to put this into use we should:

1. add Nix to the Travis CI setup
2. Nixify the build
3. Use Nix in CI
4. make sure that we cache the Nix stuff so CI times don't blow up

This kind of opens up the question of "why use the whole stack thing if we can just build with Nix", which is relevant because I wouldn't want to maintain two build systems that need attention when doing routine things like updating a dependency. So we could figure out a way to reduce this burden - e.g. haskell.nix might help with this?

cc @cdepillabout

[hdgarrood]

Doesn’t spago depend on terminfo at the moment too? I think you’d need to drop that dependency for this to work, right?

[f-f]

@hdgarrood I don't think so: IIRC we inherit that dependency from Dhall, which seems to be fine building static binaries with the same Nix machinery

[hdgarrood]

Interesting, thanks.

[cdepillabout]

@f-f I made a small proof-of-concept that builds spago statically:

let
  # This is a derivation for dhall-1.27.0.  nixpkgs master currently has
  # dhall-1.28.0, but spago doesn't work with that yet.
  # See below where this is used.
  dhall_1_27_0_from_cabal2nix =
    { mkDerivation, aeson, aeson-pretty, ansi-terminal, base
    , bytestring, case-insensitive, cborg, cborg-json, containers
    , contravariant, cryptonite, data-fix, deepseq, Diff, directory
    , doctest, dotgen, either, exceptions, filepath, foldl, gauge
    , generic-random, hashable, haskeline, http-client, http-client-tls
    , http-types, lens-family-core, megaparsec, memory, mockery, mtl
    , network-uri, optparse-applicative, parsers, prettyprinter
    , prettyprinter-ansi-terminal, profunctors, QuickCheck
    , quickcheck-instances, repline, scientific, semigroups, serialise
    , special-values, spoon, stdenv, tasty, tasty-expected-failure
    , tasty-hunit, tasty-quickcheck, template-haskell, text
    , th-lift-instances, transformers, transformers-compat, turtle
    , unordered-containers, uri-encode, vector
    }:
    mkDerivation {
      pname = "dhall";
      version = "1.27.0";
      sha256 = "e189fecd9ea22153252609a4d7c5cc4d61f2c36326b53758b61e5a851e701712";
      isLibrary = true;
      isExecutable = true;
      libraryHaskellDepends = [
        aeson aeson-pretty ansi-terminal base bytestring case-insensitive
        cborg cborg-json containers contravariant cryptonite data-fix
        deepseq Diff directory dotgen either exceptions filepath hashable
        haskeline http-client http-client-tls http-types lens-family-core
        megaparsec memory mtl network-uri optparse-applicative parsers
        prettyprinter prettyprinter-ansi-terminal profunctors repline
        scientific serialise template-haskell text th-lift-instances
        transformers transformers-compat unordered-containers uri-encode
        vector
      ];
      executableHaskellDepends = [ base ];
      testHaskellDepends = [
        base bytestring cborg containers data-fix deepseq directory doctest
        either filepath foldl generic-random lens-family-core megaparsec
        mockery prettyprinter QuickCheck quickcheck-instances scientific
        semigroups serialise special-values spoon tasty
        tasty-expected-failure tasty-hunit tasty-quickcheck text
        transformers turtle unordered-containers vector
      ];
      benchmarkHaskellDepends = [
        base bytestring containers directory gauge serialise text
      ];
      description = "A configuration language guaranteed to terminate";
      license = stdenv.lib.licenses.bsd3;
    };

  # This is an overlay that fixes spago so it builds in nixpkgs master.
  spagoOverlay = self: super: {
    haskell = super.haskell // {
      packages = super.haskell.packages // {
        ghc865 = super.haskell.packages.ghc865.override {
          overrides = hself: hsuper: {
            # Spago in nixpkgs is currently broken.  Here is a PR fixing it up to work again:
            # https://github.com/NixOS/nixpkgs/pull/75931#issuecomment-567526192
            # However, instead of using `dhall-1.28.0`, it is easier just to go
            # back to dhall-1.27.0.
            spago = hsuper.spago.override {
              dhall = hself.dhall_1_27_0;

              # This is due to my mistake when originally packaging spago...
              # https://github.com/NixOS/nixpkgs/pull/75931#issuecomment-567526192
              directory = hself.directory;
            };

            # I'm not sure why callHackage doesn't work here.
            # Maybe: https://github.com/nh2/static-haskell-nix/issues/9
            # dhall_1_27_0 = hsuper.callHackage "dhall" "1.27.0" {};

            dhall_1_27_0 = self.haskell.lib.dontCheck (self.haskell.lib.doJailbreak (hself.callPackage dhall_1_27_0_from_cabal2nix {}));
          };
        };
      };
    };
  };

  nixpkgs-src = builtins.fetchTarball {
    url = "https://github.com/NixOS/nixpkgs/archive/96d73edaf33f4c9ee452f65504bfdfab9a653eaa.tar.gz";
    sha256 = "1qgxfaa23f982b6a9alghl3f8hgm61b4097bmkvn8084n05iv9sm";
  };

  nixpkgs = import nixpkgs-src {
    overlays = [ spagoOverlay ];
  };

  static-haskell-nix-src = builtins.fetchTarball {
    url = "https://github.com/nh2/static-haskell-nix/archive/d24dea3d46a727e1cd93e67458ce2768109efe0a.tar.gz";
    sha256 = "0ri4r77md77kmbqmx0j3h0paj538zpq4hki7vy8ycpwssm2xd442";
  };

  static-haskell-nix-survey-src = static-haskell-nix-src + "/survey/default.nix";

  static-haskell-nix = import static-haskell-nix-survey-src {
    normalPkgs = nixpkgs;
  };

in

static-haskell-nix.haskellPackages.spago

This is using the great static-haskell-nix from @nh2.

If you have nix installed, and you save the above file as static-spago-test.nix, you should be able to build it as follows:

$ nix-build ./static-spago-test.nix
...
/nix/store/x1y6a6qh9rfkk2ngxi3602lsq87f8zrg-spago-0.12.1.0
$ file /nix/store/x1y6a6qh9rfkk2ngxi3602lsq87f8zrg-spago-0.12.1.0/bin/spago
/nix/store/x1y6a6qh9rfkk2ngxi3602lsq87f8zrg-spago-0.12.1.0/bin/spago: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, stripped
$ ldd /nix/store/x1y6a6qh9rfkk2ngxi3602lsq87f8zrg-spago-0.12.1.0/bin/spago
	not a dynamic executable
$ /nix/store/x1y6a6qh9rfkk2ngxi3602lsq87f8zrg-spago-0.12.1.0/bin/spago version
0.12.1.0

I didn't test everything, but it appears that spago init and spago build are working as well.

The initial build took about 6 hours on my machine (although static-haskell-nix provides a cache that can be used to get a much shorter build).

---

This is a good proof-of-concept, but it won't work for things like CI.

nixpkgs isn't very good for Haskell packages that aren't in Stackage, since there is no way to easily have the correct version of dependencies you need.

So if you bump some versions of spago's dependencies and release a new version of spago, we might have to update the spago derivation in nixpkgs to get it building again.

The way to work around this is to automatically generate a nix Haskell package set based on the stack.yaml file for spago. The most popular tool for doing this now is haskell.nix. My coworker @considerate was able to get some of our Haskell tools at work building statically using haskell.nix, so I believe there is some support for building statically with musl in haskell.nix, but I'm not sure if they have put as much work into it as @nh2 has in static-haskell-nix.

---

My recommendation on how to proceed with this would be as follows:

1. Build a statically-linked spago and spago-curator binary for the current release of spago (and maybe the next release)? Manually upload it to the releases page on GitHub. Or if you added me as a maintainer I could do this.

   See how many users actually use this statically-linked release. See if anyone reports any big problems.

2. If people are actually using it, I can throw together some code building spago statically with haskell.nix. I'm not sure how easy this will be to get working. (Although if static-haskell-nix works, then in theory it should be possible to get haskell.nix working as well).

   We can try to add this to CI. haskell.nix provides a nix cache to speed up builds, but I am worried that statically-linked stuff might not be available in their cache.

   I'm not sure what to do in this case. The initial build might time-out on things like Travis.

   One solution might be to use the GitHub actions provided by the cachix team: https://github.com/marketplace/actions/cachix. I think GitHub actions have a 6 hour timeout, so this might be easier to get working.

[cdepillabout]

One last thing, if you change the last line of that nix script above to be static-haskell-nix.haskellPackages.purescript, you can actually build purs statically as well:

$ nix-build static-spago-test.nix # with last line changed
...
/nix/store/h4588m743p091br79z0a0grsvrk7hipf-purescript-0.13.5
$ file /nix/store/h4588m743p091br79z0a0grsvrk7hipf-purescript-0.13.5/bin/purs 
/nix/store/h4588m743p091br79z0a0grsvrk7hipf-purescript-0.13.5/bin/purs: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, stripped
$ ldd /nix/store/h4588m743p091br79z0a0grsvrk7hipf-purescript-0.13.5/bin/purs 
	not a dynamic executable
$ /nix/store/h4588m743p091br79z0a0grsvrk7hipf-purescript-0.13.5/bin/purs --version
0.13.5 [development build; commit: UNKNOWN]
$ /nix/store/h4588m743p091br79z0a0grsvrk7hipf-purescript-0.13.5/bin/purs
...

Maybe @hdgarrood or the other PureScript maintainers would be interested in this. It doesn't look like there is currently a statically linked release of PureScript.

[f-f]

@cdepillabout thank you so much for looking into this and for the great writeup ❤️

To answer your points:

1. Yes! I added you as a contributor, so we can do this right away. I actually just cut a new release and I'll wait for your upload before announcing it
2. This sounds great too. We could try Travis, or just switch to Github Actions. I wouldn't mind moving the whole Linux build there too (i.e. both static and dynamic binaries), as the Travis build is full of macOS/Linux conditional statements that make it annoying to work with, so having it macOS only might be nicer

[cdepillabout]

@f-f Sounds good!

However, it looks like there is already a statically linked version of spago uploaded:

https://github.com/spacchetti/spago/releases/download/0.13.0/linux-static.tar.gz

Did you already add this? Is this sufficient for now? Or is there something else you wanted me to add?

[f-f]

@cdepillabout oh right, CI is building and uploading a broken static binary. Please remove that one and upload the good one there 😄

[cdepillabout]

@f-f I've uploaded a static binary for spago for linux:

https://github.com/spacchetti/spago/releases/download/0.13.0/linux-static.tar.gz

It can be reproduced with this nix script:

let
  nixpkgs-src = builtins.fetchTarball {
    # This is my branch that has spago-0.13.0 building:
    # https://github.com/NixOS/nixpkgs/pull/76084
    url = "https://github.com/NixOS/nixpkgs/archive/00303de7e2ef5967e2c8d51069c8c65dcb9ba09b.tar.gz";
    sha256 = "1nylgj97fnb7si8v31vc22j0jd1n5zg80b7cdl3si3jq75mp1fkw";
  };

  nixpkgs = import nixpkgs-src { };

  static-haskell-nix-src = builtins.fetchTarball {
    url = "https://github.com/nh2/static-haskell-nix/archive/d24dea3d46a727e1cd93e67458ce2768109efe0a.tar.gz";
    sha256 = "0ri4r77md77kmbqmx0j3h0paj538zpq4hki7vy8ycpwssm2xd442";
  };

  static-haskell-nix-survey-src = static-haskell-nix-src + "/survey/default.nix";

  static-haskell-nix = import static-haskell-nix-survey-src {
    normalPkgs = nixpkgs;
  };

  createTarball = drv: nixpkgs.runCommand "${drv.name}-tarball" {} ''
    mkdir -p "$out/"

    # Create a temporary directory that we can eventually tar up.
    tempdir="${drv.name}"
    mkdir "$tempdir"

    # Copy the spago and spago-curator binaries to the temporary directory.
    cp ${drv}/bin/* "$tempdir/"

    # Make sure temporary directory and binaries are writable by the user so
    # they can easily rm it.
    chmod u+w -R "$tempdir"

    # Create the tarball.
    tar -cf $out/linux-static.tar.gz "$tempdir"
  '';
in

createTarball static-haskell-nix.haskellPackages.spago
# static-haskell-nix.haskellPackages.purescript

If you save this script as static-spago-test.nix, you can build it and find the resulting tarball as follows:

$ nix-build ./static-spago-test.nix
...
/nix/store/1siy0765ax1v2xf57rn5i6bhpxab93gp-spago-0.13.0-tarball
$ ls /nix/store/1siy0765ax1v2xf57rn5i6bhpxab93gp-spago-0.13.0-tarball
linux-static.tar.gz

---

If people end up using this, maybe I can throw together another proof of concept using something like haskell.nix, and we can try to setup CI to build it automatically.