In a setup with two Nix Flakes, where one provides a plugin for the other's application, "path <…> is not valid". How to fix that?

I have two Nix Flakes: One contains an application, and the other contains a plugin for that application. When I build the application with the plugin, I get the error

error: path '/nix/store/3b7djb5pr87zbscggsr7vnkriw3yp21x-mainapp-go-modules' is not valid

I have no idea what this error means and how to fix it, but I can reproduce it on both macOS and Linux. The path in question is the vendor directory generated by the first step of buildGoModule.

The minimal setup to reproduce the error requires a bunch of files, so I provide a commented bash script that you can execute in an empty folder to recreate my setup:

#!/bin/bash

# I have two flakes: the main application and a plugin.
# the mainapp needs to be inside the plugin directory
# so that nix doesn't complain about the path:mainapp
# reference being outside the parent's root.
mkdir -p plugin/mainapp

# each is a go module with minimal setup
tee plugin/mainapp/go.mod <<EOF >/dev/null
module example.com/mainapp

go 1.16
EOF
tee plugin/go.mod <<EOF >/dev/null
module example.com/plugin

go 1.16
EOF

# each contain minimal Go code
tee plugin/mainapp/main.go <<EOF >/dev/null
package main

import "fmt"

func main() {
	fmt.Println("Hello, World!")
}
EOF
tee plugin/main.go <<EOF >/dev/null
package plugin

import log

func init() {
	fmt.Println("initializing plugin")
}
EOF

# the mainapp is a flake that provides a function for building
# the app, as well as a default package that is the app
# without any plugins.
tee plugin/mainapp/flake.nix <<'EOF' >/dev/null
{
  description = "main application";
  inputs = {
    nixpkgs.url = github:NixOS/nixpkgs/nixos-21.11;
    flake-utils.url = github:numtide/flake-utils;
  };
  outputs = {self, nixpkgs, flake-utils}:
  let
    # buildApp builds the application from a list of plugins.
    # plugins cause the vendorSha256 to change, hence it is
    # given as additional parameter.
    buildApp = { pkgs, vendorSha256, plugins ? [] }:
      let
        # this is appended to the mainapp's go.mod so that it
        # knows about the plugin and where to find it.
        requirePlugin = plugin: ''
          require ${plugin.goPlugin.goModName} v0.0.0
          replace ${plugin.goPlugin.goModName} => ${plugin.outPath}/src
        '';
        # since buildGoModule consumes the source two times –
        # first for vendoring, and then for building –
        # we do the necessary modifications to the sources in an
        # own derivation and then hand that to buildGoModule.
        sources = pkgs.stdenvNoCC.mkDerivation {
          name = "mainapp-with-plugins-source";
          src = self;
          phases = [ "unpackPhase" "buildPhase" "installPhase" ];
		  # write a plugins.go file that references the plugin's package via
		  #   _ = "<module path>"
          PLUGINS_GO = ''
            package main
              
            // Code generated by Nix. DO NOT EDIT.
          
            import (
            	${builtins.foldl' (a: b: a + "\n\t_ = \"${b.goPlugin.goModName}\"") "" plugins}
            )
          '';
          GO_MOD_APPEND = builtins.foldl' (a: b: a + "${requirePlugin b}\n") "" plugins;
          buildPhase = ''
            printenv PLUGINS_GO >plugins.go
            printenv GO_MOD_APPEND >>go.mod
          '';
          installPhase = ''
            mkdir -p $out
            cp -r -t $out *
          '';
        };
      in pkgs.buildGoModule {
        name = "mainapp";
        src = builtins.trace "sources at ${sources}" sources;
        inherit vendorSha256;
        nativeBuildInputs = plugins;
      };
  in (flake-utils.lib.eachDefaultSystem (system:
    let
      pkgs = nixpkgs.legacyPackages.${system};
    in rec {
      defaultPackage = buildApp {
        inherit pkgs;
		# this may be different depending on your nixpkgs; if it is, just change it.
        vendorSha256 = "sha256-pQpattmS9VmO3ZIQUFn66az8GSmB4IvYhTTCFn6SUmo=";
      };
    }
  )) // {
    lib = {
      inherit buildApp;
	  # helper that parses a go.mod file for the module's name
      pluginMetadata = goModFile: {
        goModName = with builtins; head
          (match "module ([^[:space:]]+).*" (readFile goModFile));
      };
    };
  };
}
EOF

# the plugin is a flake depending on the mainapp that outputs a plugin package,
# and also a package that is the mainapp compiled with this plugin.
tee plugin/flake.nix <<'EOF' >/dev/null
{
  description = "mainapp plugin";
  inputs = {
    nixpkgs.url = github:NixOS/nixpkgs/nixos-21.11;
    flake-utils.url = github:numtide/flake-utils;
    nix-filter.url = github:numtide/nix-filter;
    mainapp.url = path:mainapp;
    mainapp.inputs = {
      nixpkgs.follows = "nixpkgs";
      flake-utils.follows = "flake-utils";
    };
  };
  outputs = {self, nixpkgs, flake-utils, nix-filter, mainapp}:
  flake-utils.lib.eachDefaultSystem (system:
    let
      pkgs = nixpkgs.legacyPackages.${system};
    in rec {
      packages = rec {
        plugin = pkgs.stdenvNoCC.mkDerivation {
          pname = "mainapp-plugin";
          version = "0.1.0";
          src = nix-filter.lib.filter {
            root = ./.;
            exclude = [ ./mainapp ./flake.nix ./flake.lock ];  
          };
          # needed for mainapp to recognize this as plugin
          passthru.goPlugin = mainapp.lib.pluginMetadata ./go.mod;
          phases = [ "unpackPhase" "installPhase" ];
          installPhase = ''
            mkdir -p $out/src
            cp -r -t $out/src *
          '';
        };
        app = mainapp.lib.buildApp {
          inherit pkgs;
		  # this may be different depending on your nixpkgs; if it is, just change it.
          vendorSha256 = "sha256-a6HFGFs1Bu9EkXwI+DxH5QY2KBcdPzgP7WX6byai4hw=";
          plugins = [ plugin ];
        };
      };
      defaultPackage = packages.app;
    }
  );
}
EOF

You need Nix with Flake support installed to reproduce the error.
In the plugin folder created by this script, execute

$ nix build
trace: sources at /nix/store/d5arinbiaspyjjc4ypk4h5dsjx22pcsf-mainapp-with-plugins-source
error: path '/nix/store/3b7djb5pr87zbscggsr7vnkriw3yp21x-mainapp-go-modules' is not valid

(If you get hash mismatches, just update the flakes with the correct hash; I am not quite sure whether hashing when spreading flakes outside of a repository is reproducible.)

The sources directory (shown by trace) does exist and looks okay. The path given in the error message also exists and contains modules.txt with expected content.

In the folder mainapp, nix build does run successfully, which builds the app without plugins. So what is it that I do with the plugin that makes the path invalid?

The reason is that the file modules.txt generated as part of vendoring will contain the nix store path in the replace directive in this scenario. The vendor directory is a fixed output derivation and thus must not depend on any other derivations. This is violated by the reference in modules.txt.

This can only be fixed by copying the plugin's sources into the sources derivation – that way, the replace path can be relative and thus references no other nix store path.