Generate config struct from NixOS options (#33)

* add clan.lol jsonschema library

* move devshell definition, add go-jsonschema package

* use generated schema for config file parsing

* include schema generation in build process

.gitignore

@@ -3,3 +3,4 @@ hyprspace
 result
 result-*
 .data/
+*_generated.go

cli/init.go

@@ -7,7 +7,7 @@ import (
 	"path/filepath"
 
 	"github.com/DataDrake/cli-ng/v2/cmd"
-	"github.com/hyprspace/hyprspace/config"
+	"github.com/hyprspace/hyprspace/schema"
 	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/multiformats/go-multibase"
@@ -39,16 +39,15 @@ func InitRun(r *cmd.Root, c *cmd.Sub) {
 	keyBytes, err := crypto.MarshalPrivateKey(privKey)
 	checkErr(err)
 
-	// Setup an initial default command.
-	new := config.Config{
-		EncodedPrivateKey: multibase.MustNewEncoder(multibase.Base58BTC).Encode(keyBytes),
-		EncodedListenAddresses: []string{
+	// Setup an initial default config.
+	new := schema.Config{
+		PrivateKey: multibase.MustNewEncoder(multibase.Base58BTC).Encode(keyBytes),
+		ListenAddresses: []string{
 			"/ip4/0.0.0.0/tcp/8001",
 			"/ip4/0.0.0.0/udp/8001/quic-v1",
 			"/ip6/::/tcp/8001",
 			"/ip6/::/udp/8001/quic-v1",
 		},
-		Peers: make([]config.Peer, 0),
 	}
 
 	out, err := json.MarshalIndent(&new, "", "  ")

config/config.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/hyprspace/hyprspace/schema"
 	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/multiformats/go-multiaddr"
@@ -17,18 +18,15 @@ import (
 
 // Config is the main Configuration Struct for Hyprspace.
 type Config struct {
-	Path                   string                         `json:"-"`
-	Interface              string                         `json:"-"`
-	EncodedListenAddresses []string                       `json:"listenAddresses"`
-	ListenAddresses        []multiaddr.Multiaddr          `json:"-"`
-	Peers                  []Peer                         `json:"peers"`
-	PeerLookup             PeerLookup                     `json:"-"`
-	EncodedPrivateKey      string                         `json:"privateKey"`
-	PrivateKey             crypto.PrivKey                 `json:"-"`
-	BuiltinAddr4           net.IP                         `json:"-"`
-	BuiltinAddr6           net.IP                         `json:"-"`
-	EncodedServices        map[string]string              `json:"services,omitempty"`
-	Services               map[string]multiaddr.Multiaddr `json:"-"`
+	Path            string                         `json:"-"`
+	Interface       string                         `json:"-"`
+	ListenAddresses []multiaddr.Multiaddr          `json:"-"`
+	Peers           []Peer                         `json:"peers"`
+	PeerLookup      PeerLookup                     `json:"-"`
+	PrivateKey      crypto.PrivKey                 `json:"-"`
+	BuiltinAddr4    net.IP                         `json:"-"`
+	BuiltinAddr6    net.IP                         `json:"-"`
+	Services        map[string]multiaddr.Multiaddr `json:"-"`
 }
 
 // Peer defines a peer in the configuration. We might add more to this later.
@@ -37,12 +35,6 @@ type Peer struct {
 	Name         string  `json:"name"`
 	BuiltinAddr4 net.IP  `json:"-"`
 	BuiltinAddr6 net.IP  `json:"-"`
-	Routes       []Route `json:"routes"`
-}
-
-type Route struct {
-	NetworkStr string    `json:"net"`
-	Network    net.IPNet `json:"-"`
 }
 
 // PeerLookup is a helper struct for quickly looking up a peer based on various parameters
@@ -67,22 +59,16 @@ func Read(path string) (*Config, error) {
 	if err != nil {
 		return nil, err
 	}
-	result := Config{
-		EncodedListenAddresses: []string{
-			"/ip4/0.0.0.0/tcp/8001",
-			"/ip4/0.0.0.0/udp/8001/quic-v1",
-			"/ip6/::/tcp/8001",
-			"/ip6/::/udp/8001/quic-v1",
-		},
-	}
+	input := schema.Config{}
+	result := Config{}
 
 	// Read in config settings from file.
-	err = json.Unmarshal(in, &result)
+	err = json.Unmarshal(in, &input)
 	if err != nil {
 		return nil, err
 	}
 
-	_, keyBytes, err := multibase.Decode(result.EncodedPrivateKey)
+	_, keyBytes, err := multibase.Decode(input.PrivateKey)
 	if err != nil {
 		return nil, err
 	}
@@ -102,7 +88,7 @@ func Read(path string) (*Config, error) {
 	result.BuiltinAddr4 = mkBuiltinAddr4(peerID)
 	result.BuiltinAddr6 = mkBuiltinAddr6(peerID)
 
-	for _, addrString := range result.EncodedListenAddresses {
+	for _, addrString := range input.ListenAddresses {
 		addr, err := multiaddr.NewMultiaddr(addrString)
 		if err != nil {
 			return nil, err
@@ -113,49 +99,52 @@ func Read(path string) (*Config, error) {
 	result.PeerLookup.ByRoute = cidranger.NewPCTrieRanger()
 	result.PeerLookup.ByName = make(map[string]Peer)
 	result.PeerLookup.ByNetID = make(map[[4]byte]Peer)
+	result.Peers = make([]Peer, len(input.Peers))
 
-	for i, p := range result.Peers {
+	for i, configPeer := range input.Peers {
+		p := Peer{}
+		p.ID, err = peer.Decode(configPeer.Id)
+		if err != nil {
+			return nil, err
+		}
 		p.BuiltinAddr4 = mkBuiltinAddr4(p.ID)
 		p.BuiltinAddr6 = mkBuiltinAddr6(p.ID)
-		p.Routes = append(p.Routes,
-			Route{
-				Network: net.IPNet{
-					IP:   p.BuiltinAddr4,
-					Mask: net.IPv4Mask(255, 255, 255, 255),
-				},
-			},
-			Route{
-				Network: net.IPNet{
-					IP:   p.BuiltinAddr6,
-					Mask: net.CIDRMask(128, 128),
-				},
-			},
-		)
-		for _, r := range p.Routes {
-			if r.NetworkStr != "" {
-				_, n, err := net.ParseCIDR(r.NetworkStr)
-				if err != nil {
-					log.Fatal("[!] Invalid network:", r.NetworkStr)
-				}
-				r.Network = *n
+		for _, r := range configPeer.Routes {
+			_, network, err := net.ParseCIDR(r.Net)
+			if err != nil {
+				log.Fatal("[!] Invalid network:", r.Net)
 			}
 
 			result.PeerLookup.ByRoute.Insert(&RouteTableEntry{
-				Net:    r.Network,
+				Net:    *network,
 				Target: p,
 			})
 
-			fmt.Printf("[+] Route %s via /p2p/%s\n", r.Network.String(), p.ID)
+			fmt.Printf("[+] Route %s via /p2p/%s\n", network.String(), p.ID)
 		}
-		if p.Name != "" {
-			result.PeerLookup.ByName[strings.ToLower(p.Name)] = p
+		result.PeerLookup.ByRoute.Insert(&RouteTableEntry{
+			Net: net.IPNet{
+				IP:   p.BuiltinAddr4,
+				Mask: net.CIDRMask(32, 32),
+			},
+			Target: p,
+		})
+		result.PeerLookup.ByRoute.Insert(&RouteTableEntry{
+			Net: net.IPNet{
+				IP:   p.BuiltinAddr6,
+				Mask: net.CIDRMask(128, 128),
+			},
+			Target: p,
+		})
+		if configPeer.Name != "" {
+			result.PeerLookup.ByName[strings.ToLower(configPeer.Name)] = p
 		}
 		result.PeerLookup.ByNetID[[4]byte(p.BuiltinAddr6[12:16])] = p
 		result.Peers[i] = p
 	}
 
 	result.Services = make(map[string]multiaddr.Multiaddr)
-	for name, addrString := range result.EncodedServices {
+	for name, addrString := range input.Services {
 		addr, err := multiaddr.NewMultiaddr(addrString)
 		if err != nil {
 			return nil, err

dev/default.nix

@@ -0,0 +1,16 @@
+{
+  imports = [
+    ./generate-schemas.nix
+  ];
+
+  perSystem = { config, pkgs, ... }: {
+    devShells.default = pkgs.mkShell {
+      packages = [ pkgs.go ];
+
+      shellHook = ''
+        export GOPATH="$PWD/.data/go";
+        ${config.apps.dev-generate-schemas.program}
+      '';
+    };
+  };
+}

dev/generate-schemas.nix

@@ -0,0 +1,23 @@
+{ lib, ... }:
+
+{
+  perSystem = { pkgs, ... }: let
+    go-jsonschema = pkgs.callPackage ./pkgs/go-jsonschema {};
+
+    jsonschema = import ./lib/jsonschema.nix { inherit lib; };
+
+    schema = jsonschema.parseModule ../nixos/settings.nix;
+
+    schemaFile = builtins.toFile "hyprspace-config-schema.json" (builtins.toJSON (schema // {
+      title = "Config";
+    }));
+
+  in {
+    apps.dev-generate-schemas.program = pkgs.writeShellScriptBin "hyprspace-generate-schemas" ''
+      if [[ "$GOFILE" != "generate.go" ]]; then
+        cd schema
+      fi
+      ${go-jsonschema}/bin/go-jsonschema -p schema ${schemaFile} --tags json -t -o config_generated.go
+    '';
+  };
+}

dev/lib/jsonschema.nix

@@ -0,0 +1,331 @@
+# from https://git.clan.lol/clan/clan-core/raw/commit/0b34c340fc502235af25d103b2b83948b4ad925e/lib/jsonschema/default.nix
+
+{
+  lib ? import <nixpkgs/lib>,
+  excludedTypes ? [
+    "functionTo"
+    "package"
+  ],
+}:
+let
+  # remove _module attribute from options
+  clean = opts: builtins.removeAttrs opts [ "_module" ];
+
+  # throw error if option type is not supported
+  notSupported =
+    option:
+    lib.trace option throw ''
+      option type '${option.type.name}' ('${option.type.description}') not supported by jsonschema converter
+      location: ${lib.concatStringsSep "." option.loc}
+    '';
+
+  # Exclude the option if its type is in the excludedTypes list
+  # or if the option has a defaultText attribute
+  isExcludedOption =
+    option: ((lib.elem (option.type.name or null) excludedTypes) || (option ? defaultText));
+
+  filterExcluded = lib.filter (opt: !isExcludedOption opt);
+
+  filterExcludedAttrs = lib.filterAttrs (_name: opt: !isExcludedOption opt);
+
+  # Filter out options where the visible attribute is set to false
+  filterInvisibleOpts = lib.filterAttrs (_name: opt: opt.visible or true);
+
+  allBasicTypes = [
+    "boolean"
+    "integer"
+    "number"
+    "string"
+    "array"
+    "object"
+    "null"
+  ];
+in
+rec {
+
+  # parses a nixos module to a jsonschema
+  parseModule =
+    module:
+    let
+      evaled = lib.evalModules { modules = [ module ]; };
+    in
+    parseOptions evaled.options;
+
+  # parses a set of evaluated nixos options to a jsonschema
+  parseOptions =
+    options':
+    let
+      options = filterInvisibleOpts (filterExcludedAttrs (clean options'));
+      # parse options to jsonschema properties
+      properties = lib.mapAttrs (_name: option: parseOption option) options;
+      # TODO: figure out how to handle if prop.anyOf is used
+      isRequired = prop: !(prop ? default || prop.type or null == "object");
+      requiredProps = lib.filterAttrs (_: prop: isRequired prop) properties;
+      required = lib.optionalAttrs (requiredProps != { }) { required = lib.attrNames requiredProps; };
+    in
+    # return jsonschema
+    required
+    // {
+      type = "object";
+      inherit properties;
+    };
+
+  # parses and evaluated nixos option to a jsonschema property definition
+  parseOption =
+    option:
+    let
+      default = lib.optionalAttrs (option ? default) { inherit (option) default; };
+      example = lib.optionalAttrs (option ? example) {
+        examples =
+          if (builtins.typeOf option.example) == "list" then option.example else [ option.example ];
+      };
+      description = lib.optionalAttrs (option ? description) {
+        description = option.description.text or option.description;
+      };
+    in
+
+    # either type
+    # TODO: if all nested options are excluded, the parent should be excluded too
+    if
+      option.type.name or null == "either" || option.type.name or null == "coercedTo"
+    # return jsonschema property definition for either
+    then
+      let
+        optionsList' = [
+          {
+            type = option.type.nestedTypes.left or option.type.nestedTypes.coercedType;
+            _type = "option";
+            loc = option.loc;
+          }
+          {
+            type = option.type.nestedTypes.right or option.type.nestedTypes.finalType;
+            _type = "option";
+            loc = option.loc;
+          }
+        ];
+        optionsList = filterExcluded optionsList';
+      in
+      default // example // description // { anyOf = map parseOption optionsList; }
+
+    # handle nested options (not a submodule)
+    else if !option ? _type then
+      parseOptions option
+
+    # throw if not an option
+    else if option._type != "option" && option._type != "option-type" then
+      throw "parseOption: not an option"
+
+    # parse nullOr
+    else if
+      option.type.name == "nullOr"
+    # return jsonschema property definition for nullOr
+    then
+      let
+        nestedOption = {
+          type = option.type.nestedTypes.elemType;
+          _type = "option";
+          loc = option.loc;
+        };
+      in
+      default
+      // example
+      // description
+      // {
+        anyOf = [
+          { type = "null"; }
+        ] ++ (lib.optional (!isExcludedOption nestedOption) (parseOption nestedOption));
+      }
+
+    # parse bool
+    else if
+      option.type.name == "bool"
+    # return jsonschema property definition for bool
+    then
+      default // example // description // { type = "boolean"; }
+
+    # parse float
+    else if
+      option.type.name == "float"
+    # return jsonschema property definition for float
+    then
+      default // example // description // { type = "number"; }
+
+    # parse int
+    else if
+      (option.type.name == "int" || option.type.name == "positiveInt")
+    # return jsonschema property definition for int
+    then
+      default // example // description // { type = "integer"; }
+
+    # TODO: Add support for intMatching in jsonschema
+    # parse port type aka. "unsignedInt16"
+    else if
+      option.type.name == "unsignedInt16"
+      || option.type.name == "unsignedInt"
+      || option.type.name == "pkcs11"
+      || option.type.name == "intBetween"
+    then
+      default // example // description // { type = "integer"; }
+
+    # parse string
+    # TODO: parse more precise string types
+    else if
+      option.type.name == "str"
+      || option.type.name == "singleLineStr"
+      || option.type.name == "passwdEntry str"
+      || option.type.name == "passwdEntry path"
+    # return jsonschema property definition for string
+    then
+      default // example // description // { type = "string"; }
+
+    # TODO: Add support for stringMatching in jsonschema
+    # parse stringMatching
+    else if lib.strings.hasPrefix "strMatching" option.type.name then
+      default // example // description // { type = "string"; }
+
+    # TODO: Add support for separatedString in jsonschema
+    else if lib.strings.hasPrefix "separatedString" option.type.name then
+      default // example // description // { type = "string"; }
+
+    # parse string
+    else if
+      option.type.name == "path"
+    # return jsonschema property definition for path
+    then
+      default // example // description // { type = "string"; }
+
+    # parse anything
+    else if
+      option.type.name == "anything"
+    # return jsonschema property definition for anything
+    then
+      default // example // description // { type = allBasicTypes; }
+
+    # parse unspecified
+    else if
+      option.type.name == "unspecified"
+    # return jsonschema property definition for unspecified
+    then
+      default // example // description // { type = allBasicTypes; }
+
+    # parse raw
+    else if
+      option.type.name == "raw"
+    # return jsonschema property definition for raw
+    then
+      default // example // description // { type = allBasicTypes; }
+
+    # parse enum
+    else if
+      option.type.name == "enum"
+    # return jsonschema property definition for enum
+    then
+      default // example // description // { enum = option.type.functor.payload; }
+
+    # parse listOf submodule
+    else if
+      option.type.name == "listOf" && option.type.functor.wrapped.name == "submodule"
+    # return jsonschema property definition for listOf submodule
+    then
+      default
+      // example
+      // description
+      // {
+        type = "array";
+        items = parseOptions (option.type.functor.wrapped.getSubOptions option.loc);
+      }
+
+    # parse list
+    else if
+      (option.type.name == "listOf")
+    # return jsonschema property definition for list
+    then
+      let
+        nestedOption = {
+          type = option.type.functor.wrapped;
+          _type = "option";
+          loc = option.loc;
+        };
+      in
+      default
+      // example
+      // description
+      // {
+        type = "array";
+      }
+      // (lib.optionalAttrs (!isExcludedOption nestedOption) { items = parseOption nestedOption; })
+
+    # parse list of unspecified
+    else if
+      (option.type.name == "listOf") && (option.type.functor.wrapped.name == "unspecified")
+    # return jsonschema property definition for list
+    then
+      default // example // description // { type = "array"; }
+
+    # parse attrsOf submodule
+    else if
+      option.type.name == "attrsOf" && option.type.nestedTypes.elemType.name == "submodule"
+    # return jsonschema property definition for attrsOf submodule
+    then
+      default
+      // example
+      // description
+      // {
+        type = "object";
+        additionalProperties = parseOptions (option.type.nestedTypes.elemType.getSubOptions option.loc);
+      }
+
+    # parse attrs
+    else if
+      option.type.name == "attrs"
+    # return jsonschema property definition for attrs
+    then
+      default
+      // example
+      // description
+      // {
+        type = "object";
+        additionalProperties = true;
+      }
+
+    # parse attrsOf
+    # TODO: if nested option is excluded, the parent sould be excluded too
+    else if
+      option.type.name == "attrsOf" || option.type.name == "lazyAttrsOf"
+    # return jsonschema property definition for attrs
+    then
+      let
+        nestedOption = {
+          type = option.type.nestedTypes.elemType;
+          _type = "option";
+          loc = option.loc;
+        };
+      in
+      default
+      // example
+      // description
+      // {
+        type = "object";
+        additionalProperties =
+          if !isExcludedOption nestedOption then
+            parseOption {
+              type = option.type.nestedTypes.elemType;
+              _type = "option";
+              loc = option.loc;
+            }
+          else
+            false;
+      }
+
+    # parse submodule
+    else if
+      option.type.name == "submodule"
+    # return jsonschema property definition for submodule
+    # then (lib.attrNames (option.type.getSubOptions option.loc).opt)
+    then
+      parseOptions (option.type.getSubOptions option.loc)
+
+    # throw error if option type is not supported
+    else
+      notSupported option;
+}

dev/pkgs/go-jsonschema/default.nix

@@ -0,0 +1,34 @@
+{ lib
+, buildGoModule
+, fetchFromGitHub
+}:
+
+buildGoModule rec {
+  pname = "go-jsonschema";
+  version = "0.16.0";
+
+  src = fetchFromGitHub {
+    owner = "omissis";
+    repo = "go-jsonschema";
+    rev = "v${version}";
+    hash = "sha256-+CapTmg4RObK6mzjAS/EFbX4s2AtQvlFXmT119aUkZA=";
+  };
+
+  vendorHash = "sha256-gk+aKGqcHEjuYxc2o+83HA2AxU+jT7URt0N/q+uyUtA=";
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X=main.version=${version}"
+    "-X=main.gitCommit=${src.rev}"
+    "-X=main.buildTime=1970-01-01T00:00:00Z"
+  ];
+
+  subPackages = ["."];
+  meta = with lib; {
+    description = "A tool to generate Go data types from JSON Schema definitions";
+    homepage = "https://github.com/omissis/go-jsonschema";
+    license = licenses.mit;
+    mainProgram = "go-jsonschema";
+  };
+}

flake.nix

@@ -23,23 +23,21 @@
         "aarch64-linux"
       ];
 
+      imports = [
+        ./dev
+      ];
+
       perSystem =
         { config, pkgs, ... }:
         {
           packages = {
-            default = pkgs.callPackage ./package.nix {};
+            default = pkgs.callPackage ./package.nix {
+              generateSchemasProgram = config.apps.dev-generate-schemas.program;
+            };
             docs = pkgs.callPackage ./docs/package.nix {
               hyprspace = config.packages.default;
             };
           };
-
-          devShells.default = pkgs.mkShell {
-            packages = [ pkgs.go ];
-
-            shellHook = ''
-              export GOPATH="$PWD/.data/go";
-            '';
-          };
         };
     };
 }

package.nix

@@ -1,4 +1,4 @@
-{ lib, buildGoModule }:
+{ lib, buildGoModule, generateSchemasProgram }:
 let
   inherit (lib.fileset) toSource unions fileFilter;
   pname = "hyprspace";
@@ -26,6 +26,10 @@ buildGoModule {
     "-X github.com/hyprspace/hyprspace/cli.appVersion=${version}"
   ];
 
+  postPatch = ''
+    ( set -x; ${generateSchemasProgram} )
+  '';
+
   meta = {
     description = "A Lightweight VPN Built on top of Libp2p for Truly Distributed Networks.";
     homepage = "https://github.com/hyprspace/hyprspace";

schema/generate.go

@@ -0,0 +1,3 @@
+package schema
+
+//go:generate nix run ..#dev-generate-schemas