From f8393c418e03fae54a6ec7cd8f3b086afff0583e Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 2 Jun 2024 19:03:02 +0200 Subject: [PATCH] Generate config struct from NixOS options (#33) * add clan.lol jsonschema library * move devshell definition, add go-jsonschema package * use generated schema for config file parsing * include schema generation in build process --- .gitignore | 1 + cli/init.go | 11 +- config/config.go | 101 ++++----- dev/default.nix | 16 ++ dev/generate-schemas.nix | 23 ++ dev/lib/jsonschema.nix | 331 +++++++++++++++++++++++++++++ dev/pkgs/go-jsonschema/default.nix | 34 +++ flake.nix | 16 +- package.nix | 6 +- schema/generate.go | 3 + 10 files changed, 470 insertions(+), 72 deletions(-) create mode 100644 dev/default.nix create mode 100644 dev/generate-schemas.nix create mode 100644 dev/lib/jsonschema.nix create mode 100644 dev/pkgs/go-jsonschema/default.nix create mode 100644 schema/generate.go diff --git a/.gitignore b/.gitignore index 4fd9283..b9e8de4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ hyprspace result result-* .data/ +*_generated.go diff --git a/cli/init.go b/cli/init.go index ad63381..0a0714d 100644 --- a/cli/init.go +++ b/cli/init.go @@ -7,7 +7,7 @@ import ( "path/filepath" "github.com/DataDrake/cli-ng/v2/cmd" - "github.com/hyprspace/hyprspace/config" + "github.com/hyprspace/hyprspace/schema" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multibase" @@ -39,16 +39,15 @@ func InitRun(r *cmd.Root, c *cmd.Sub) { keyBytes, err := crypto.MarshalPrivateKey(privKey) checkErr(err) - // Setup an initial default command. - new := config.Config{ - EncodedPrivateKey: multibase.MustNewEncoder(multibase.Base58BTC).Encode(keyBytes), - EncodedListenAddresses: []string{ + // Setup an initial default config. + new := schema.Config{ + PrivateKey: multibase.MustNewEncoder(multibase.Base58BTC).Encode(keyBytes), + ListenAddresses: []string{ "/ip4/0.0.0.0/tcp/8001", "/ip4/0.0.0.0/udp/8001/quic-v1", "/ip6/::/tcp/8001", "/ip6/::/udp/8001/quic-v1", }, - Peers: make([]config.Peer, 0), } out, err := json.MarshalIndent(&new, "", " ") diff --git a/config/config.go b/config/config.go index 7b375cb..efe6f3d 100644 --- a/config/config.go +++ b/config/config.go @@ -8,6 +8,7 @@ import ( "os" "strings" + "github.com/hyprspace/hyprspace/schema" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" @@ -17,18 +18,15 @@ import ( // Config is the main Configuration Struct for Hyprspace. type Config struct { - Path string `json:"-"` - Interface string `json:"-"` - EncodedListenAddresses []string `json:"listenAddresses"` - ListenAddresses []multiaddr.Multiaddr `json:"-"` - Peers []Peer `json:"peers"` - PeerLookup PeerLookup `json:"-"` - EncodedPrivateKey string `json:"privateKey"` - PrivateKey crypto.PrivKey `json:"-"` - BuiltinAddr4 net.IP `json:"-"` - BuiltinAddr6 net.IP `json:"-"` - EncodedServices map[string]string `json:"services,omitempty"` - Services map[string]multiaddr.Multiaddr `json:"-"` + Path string `json:"-"` + Interface string `json:"-"` + ListenAddresses []multiaddr.Multiaddr `json:"-"` + Peers []Peer `json:"peers"` + PeerLookup PeerLookup `json:"-"` + PrivateKey crypto.PrivKey `json:"-"` + BuiltinAddr4 net.IP `json:"-"` + BuiltinAddr6 net.IP `json:"-"` + Services map[string]multiaddr.Multiaddr `json:"-"` } // Peer defines a peer in the configuration. We might add more to this later. @@ -37,12 +35,6 @@ type Peer struct { Name string `json:"name"` BuiltinAddr4 net.IP `json:"-"` BuiltinAddr6 net.IP `json:"-"` - Routes []Route `json:"routes"` -} - -type Route struct { - NetworkStr string `json:"net"` - Network net.IPNet `json:"-"` } // PeerLookup is a helper struct for quickly looking up a peer based on various parameters @@ -67,22 +59,16 @@ func Read(path string) (*Config, error) { if err != nil { return nil, err } - result := Config{ - EncodedListenAddresses: []string{ - "/ip4/0.0.0.0/tcp/8001", - "/ip4/0.0.0.0/udp/8001/quic-v1", - "/ip6/::/tcp/8001", - "/ip6/::/udp/8001/quic-v1", - }, - } + input := schema.Config{} + result := Config{} // Read in config settings from file. - err = json.Unmarshal(in, &result) + err = json.Unmarshal(in, &input) if err != nil { return nil, err } - _, keyBytes, err := multibase.Decode(result.EncodedPrivateKey) + _, keyBytes, err := multibase.Decode(input.PrivateKey) if err != nil { return nil, err } @@ -102,7 +88,7 @@ func Read(path string) (*Config, error) { result.BuiltinAddr4 = mkBuiltinAddr4(peerID) result.BuiltinAddr6 = mkBuiltinAddr6(peerID) - for _, addrString := range result.EncodedListenAddresses { + for _, addrString := range input.ListenAddresses { addr, err := multiaddr.NewMultiaddr(addrString) if err != nil { return nil, err @@ -113,49 +99,52 @@ func Read(path string) (*Config, error) { result.PeerLookup.ByRoute = cidranger.NewPCTrieRanger() result.PeerLookup.ByName = make(map[string]Peer) result.PeerLookup.ByNetID = make(map[[4]byte]Peer) + result.Peers = make([]Peer, len(input.Peers)) - for i, p := range result.Peers { + for i, configPeer := range input.Peers { + p := Peer{} + p.ID, err = peer.Decode(configPeer.Id) + if err != nil { + return nil, err + } p.BuiltinAddr4 = mkBuiltinAddr4(p.ID) p.BuiltinAddr6 = mkBuiltinAddr6(p.ID) - p.Routes = append(p.Routes, - Route{ - Network: net.IPNet{ - IP: p.BuiltinAddr4, - Mask: net.IPv4Mask(255, 255, 255, 255), - }, - }, - Route{ - Network: net.IPNet{ - IP: p.BuiltinAddr6, - Mask: net.CIDRMask(128, 128), - }, - }, - ) - for _, r := range p.Routes { - if r.NetworkStr != "" { - _, n, err := net.ParseCIDR(r.NetworkStr) - if err != nil { - log.Fatal("[!] Invalid network:", r.NetworkStr) - } - r.Network = *n + for _, r := range configPeer.Routes { + _, network, err := net.ParseCIDR(r.Net) + if err != nil { + log.Fatal("[!] Invalid network:", r.Net) } result.PeerLookup.ByRoute.Insert(&RouteTableEntry{ - Net: r.Network, + Net: *network, Target: p, }) - fmt.Printf("[+] Route %s via /p2p/%s\n", r.Network.String(), p.ID) + fmt.Printf("[+] Route %s via /p2p/%s\n", network.String(), p.ID) } - if p.Name != "" { - result.PeerLookup.ByName[strings.ToLower(p.Name)] = p + result.PeerLookup.ByRoute.Insert(&RouteTableEntry{ + Net: net.IPNet{ + IP: p.BuiltinAddr4, + Mask: net.CIDRMask(32, 32), + }, + Target: p, + }) + result.PeerLookup.ByRoute.Insert(&RouteTableEntry{ + Net: net.IPNet{ + IP: p.BuiltinAddr6, + Mask: net.CIDRMask(128, 128), + }, + Target: p, + }) + if configPeer.Name != "" { + result.PeerLookup.ByName[strings.ToLower(configPeer.Name)] = p } result.PeerLookup.ByNetID[[4]byte(p.BuiltinAddr6[12:16])] = p result.Peers[i] = p } result.Services = make(map[string]multiaddr.Multiaddr) - for name, addrString := range result.EncodedServices { + for name, addrString := range input.Services { addr, err := multiaddr.NewMultiaddr(addrString) if err != nil { return nil, err diff --git a/dev/default.nix b/dev/default.nix new file mode 100644 index 0000000..d30239d --- /dev/null +++ b/dev/default.nix @@ -0,0 +1,16 @@ +{ + imports = [ + ./generate-schemas.nix + ]; + + perSystem = { config, pkgs, ... }: { + devShells.default = pkgs.mkShell { + packages = [ pkgs.go ]; + + shellHook = '' + export GOPATH="$PWD/.data/go"; + ${config.apps.dev-generate-schemas.program} + ''; + }; + }; +} diff --git a/dev/generate-schemas.nix b/dev/generate-schemas.nix new file mode 100644 index 0000000..0dd4084 --- /dev/null +++ b/dev/generate-schemas.nix @@ -0,0 +1,23 @@ +{ lib, ... }: + +{ + perSystem = { pkgs, ... }: let + go-jsonschema = pkgs.callPackage ./pkgs/go-jsonschema {}; + + jsonschema = import ./lib/jsonschema.nix { inherit lib; }; + + schema = jsonschema.parseModule ../nixos/settings.nix; + + schemaFile = builtins.toFile "hyprspace-config-schema.json" (builtins.toJSON (schema // { + title = "Config"; + })); + + in { + apps.dev-generate-schemas.program = pkgs.writeShellScriptBin "hyprspace-generate-schemas" '' + if [[ "$GOFILE" != "generate.go" ]]; then + cd schema + fi + ${go-jsonschema}/bin/go-jsonschema -p schema ${schemaFile} --tags json -t -o config_generated.go + ''; + }; +} diff --git a/dev/lib/jsonschema.nix b/dev/lib/jsonschema.nix new file mode 100644 index 0000000..53a4c06 --- /dev/null +++ b/dev/lib/jsonschema.nix @@ -0,0 +1,331 @@ +# from https://git.clan.lol/clan/clan-core/raw/commit/0b34c340fc502235af25d103b2b83948b4ad925e/lib/jsonschema/default.nix + +{ + lib ? import , + excludedTypes ? [ + "functionTo" + "package" + ], +}: +let + # remove _module attribute from options + clean = opts: builtins.removeAttrs opts [ "_module" ]; + + # throw error if option type is not supported + notSupported = + option: + lib.trace option throw '' + option type '${option.type.name}' ('${option.type.description}') not supported by jsonschema converter + location: ${lib.concatStringsSep "." option.loc} + ''; + + # Exclude the option if its type is in the excludedTypes list + # or if the option has a defaultText attribute + isExcludedOption = + option: ((lib.elem (option.type.name or null) excludedTypes) || (option ? defaultText)); + + filterExcluded = lib.filter (opt: !isExcludedOption opt); + + filterExcludedAttrs = lib.filterAttrs (_name: opt: !isExcludedOption opt); + + # Filter out options where the visible attribute is set to false + filterInvisibleOpts = lib.filterAttrs (_name: opt: opt.visible or true); + + allBasicTypes = [ + "boolean" + "integer" + "number" + "string" + "array" + "object" + "null" + ]; +in +rec { + + # parses a nixos module to a jsonschema + parseModule = + module: + let + evaled = lib.evalModules { modules = [ module ]; }; + in + parseOptions evaled.options; + + # parses a set of evaluated nixos options to a jsonschema + parseOptions = + options': + let + options = filterInvisibleOpts (filterExcludedAttrs (clean options')); + # parse options to jsonschema properties + properties = lib.mapAttrs (_name: option: parseOption option) options; + # TODO: figure out how to handle if prop.anyOf is used + isRequired = prop: !(prop ? default || prop.type or null == "object"); + requiredProps = lib.filterAttrs (_: prop: isRequired prop) properties; + required = lib.optionalAttrs (requiredProps != { }) { required = lib.attrNames requiredProps; }; + in + # return jsonschema + required + // { + type = "object"; + inherit properties; + }; + + # parses and evaluated nixos option to a jsonschema property definition + parseOption = + option: + let + default = lib.optionalAttrs (option ? default) { inherit (option) default; }; + example = lib.optionalAttrs (option ? example) { + examples = + if (builtins.typeOf option.example) == "list" then option.example else [ option.example ]; + }; + description = lib.optionalAttrs (option ? description) { + description = option.description.text or option.description; + }; + in + + # either type + # TODO: if all nested options are excluded, the parent should be excluded too + if + option.type.name or null == "either" || option.type.name or null == "coercedTo" + # return jsonschema property definition for either + then + let + optionsList' = [ + { + type = option.type.nestedTypes.left or option.type.nestedTypes.coercedType; + _type = "option"; + loc = option.loc; + } + { + type = option.type.nestedTypes.right or option.type.nestedTypes.finalType; + _type = "option"; + loc = option.loc; + } + ]; + optionsList = filterExcluded optionsList'; + in + default // example // description // { anyOf = map parseOption optionsList; } + + # handle nested options (not a submodule) + else if !option ? _type then + parseOptions option + + # throw if not an option + else if option._type != "option" && option._type != "option-type" then + throw "parseOption: not an option" + + # parse nullOr + else if + option.type.name == "nullOr" + # return jsonschema property definition for nullOr + then + let + nestedOption = { + type = option.type.nestedTypes.elemType; + _type = "option"; + loc = option.loc; + }; + in + default + // example + // description + // { + anyOf = [ + { type = "null"; } + ] ++ (lib.optional (!isExcludedOption nestedOption) (parseOption nestedOption)); + } + + # parse bool + else if + option.type.name == "bool" + # return jsonschema property definition for bool + then + default // example // description // { type = "boolean"; } + + # parse float + else if + option.type.name == "float" + # return jsonschema property definition for float + then + default // example // description // { type = "number"; } + + # parse int + else if + (option.type.name == "int" || option.type.name == "positiveInt") + # return jsonschema property definition for int + then + default // example // description // { type = "integer"; } + + # TODO: Add support for intMatching in jsonschema + # parse port type aka. "unsignedInt16" + else if + option.type.name == "unsignedInt16" + || option.type.name == "unsignedInt" + || option.type.name == "pkcs11" + || option.type.name == "intBetween" + then + default // example // description // { type = "integer"; } + + # parse string + # TODO: parse more precise string types + else if + option.type.name == "str" + || option.type.name == "singleLineStr" + || option.type.name == "passwdEntry str" + || option.type.name == "passwdEntry path" + # return jsonschema property definition for string + then + default // example // description // { type = "string"; } + + # TODO: Add support for stringMatching in jsonschema + # parse stringMatching + else if lib.strings.hasPrefix "strMatching" option.type.name then + default // example // description // { type = "string"; } + + # TODO: Add support for separatedString in jsonschema + else if lib.strings.hasPrefix "separatedString" option.type.name then + default // example // description // { type = "string"; } + + # parse string + else if + option.type.name == "path" + # return jsonschema property definition for path + then + default // example // description // { type = "string"; } + + # parse anything + else if + option.type.name == "anything" + # return jsonschema property definition for anything + then + default // example // description // { type = allBasicTypes; } + + # parse unspecified + else if + option.type.name == "unspecified" + # return jsonschema property definition for unspecified + then + default // example // description // { type = allBasicTypes; } + + # parse raw + else if + option.type.name == "raw" + # return jsonschema property definition for raw + then + default // example // description // { type = allBasicTypes; } + + # parse enum + else if + option.type.name == "enum" + # return jsonschema property definition for enum + then + default // example // description // { enum = option.type.functor.payload; } + + # parse listOf submodule + else if + option.type.name == "listOf" && option.type.functor.wrapped.name == "submodule" + # return jsonschema property definition for listOf submodule + then + default + // example + // description + // { + type = "array"; + items = parseOptions (option.type.functor.wrapped.getSubOptions option.loc); + } + + # parse list + else if + (option.type.name == "listOf") + # return jsonschema property definition for list + then + let + nestedOption = { + type = option.type.functor.wrapped; + _type = "option"; + loc = option.loc; + }; + in + default + // example + // description + // { + type = "array"; + } + // (lib.optionalAttrs (!isExcludedOption nestedOption) { items = parseOption nestedOption; }) + + # parse list of unspecified + else if + (option.type.name == "listOf") && (option.type.functor.wrapped.name == "unspecified") + # return jsonschema property definition for list + then + default // example // description // { type = "array"; } + + # parse attrsOf submodule + else if + option.type.name == "attrsOf" && option.type.nestedTypes.elemType.name == "submodule" + # return jsonschema property definition for attrsOf submodule + then + default + // example + // description + // { + type = "object"; + additionalProperties = parseOptions (option.type.nestedTypes.elemType.getSubOptions option.loc); + } + + # parse attrs + else if + option.type.name == "attrs" + # return jsonschema property definition for attrs + then + default + // example + // description + // { + type = "object"; + additionalProperties = true; + } + + # parse attrsOf + # TODO: if nested option is excluded, the parent sould be excluded too + else if + option.type.name == "attrsOf" || option.type.name == "lazyAttrsOf" + # return jsonschema property definition for attrs + then + let + nestedOption = { + type = option.type.nestedTypes.elemType; + _type = "option"; + loc = option.loc; + }; + in + default + // example + // description + // { + type = "object"; + additionalProperties = + if !isExcludedOption nestedOption then + parseOption { + type = option.type.nestedTypes.elemType; + _type = "option"; + loc = option.loc; + } + else + false; + } + + # parse submodule + else if + option.type.name == "submodule" + # return jsonschema property definition for submodule + # then (lib.attrNames (option.type.getSubOptions option.loc).opt) + then + parseOptions (option.type.getSubOptions option.loc) + + # throw error if option type is not supported + else + notSupported option; +} diff --git a/dev/pkgs/go-jsonschema/default.nix b/dev/pkgs/go-jsonschema/default.nix new file mode 100644 index 0000000..0749263 --- /dev/null +++ b/dev/pkgs/go-jsonschema/default.nix @@ -0,0 +1,34 @@ +{ lib +, buildGoModule +, fetchFromGitHub +}: + +buildGoModule rec { + pname = "go-jsonschema"; + version = "0.16.0"; + + src = fetchFromGitHub { + owner = "omissis"; + repo = "go-jsonschema"; + rev = "v${version}"; + hash = "sha256-+CapTmg4RObK6mzjAS/EFbX4s2AtQvlFXmT119aUkZA="; + }; + + vendorHash = "sha256-gk+aKGqcHEjuYxc2o+83HA2AxU+jT7URt0N/q+uyUtA="; + + ldflags = [ + "-s" + "-w" + "-X=main.version=${version}" + "-X=main.gitCommit=${src.rev}" + "-X=main.buildTime=1970-01-01T00:00:00Z" + ]; + + subPackages = ["."]; + meta = with lib; { + description = "A tool to generate Go data types from JSON Schema definitions"; + homepage = "https://github.com/omissis/go-jsonschema"; + license = licenses.mit; + mainProgram = "go-jsonschema"; + }; +} diff --git a/flake.nix b/flake.nix index 7a443ff..338bf27 100644 --- a/flake.nix +++ b/flake.nix @@ -23,23 +23,21 @@ "aarch64-linux" ]; + imports = [ + ./dev + ]; + perSystem = { config, pkgs, ... }: { packages = { - default = pkgs.callPackage ./package.nix {}; + default = pkgs.callPackage ./package.nix { + generateSchemasProgram = config.apps.dev-generate-schemas.program; + }; docs = pkgs.callPackage ./docs/package.nix { hyprspace = config.packages.default; }; }; - - devShells.default = pkgs.mkShell { - packages = [ pkgs.go ]; - - shellHook = '' - export GOPATH="$PWD/.data/go"; - ''; - }; }; }; } diff --git a/package.nix b/package.nix index 2795147..435852f 100644 --- a/package.nix +++ b/package.nix @@ -1,4 +1,4 @@ -{ lib, buildGoModule }: +{ lib, buildGoModule, generateSchemasProgram }: let inherit (lib.fileset) toSource unions fileFilter; pname = "hyprspace"; @@ -26,6 +26,10 @@ buildGoModule { "-X github.com/hyprspace/hyprspace/cli.appVersion=${version}" ]; + postPatch = '' + ( set -x; ${generateSchemasProgram} ) + ''; + meta = { description = "A Lightweight VPN Built on top of Libp2p for Truly Distributed Networks."; homepage = "https://github.com/hyprspace/hyprspace"; diff --git a/schema/generate.go b/schema/generate.go new file mode 100644 index 0000000..cf5d1cf --- /dev/null +++ b/schema/generate.go @@ -0,0 +1,3 @@ +package schema + +//go:generate nix run ..#dev-generate-schemas