[1.13] Caused by: java.lang.NullPointerException: Cannot get config value without assigned Config object present

The full source is at: https://github.com/Fernthedev/controller-remap/tree/beta

I'm having an issue with using the config system in which I am getting this error: 

[09:28:09.236] [modloading-worker-1/ERROR] [ne.mi.fm.ja.FMLModContainer/LOADING]: Caught exception during event FMLCommonSetupEvent dispatch for modid controller-remap
java.lang.ExceptionInInitializerError: null
	at com.github.fernthedev.controllerremapmod.ControllerRemapModMain.getConfigHandler(ControllerRemapModMain.java:216) ~[main/:?] {pl:eventbus:A,pl:object_holder_definalize:A,pl:runtime_enum_extender:A,pl:capability_inject_definalize:A,pl:runtimedistcleaner:A}
	at com.github.fernthedev.controllerremapmod.core.ControllerHandler.setHandler(ControllerHandler.java:30) ~[mod-core-1.0-SNAPSHOT.jar:?] {}
	at com.github.fernthedev.controllerremapmod.ControllerRemapModMain.setup(ControllerRemapModMain.java:77) ~[main/:?] {pl:eventbus:A,pl:object_holder_definalize:A,pl:runtime_enum_extender:A,pl:capability_inject_definalize:A,pl:runtimedistcleaner:A}
	at net.minecraftforge.eventbus.EventBus.doCastFilter(EventBus.java:208) ~[eventbus-0.9.2-service.jar:?] {}
	at net.minecraftforge.eventbus.EventBus.lambda$addListener$11(EventBus.java:200) ~[eventbus-0.9.2-service.jar:?] {}
	at net.minecraftforge.eventbus.EventBus.post(EventBus.java:254) ~[eventbus-0.9.2-service.jar:?] {}
	at net.minecraftforge.fml.javafmlmod.FMLModContainer.fireEvent(FMLModContainer.java:105) ~[?:25.0] {pl:eventbus:A,pl:object_holder_definalize:A,pl:runtime_enum_extender:A,pl:capability_inject_definalize:A,pl:runtimedistcleaner:A}
	at java.util.function.Consumer.lambda$andThen$0(Consumer.java:65) ~[?:1.8.0_172] {}
	at java.util.function.Consumer.lambda$andThen$0(Consumer.java:65) ~[?:1.8.0_172] {}
	at net.minecraftforge.fml.ModContainer.transitionState(ModContainer.java:111) ~[?:?] {pl:eventbus:A,pl:object_holder_definalize:A,pl:runtime_enum_extender:A,pl:capability_inject_definalize:A,pl:runtimedistcleaner:A}
	at net.minecraftforge.fml.ModList.lambda$null$9(ModList.java:120) ~[?:?] {pl:eventbus:A,pl:object_holder_definalize:A,pl:runtime_enum_extender:A,pl:capability_inject_definalize:A,pl:runtimedistcleaner:A}
	at java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:184) ~[?:1.8.0_172] {}
	at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382) ~[?:1.8.0_172] {}
	at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481) ~[?:1.8.0_172] {}
	at java.util.stream.ForEachOps$ForEachTask.compute(ForEachOps.java:291) ~[?:1.8.0_172] {}
	at java.util.concurrent.CountedCompleter.exec(CountedCompleter.java:731) ~[?:1.8.0_172] {}
	at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ~[?:1.8.0_172] {}
	at java.util.concurrent.ForkJoinTask.doInvoke(ForkJoinTask.java:401) ~[?:1.8.0_172] {}
	at java.util.concurrent.ForkJoinTask.invoke(ForkJoinTask.java:734) ~[?:1.8.0_172] {}
	at java.util.stream.ForEachOps$ForEachOp.evaluateParallel(ForEachOps.java:160) ~[?:1.8.0_172] {}
	at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateParallel(ForEachOps.java:174) ~[?:1.8.0_172] {}
	at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:233) ~[?:1.8.0_172] {}
	at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:418) ~[?:1.8.0_172] {}
	at java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:583) ~[?:1.8.0_172] {}
	at net.minecraftforge.fml.ModList.lambda$dispatchParallelEvent$10(ModList.java:120) ~[?:?] {pl:eventbus:A,pl:object_holder_definalize:A,pl:runtime_enum_extender:A,pl:capability_inject_definalize:A,pl:runtimedistcleaner:A}
	at java.util.concurrent.ForkJoinTask$AdaptedRunnableAction.exec(ForkJoinTask.java:1386) [?:1.8.0_172] {}
	at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_172] {}
	at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) [?:1.8.0_172] {}
	at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) [?:1.8.0_172] {}
	at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:157) [?:1.8.0_172] {}
Caused by: java.lang.NullPointerException: Cannot get config value without assigned Config object present
	at com.google.common.base.Preconditions.checkNotNull(Preconditions.java:787) ~[guava-21.0.jar:?] {}
	at net.minecraftforge.common.ForgeConfigSpec$ConfigValue.get(ForgeConfigSpec.java:632) ~[?:?] {pl:eventbus:A,pl:object_holder_definalize:A,pl:runtime_enum_extender:A,pl:capability_inject_definalize:A,pl:runtimedistcleaner:A}
	at com.github.fernthedev.controllerremapmod.config.TomlSettingsConfig.parseFromConfig(TomlSettingsConfig.java:87) ~[?:?] {pl:eventbus:A,pl:object_holder_definalize:A,pl:runtime_enum_extender:A,pl:capability_inject_definalize:A,pl:runtimedistcleaner:A}
	at com.github.fernthedev.controllerremapmod.config.TomlSettingsConfig.build(TomlSettingsConfig.java:29) ~[?:?] {pl:eventbus:A,pl:object_holder_definalize:A,pl:runtime_enum_extender:A,pl:capability_inject_definalize:A,pl:runtimedistcleaner:A}
	at com.github.fernthedev.controllerremapmod.config.ConfigHandler.<init>(ConfigHandler.java:57) ~[?:?] {pl:eventbus:A,pl:object_holder_definalize:A,pl:runtime_enum_extender:A,pl:capability_inject_definalize:A,pl:runtimedistcleaner:A}
	at net.minecraftforge.common.ForgeConfigSpec$Builder.configure(ForgeConfigSpec.java:478) ~[?:?] {pl:eventbus:A,pl:object_holder_definalize:A,pl:runtime_enum_extender:A,pl:capability_inject_definalize:A,pl:runtimedistcleaner:A}
	at com.github.fernthedev.controllerremapmod.config.ConfigHandler.<clinit>(ConfigHandler.java:49) ~[?:?] {pl:eventbus:A,pl:object_holder_definalize:A,pl:runtime_enum_extender:A,pl:capability_inject_definalize:A,pl:runtimedistcleaner:A}
	... 30 more

The error seems to be called at 

sensitivity = sensitivityConfig.get();

when this check is being done: 

Preconditions.checkNotNull(spec.childConfig, "Cannot get config value without assigned Config object present");

Am I supposed to do something other than using this: (Merged some classes together)

        final Pair<ConfigHandler, ForgeConfigSpec> specPair = new ForgeConfigSpec.Builder().configure(ConfigHandler::new);
        CLIENT_SPEC = specPair.getRight();

//After all of the functions in the constructor is called
        final ModLoadingContext modLoadingContext = ModLoadingContext.get();
        modLoadingContext.registerConfig(ModConfig.Type.CLIENT,ConfigHandler.CLIENT_SPEC);


    private ConfigHandler(ForgeConfigSpec.Builder builder) {
        settingsConfig = new TomlSettingsConfig();
        settingsConfig.build(builder);
    }


	    public void build(ForgeConfigSpec.Builder builder) {
        load(builder);


        parseFromConfig(builder);
    }

	    private void load(ForgeConfigSpec.Builder builder) {


        builder.push(MAIN_CATEGORY);

        sensitivityConfig = builder.comment("The sensitivity of the controller").defineInRange("sensitivity",1,-3,-3);

        selectedMappingConfig = builder.comment("The controller mapping that should be used").define("selectedmapping",new XboxOneMapping().toJson());



        builder.pop();

        builder.build();


    }



    @Override
    public void parseFromConfig(Object configObject) {

        sensitivity = sensitivityConfig.get();

        selectedMapping = Mapping.loadFromJSON(selectedMappingConfig.get());

    }