Use configuration overlay files

You can use two configuration files when setting up a simulation. One file contains default parameter settings and an overlay file contains additional parameters or different parameter values that override the values in the default file. This topic describes how to set up the default and overlay file and flatten them into a single file before running a simulation.

Overlay files allow you to easily separate a subset of parameters that are of particular interest from the rest of the parameters needed to run a simulation. You can easily modify the parameters in the overlay file without needing to maintain a complete configuration or campaign file. This can be especially helpful when you want to experiment with the values set in certain parameters of interest without modifying the rest of the settings. You can have one default file and many different overlay files for different configurations. It also allows you to easily update the default values across multiple simulations.

In addition to being used for model experimentation, overlay files are used when testing the software functionality after making source code changes. If you run the EMOD regression tests using regression_test.py, configuration files will be flattened as part of those tests. However, this may take several hours if run locally. More guidance on modifying the EMOD source code is in the “Advance the model” section.

Follow the steps below to quickly flatten configuration files.

  1. Create the default configuration file in JSON. You may, though it is not required, organize the parameters into logical categories of nested JSON objects to make managing the parameters easier. See Configuration parameters for a complete list of all parameters that are available. See the example default configuration file below.

    {
        "parameters": {
            "CAMPAIGNS": {
                "Campaign_Filename": "campaign.json",
                "Enable_Interventions": 1,
                "Listed_Events": [],
                "PKPD_Model": "FIXED_DURATION_CONSTANT_EFFECT"
            },
            "CLIMATE": {
                "Climate_Model": "CLIMATE_OFF"
            },
            "DEMOGRAPHICS": {
                "Age_Initialization_Distribution_Type": "DISTRIBUTION_SIMPLE",
                "Base_Population_Scale_Factor": 1,
                "Birth_Rate_Dependence": "DEMOGRAPHIC_DEP_RATE",
                "Birth_Rate_Time_Dependence": "NONE",
                "Demographics_Filenames": ["NO_DEFAULT_DEMOGRAPHICS"],
                "Default_Geography_Initial_Node_Population": 1000,
                "Default_Geography_Torus_Size": 10,
                "Enable_Aging": 1,
                "Enable_Birth": 1,
                "Enable_Demographics_Birth": 0,
                "Enable_Demographics_Gender": 1,
                "Enable_Demographics_Builtin": 0,
                "Enable_Demographics_Other": 0,
                "Enable_Demographics_Reporting": 0,
                "Enable_Vital_Dynamics": 1,
                "Minimum_Adult_Age_Years": 15,
                "IMMUNITY": {
                    "Acquisition_Blocking_Immunity_Decay_Rate": 0.1,
                    "Acquisition_Blocking_Immunity_Duration_Before_Decay": 60,
                    "Enable_Immune_Decay": 1,
                    "Enable_Immunity": 1,
                    "Immunity_Acquisition_Factor": 0,
                    "Immunity_Transmission_Factor": 0,
                    "Immunity_Initialization_Distribution_Type": "DISTRIBUTION_OFF",
                    "Susceptibility_Scale_Type": "CONSTANT_SUSCEPTIBILITY",
                    "Transmission_Blocking_Immunity_Decay_Rate": 0.1,
                    "Transmission_Blocking_Immunity_Duration_Before_Decay": 60
                },
                "MORTALITY": {
                    "Base_Mortality": 0,
                    "Enable_Disease_Mortality": 0,
                    "Death_Rate_Dependence": "NONDISEASE_MORTALITY_BY_AGE_AND_GENDER",
                    "Immunity_Mortality_Factor": 0,
                    "Mortality_Blocking_Immunity_Decay_Rate": 0.001,
                    "Mortality_Blocking_Immunity_Duration_Before_Decay": 60,
                    "Mortality_Time_Course": "DAILY_MORTALITY"
                },
                "Population_Density_C50": 30,
                "Population_Scale_Type": "USE_INPUT_FILE",
                "SAMPLING": {
                    "Base_Individual_Sample_Rate": 1,
                    "Individual_Sampling_Type": "TRACK_ALL",
                    "Max_Node_Population_Samples": 40,
                    "Sample_Rate_0_18mo": 1,
                    "Sample_Rate_10_14": 1,
                    "Sample_Rate_15_19": 1,
                    "Sample_Rate_18mo_4yr": 1,
                    "Sample_Rate_20_Plus": 1,
                    "Sample_Rate_5_9": 1,
                    "Sample_Rate_Birth": 2
                }
            },
            "DISEASE": {
                "Animal_Reservoir_Type": "NO_ZOONOSIS",
                "Enable_Superinfection": 0,
                "INCUBATION": {
                    "Base_Incubation_Period": 3,
                    "Incubation_Period_Distribution": "FIXED_DURATION"
                },
                "INFECTIOUSNESS": {
                    "Base_Infectious_Period": 7,
                    "Base_Infectivity": 0.3,
                    "Infectious_Period_Distribution": "EXPONENTIAL_DURATION",
                    "Infectivity_Scale_Type": "CONSTANT_INFECTIVITY",
                    "Population_Density_Infectivity_Correction": "CONSTANT_INFECTIVITY"
                },
                "Infection_Updates_Per_Timestep": 1,
                "Max_Individual_Infections": 1,
                "TRANSMISSION": {
                    "Enable_Maternal_Infection_Transmission": 0,
                    "Maternal_Transmission_Probability": 0
                }
            },
            "FUDGE_FACTORS": {
                "x_Air_Migration": 1,
                "x_Birth": 1,
                "x_Local_Migration": 1,
                "x_Other_Mortality": 1,
                "x_Population_Immunity": 1,
                "x_Regional_Migration": 1,
                "x_Sea_Migration": 1,
                "x_Temporary_Larval_Habitat": 1
            },
            "HPC": {
                "Job_Node_Groups": "Chassis08",
                "Job_Priority": "BELOWNORMAL",
                "Load_Balance_Filename": "",
                "Local_Simulation": 0,
                "Num_Cores": 1
            },
            "INTRANODE_TRANSMISSION": {
                "Enable_Default_Shedding_Function": 1,
                "Enable_Heterogeneous_Intranode_Transmission": 0
            },
            "MIGRATION": {
                "Migration_Model": "NO_MIGRATION"
            },
            "OUTPUT": {
                "Custom_Reports_Filename": "NoCustomReports",
                "Report_Event_Recorder": 0,
                "Enable_Default_Reporting": 1,
                "Enable_Property_Output": 0,
                "Enable_Spatial_Output": 0
            },
            "POLIO": {},
            "PRIMARY": {
                "Config_Name": "DEFAULT_CONFIG_NAME_SHOULD_BE_SET",
                "ENUMS": {
                    "Simulation_Type": "GENERIC_SIM"
                },
                "Geography": "DEFAULT_GEOGRAPHY_SHOULD_BE_SET",
                "Node_Grid_Size": 0.042,
                "Run_Number": 0,
                "Simulation_Duration": 365,
                "Simulation_Timestep": 1,
                "Start_Time": 0,
                "Enable_Absolute_Time": "NO"
    
            },
            "SERIALIZATION": {
                "Burnin_Cache_Mode": "none",
                "Burnin_Cache_Period": 0,
                "Burnin_Name": "",
                "Serialization_Test_Cycles": 0
            },
            "STRAIN_TRACKING": {
                "Number_Basestrains": 1,
                "Number_Substrains": 1
            }
        }
    }
    
  2. Create the overlay configuration file in JSON. This file must include the parameter Default_Config_Path, set to the path to the default configuration file, relative to the location of the flatten_config.py script in the EMOD Regression folder. Again, you may organize the parameters into logical categories if you desire. See the example overlay configuration file below.

    {
         "Default_Config_Path": "defaults/generic-default-config.json",
         "parameters": {
              "DEMOGRAPHICS": {
                  "Enable_Demographics_Builtin": 0,
                  "Birth_Rate_Dependence": "POPULATION_DEP_RATE",
                  "Death_Rate_Dependence" : "NONDISEASE_MORTALITY_OFF",
                  "Enable_Vital_Dynamics": 0,
                  "Sample_Rate_Birth": 1,
                  "Enable_Demographics_Reporting": 1
              },
              "DISEASE": {
                  "Base_Incubation_Period": 0,
                  "Base_Infectious_Period": 4,
                  "Base_Infectivity": 3.5,
                  "Enable_Immune_Decay": 0
              },
              "PRIMARY": {
                   "Config_Name": "00_Generic_DEFAULT",
                   "Demographics_Filenames": ["demographics.json"],
                   "Geography": "",
                   "Run_Number": 1,
                   "Simulation_Duration": 90
              }
         }
    }
    
  3. In a Command Prompt window, navigate to the Regression folder.

  4. Run the flatten_config.py script, providing the relative path to the overlay file:

    python flatten_config.py experiment/param_overlay.json
    
  5. Open the resulting config.json file in the same folder as param_overlay.json and see that it has been flattened into a single layer with all parameters listed alphabetically and any logical categories removed. Eradication.exe will not accept a configuration file with nested JSON objects.

    {
        "parameters": {
            "Acquisition_Blocking_Immunity_Decay_Rate": 0.1,
            "Acquisition_Blocking_Immunity_Duration_Before_Decay": 60,
            "Age_Initialization_Distribution_Type": "DISTRIBUTION_SIMPLE",
            "Animal_Reservoir_Type": "NO_ZOONOSIS",
            "Base_Incubation_Period": 0,
            "Base_Individual_Sample_Rate": 1,
            "Base_Infectious_Period": 4,
            "Base_Infectivity": 3.5,
            "Base_Mortality": 0,
            "Base_Population_Scale_Factor": 1,
            "Birth_Rate_Dependence": "POPULATION_DEP_RATE",
            "Birth_Rate_Time_Dependence": "NONE",
            "Burnin_Cache_Mode": "none",
            "Burnin_Cache_Period": 0,
            "Burnin_Name": "",
            "Campaign_Filename": "campaign.json",
            "Climate_Model": "CLIMATE_OFF",
            "Config_Name": "00_DEFAULT",
            "Custom_Reports_Filename": "NoCustomReports",
            "Death_Rate_Dependence": "NONDISEASE_MORTALITY_OFF",
            "Default_Geography_Initial_Node_Population": 1000,
            "Default_Geography_Torus_Size": 10,
            "Demographics_Filenames": [
                "../00_Default/demographics.json"
            ],
            "Enable_Absolute_Time": "NO",
            "Enable_Aging": 1,
            "Enable_Birth": 1,
            "Enable_Default_Reporting": 1,
            "Enable_Default_Shedding_Function": 1,
            "Enable_Demographics_Birth": 0,
            "Enable_Demographics_Builtin": 0,
            "Enable_Demographics_Gender": 1,
            "Enable_Demographics_Other": 0,
            "Enable_Demographics_Reporting": 1,
            "Enable_Disease_Mortality": 0,
            "Enable_Heterogeneous_Intranode_Transmission": 0,
            "Enable_Immune_Decay": 0,
            "Enable_Immunity": 1,
            "Enable_Interventions": 1,
            "Enable_Maternal_Infection_Transmission": 0,
            "Enable_Property_Output": 0,
            "Enable_Spatial_Output": 0,
            "Enable_Superinfection": 0,
            "Enable_Vital_Dynamics": 0,
            "Geography": "",
            "Immunity_Acquisition_Factor": 0,
            "Immunity_Initialization_Distribution_Type": "DISTRIBUTION_OFF",
            "Immunity_Mortality_Factor": 0,
            "Immunity_Transmission_Factor": 0,
            "Incubation_Period_Distribution": "FIXED_DURATION",
            "Individual_Sampling_Type": "TRACK_ALL",
            "Infection_Updates_Per_Timestep": 1,
            "Infectious_Period_Distribution": "EXPONENTIAL_DURATION",
            "Infectivity_Scale_Type": "CONSTANT_INFECTIVITY",
            "Job_Node_Groups": "Chassis08",
            "Job_Priority": "BELOWNORMAL",
            "Listed_Events": [],
            "Load_Balance_Filename": "",
            "Local_Simulation": 0,
            "Maternal_Transmission_Probability": 0,
            "Max_Individual_Infections": 1,
            "Max_Node_Population_Samples": 40,
            "Migration_Model": "NO_MIGRATION",
            "Minimum_Adult_Age_Years": 15,
            "Mortality_Blocking_Immunity_Decay_Rate": 0.001,
            "Mortality_Blocking_Immunity_Duration_Before_Decay": 60,
            "Mortality_Time_Course": "DAILY_MORTALITY",
            "Node_Grid_Size": 0.042,
            "Num_Cores": 1,
            "Number_Basestrains": 1,
            "Number_Substrains": 1,
            "PKPD_Model": "FIXED_DURATION_CONSTANT_EFFECT",
            "Population_Density_C50": 30,
            "Population_Density_Infectivity_Correction": "CONSTANT_INFECTIVITY",
            "Population_Scale_Type": "USE_INPUT_FILE",
            "Report_Event_Recorder": 0,
            "Run_Number": 1,
            "Sample_Rate_0_18mo": 1,
            "Sample_Rate_10_14": 1,
            "Sample_Rate_15_19": 1,
            "Sample_Rate_18mo_4yr": 1,
            "Sample_Rate_20_Plus": 1,
            "Sample_Rate_5_9": 1,
            "Sample_Rate_Birth": 1,
            "Serialization_Test_Cycles": 0,
            "Simulation_Duration": 90,
            "Simulation_Timestep": 1,
            "Simulation_Type": "GENERIC_SIM",
            "Start_Time": 0,
            "Susceptibility_Scale_Type": "CONSTANT_SUSCEPTIBILITY",
            "Transmission_Blocking_Immunity_Decay_Rate": 0.1,
            "Transmission_Blocking_Immunity_Duration_Before_Decay": 60,
            "x_Air_Migration": 1,
            "x_Birth": 1,
            "x_Local_Migration": 1,
            "x_Other_Mortality": 1,
            "x_Population_Immunity": 1,
            "x_Regional_Migration": 1,
            "x_Sea_Migration": 1,
            "x_Temporary_Larval_Habitat": 1
        }
    }
    

Note

These same steps can also be used to flatted a single configuration file that has been hierarchically organized into logical categories.