Merge pull request #389 from PyPSA/meta

store config and wildcards in n.meta

scripts/_helpers.py

@@ -28,7 +28,7 @@ def configure_logging(snakemake, skip_handlers=False):
 
     import logging
 
-    kwargs = snakemake.config.get('logging', dict())
+    kwargs = snakemake.config.get('logging', dict()).copy()
     kwargs.setdefault("level", "INFO")
 
     if skip_handlers is False:

scripts/add_electricity.py

@@ -635,4 +635,5 @@ if __name__ == "__main__":
 
     add_nice_carrier_names(n, snakemake.config)
 
+    n.meta = snakemake.config
     n.export_to_netcdf(snakemake.output[0])

scripts/add_extra_components.py

@@ -193,7 +193,7 @@ def attach_hydrogen_pipelines(n, costs, elec_opts):
 if __name__ == "__main__":
     if 'snakemake' not in globals():
         from _helpers import mock_snakemake
-        snakemake = mock_snakemake('add_extra_components', network='elec',
+        snakemake = mock_snakemake('add_extra_components',
                                   simpl='', clusters=5)
     configure_logging(snakemake)
 
@@ -209,4 +209,5 @@ if __name__ == "__main__":
 
     add_nice_carrier_names(n, snakemake.config)
 
+    n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output[0])

scripts/base_network.py

@@ -596,4 +596,5 @@ if __name__ == "__main__":
                      snakemake.input.links_p_nom, snakemake.input.links_tyndp, snakemake.input.europe_shape, snakemake.input.country_shapes, snakemake.input.offshore_shapes,
                      snakemake.input.parameter_corrections, snakemake.config)
 
+    n.meta = snakemake.config
     n.export_to_netcdf(snakemake.output[0])

scripts/cluster_network.py

@@ -382,7 +382,7 @@ def plot_busmap_for_n_clusters(n, n_clusters, fn=None):
 if __name__ == "__main__":
     if 'snakemake' not in globals():
         from _helpers import mock_snakemake
-        snakemake = mock_snakemake('cluster_network', network='elec', simpl='', clusters='5')
+        snakemake = mock_snakemake('cluster_network', simpl='', clusters='5')
     configure_logging(snakemake)
 
     n = pypsa.Network(snakemake.input.network)
@@ -444,6 +444,7 @@ if __name__ == "__main__":
 
     update_p_nom_max(clustering.network)
 
+    clustering.network.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     clustering.network.export_to_netcdf(snakemake.output.network)
     for attr in ('busmap', 'linemap'): #also available: linemap_positive, linemap_negative
         getattr(clustering, attr).to_csv(snakemake.output[attr])

scripts/make_summary.py

@@ -427,7 +427,7 @@ def to_csv(dfs, dir):
 if __name__ == "__main__":
     if 'snakemake' not in globals():
         from _helpers import mock_snakemake
-        snakemake = mock_snakemake('make_summary', network='elec', simpl='',
+        snakemake = mock_snakemake('make_summary', simpl='',
                            clusters='5', ll='copt', opts='Co2L-24H', country='all')
         network_dir = os.path.join('..', 'results', 'networks')
     else:

scripts/plot_network.py

@@ -251,7 +251,7 @@ def plot_total_cost_bar(n, opts, ax=None):
 if __name__ == "__main__":
     if 'snakemake' not in globals():
         from _helpers import mock_snakemake
-        snakemake = mock_snakemake('plot_network', network='elec', simpl='',
+        snakemake = mock_snakemake('plot_network', simpl='',
                                   clusters='5', ll='copt', opts='Co2L-24H',
                                   attr='p_nom', ext="pdf")
     configure_logging(snakemake)

scripts/plot_p_nom_max.py

@@ -48,7 +48,7 @@ def cum_p_nom_max(net, tech, country=None):
 if __name__ == "__main__":
     if 'snakemake' not in globals():
         from _helpers import mock_snakemake
-        snakemake = mock_snakemake('plot_p_nom_max', network='elec', simpl='',
+        snakemake = mock_snakemake('plot_p_nom_max', simpl='',
                                   techs='solar,onwind,offwind-dc', ext='png',
                                   clusts= '5,full', country= 'all')
     configure_logging(snakemake)

scripts/plot_summary.py

@@ -165,7 +165,7 @@ def plot_energy(infn, config, fn=None):
 if __name__ == "__main__":
     if 'snakemake' not in globals():
         from _helpers import mock_snakemake
-        snakemake = mock_snakemake('plot_summary', summary='energy', network='elec',
+        snakemake = mock_snakemake('plot_summary', summary='energy',
                                   simpl='', clusters=5, ll='copt', opts='Co2L-24H',
                                   attr='', ext='png', country='all')
     configure_logging(snakemake)

scripts/prepare_links_p_nom.py

@@ -60,7 +60,7 @@ def extract_coordinates(s):
 if __name__ == "__main__":
     if 'snakemake' not in globals():
         from _helpers import mock_snakemake #rule must be enabled in config
-        snakemake = mock_snakemake('prepare_links_p_nom', simpl='', network='elec')
+        snakemake = mock_snakemake('prepare_links_p_nom', simpl='')
     configure_logging(snakemake)
 
     links_p_nom = pd.read_html('https://en.wikipedia.org/wiki/List_of_HVDC_projects', header=0, match="SwePol")[0]

scripts/prepare_network.py

@@ -213,7 +213,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf):
 if __name__ == "__main__":
     if 'snakemake' not in globals():
         from _helpers import mock_snakemake
-        snakemake = mock_snakemake('prepare_network', network='elec', simpl='',
+        snakemake = mock_snakemake('prepare_network', simpl='',
                                   clusters='40', ll='v0.3', opts='Co2L-24H')
     configure_logging(snakemake)
 
@@ -301,4 +301,5 @@ if __name__ == "__main__":
     elif "ATKc" in opts:
         enforce_autarky(n, only_crossborder=True)
 
+    n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output[0])

scripts/simplify_network.py

@@ -391,7 +391,7 @@ def cluster(n, n_clusters, config, algorithm="hac", feature=None, aggregation_st
 if __name__ == "__main__":
     if 'snakemake' not in globals():
         from _helpers import mock_snakemake
-        snakemake = mock_snakemake('simplify_network', simpl='', network='elec')
+        snakemake = mock_snakemake('simplify_network', simpl='f')
     configure_logging(snakemake)
 
     n = pypsa.Network(snakemake.input.network)
@@ -449,6 +449,7 @@ if __name__ == "__main__":
 
     update_p_nom_max(n)
 
+    n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output.network)
 
     busmap_s = reduce(lambda x, y: x.map(y), busmaps[1:], busmaps[0])

scripts/solve_network.py

@@ -349,7 +349,7 @@ def solve_network(n, config, opts='', **kwargs):
 if __name__ == "__main__":
     if 'snakemake' not in globals():
         from _helpers import mock_snakemake
-        snakemake = mock_snakemake('solve_network', network='elec', simpl='',
+        snakemake = mock_snakemake('solve_network', simpl='',
                                   clusters='5', ll='copt', opts='Co2L-BAU-CCL-24H')
     configure_logging(snakemake)
 
@@ -365,6 +365,7 @@ if __name__ == "__main__":
         n = prepare_network(n, solve_opts)
         n = solve_network(n, snakemake.config, opts, solver_dir=tmpdir,
                           solver_logfile=snakemake.log.solver)
+        n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
         n.export_to_netcdf(snakemake.output[0])
 
     logger.info("Maximum memory usage: {}".format(mem.mem_usage))

scripts/solve_operations_network.py

@@ -96,7 +96,7 @@ def set_parameters_from_optimized(n, n_optim):
 if __name__ == "__main__":
     if 'snakemake' not in globals():
         from _helpers import mock_snakemake
-        snakemake = mock_snakemake('solve_operations_network', network='elec',
+        snakemake = mock_snakemake('solve_operations_network',
                                   simpl='', clusters='5', ll='copt', opts='Co2L-BAU-24H')
     configure_logging(snakemake)
 
@@ -117,6 +117,7 @@ if __name__ == "__main__":
         n = prepare_network(n, snakemake.config['solving']['options'])
         n = solve_network(n, snakemake.config, opts, solver_dir=tmpdir,
                           solver_logfile=snakemake.log.solver)
+        n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
         n.export_to_netcdf(snakemake.output[0])
 
     logger.info("Maximum memory usage: {}".format(mem.mem_usage))