diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 3f1edbd8..01156924 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -2,8 +2,325 @@ # # SPDX-License-Identifier: CC0-1.0 -# Exclude pre-commit applications -5d1ef8a64055a039aa4a0834d2d26fe7752fe9a0 -92080b1cd2ca5f123158571481722767b99c2b27 -13769f90af4500948b0376d57df4cceaa13e78b5 +# Note +# Needs to be setup via +# git config blame.ignoreRevsFile .git-blame-ignore-revs + +# Custom commits 9865a970893d9e515786f33c629b14f71645bf1e + +# pre-commit commits +4a1933f49aaf2ed530d13100efbd34b8f32e103e +5168b3fe0b66230e9aadc5a843a27d921b7ed956 +47889d728f50b5a200c41282973740926a294aa4 +1da76dd1aeb4445f39b87e3de40bb08b6a7d46cd +288ccbd5133a31d1a9abbf4ef29e1666214a957a +b29b9e6c5d91cb631fdfee8c7d2220d03278bb82 +6722e584ba179a99ae5684c2fff93dab3d418934 +b890149d39aa05cd91d52d792a2b395e5aae06cb +a352cc346b57912cac9734148c4b6e70648fc23c +5dc2e6862b657266cef411824a46f5e0d7b6dfa2 +985aecb0c4629c66337b86f3b31fd711099f3a72 +c311f7bdf83b3b8d80c5ba103753da66daf967b0 +b529ab1d3c97ee5e30c753baa37ef83987c68e68 +a9d0f295fb5aebe44e86522f2c0c20a133796049 +1d709913edd92fdb2569e9a5e1c0f69c9e29ac4a +f7dc8a7f22e3ec7f7a934acc093590504ac00591 +5364e7dfcc37ea5e100ce2f471f74e12198b30f9 +0fb68d11d844779d2137bfa897f87bb9249eee64 +85a2e97f1ed542cd855ec01e4e70ef63a7f73fbb +1b1b4528bf785dc2262c4588b06089ce1e47d9fd +2d2b0338cc23c534e1d4ffca58c0e8cd01cf79a0 +c66549c577469e60d1d406f643c154aba0d6060f +3e93c14dd4cac4cdba266315b29ed8ef531885b8 +105cadaf7b83a1a549482c3f39f33e2285a0894c +29f6c9507f01c5a9cca7f8435c6260143f60dd6f +b313b3940c6464ca2e4915588c556559832e9b22 +bc786079268b6768bc7d832be7e58deb6679c4f5 +da15ab3ac25ea6c85dc60d100e239cd666c81bbd +7436634bf87a42f807ccecfb1ede9ac43da62b86 +ba93a404efecbb01b72692d796a4b30170d8e330 +ed6fa8f32c8a7715fc6604211735bad0ed1aed45 +1c321b72cf57b0d24307cff96e3967d0bbcda2a1 +f8b33e8eb28982b424e3637c96fa53f44bb2b8cb +6965ce31ccca19b88ded25b0102973b87b333b9f +73971199d2f6416f219281b1823ecf39784ebffd +7d0b775ca9c84ead87cafc547dca5b2d28f6c824 +f3ad1a36c7a12ca70b5974e11921e801cf17fdf9 +6ba598f426f5bb920f52c9f960be1a68cf1408d4 +435350e07b626e98601f015de9c0ba8d45c7a65c +dda0c98b2833adb21424cdfd57d9af81774eadf4 +0bbda371d4a789a2e70a6d074a3d426842672450 +a6ea15ea4a811a4874209c780c5ea42958092d4c +ad9da6118757ccfc44dc92cac00b7748fe7f383a +090792c9ec7583cf2f2f8ad7ee839da636f7a411 +721c64a5bda180301e0bead5e5b8b21c4cf8dbb0 +d2968230bbdfa07324c37b383f2a16211fdb7780 +ca91c02bf77e5d33c787d2e4b78f65e2c694dfce +3da649c13fbd48ed9d7d7c602bb0bf436eba35e2 +8c465dbe3cbadb54a61336b9f75b94d454fdea97 +8aa6d98f5635064262b43bac526674b3fe4ac631 +f4f4c1bf375eff2cedf15027afd889db1041a527 +7b584f16bbb467f24ac44bba7fbdd497581f2394 +a805f5cfb3f6873144b6f5cfc20250535d102b29 +3549e6843d1362374aa45997502735b8b6df00d6 +9607eb0e6bd2171914741a68eaf0da1798cb56fa +ff0093c8fed0b3b778c8a175c7a9d61cb80c75c6 +fb08d16be95094c0c16a53edbaee495cee53e60e +fa935be5cb2561d0c63673619a6c89b0c187df9f +4242a0841ce56c34e3c25ceddb7f7c01984e966c +8dfab454537fa3892cce1a939625e671aca3091e +9182d6d667df59f435cd7d5d9c9f651665cd3df8 +60653de9f628e911a50c06400614d72e4c527045 +ace51b9e5547ec9c23730c832f4d9a7b5212d5a2 +d713e3c52b2e77b30816a3b829faee3fc3ebd381 +ac2322cd16eff8f9d056f7021329d9e6b40b324d +e68743ffa6601bcbdf02025372841211e58bac2e +f4422f13614757ea3c68f5b378fdaa47b7bfd346 +1fb90da743e54a096f2c193b615cdbc8f7a059d5 +831f0779b3f0f333dc4527d473dc725c3d1c3bed +cc3d7a4cbb29d6ec1f3f70cfcaabfe8a053fd2ba +de244414df2f36298ed5f3b36adab2b967940b1b +b8d57a6566c410e871fe4ca97096e75dc060fb0e +4fa504b0dbd60aa0b410d2bffba4c68e525159ff +b45df1724bda1611c5792f0775b6496e36d88638 +78b184ad0f457b4ba3ef9dd44306802e9c22f670 +abe06a68fbc28c3d71c459eb66ed7daf5b8d42b0 +e0b6ebd174a9f5f55bb462c8650e0754a1121a9d +1a883debb5170e754f83ace5c5cf598453041019 +17105b81256a5364739a4a2f4c3e865a3ba3fc95 +4ff06046fc72b316a7fbc01ad8295d2546db322f +e838b63ded1eb2084964639c132b24c91f1bc123 +3e411fe79d9a8d707053ae9b37fd6c17cd81ee0d +39890630efc3bc1b2356ef274faf9fbebf92ce82 +d2f2d4ab0b6a77446573b6c85ad1927861804d69 +d6288d3d4dfed4bb2ce58fa4053eabf0123e3404 +a56bd830f8b5808189020414efcae11a7bfc184a +5fcfafe971cdb0b7ccbd31fa4a613d989f499b0d +30ccde5b908aa680ea858fcdb08b09b593e901cf +9843397b5a4effda6418f85fcf5c03291435804b +bdc36dc5da5834275450119ea823916adb247b91 +58ac6629fee01b6f06ec3ea4b2c86bc5cba53d5d +025f48c0c2ba402f61f9cf2d0cf8c4656beae1ac +757fbcc464c4c04b94d2a29cd325963f0a35aa8a +58b012e03dbc7da4701f77f390827fa10b13c680 +35f1c06f75a4a4e540277bd760c96ec7b3bd4083 +213c4b025e9b53a8ed56a4745e0410baa86dbb2f +e07f0e5c2b5bb34e8fcfb3b127ac9e30109beedc +aaa587d34f84dc22f7662d7f5855ae1e1f2161fa +446b0b37227111e49fa3e4e566b68b031f558e28 +21a223262f145476065e2dd9dd30943bfd24594e +68421e6aabbe676982b448ab2759b8f1f7691d31 +0bac08934aa701e89887afe8085edf715deda22a +7ebf3b186c5c5d9840208b755e955813e5d8f60f +5ef317ac399ac86efcd6b6c907f1a8c8426c757e +dc90398b12bb49788f5f588e466e656f8737a893 +33edb0b5166f3f0568a08832b01dfc2709be03f5 +6e630a85e10e1a8218d51da9c7bcdc0cdbb832de +bf5c323c276235034e78e6cd26bfb91a32fc388f +29cda7042b2b506001d2955059ca0d6ce75f1d0c +7b1600164fa292eb1ba2288340340deaf1cb3f59 +0534f574e9ef15b59d475991080fd2c47c8cd2c0 +48832874171601e2eec0bf2b7512ae5a4ce718e1 +817995c8a25dfc2f152c817fc383944860db0939 +60493fc55829ddc95bd8d55d35b0f505cef5f624 +045eeba4cfc17500c9740706c60c9f61fc4a3a68 +d7051e7f66eb3bdbe0f790ea4513cbf01133b09a +4606cb131b292c02e95b2af3583e7df48561fcb9 +38d587944b8625cfb208f6cc0c5046b1a3ee97d6 +fdb63bc6ca4c3aa332104d26bca1c0a5d5c546c1 +2b2bad392f6c83771472d93ca2df597608ea6b26 +fb5b10780536f1c3f337f7cbba5da185876795ba +777899f686b4641d9fc52c09b6c9db6a30be4e1d +815b8283115b427189f7fe364c6ca1070427e2d8 +6714858e177ca9a040862906e9b326ce22ecca6a +2678fdef993eb2c0d1b325c4e260040ed2e19174 +e580ac85d962e0ef9d24716125655a0e59712f8e +f494dd85b969f9491a2d9bf81ea98008452440a7 +27ee2666be6300ac6a07f2e74c5ec6e1d49e2f64 +876a28b688719273cc8fd9531845ae78c0e7edbe +fba320bfa7ae05a86e567426848577d42a25b337 +74cf849ac4b1c054ef32cf720ee95ada52c8ef70 +de3b6c9573f075d2bdb93ea0af852c8b31ca03b3 +125c40601305eea314786ca73ff6d82e57b2f060 +fec641ce0a9d1cdc66f6a035b376cec44abed8c6 +3a474af71fcd5f4793e7ab64f90bce4c295fc173 +20b847c62743690009a8ff3e18832ec02eb65c6b +5ab9d149ba6d145b3dd81357a0d368099c93371d +146fb170d8c29932faa1d50b31c8d30dd7491403 +dfc2f060330ed5f270ec32be4bac3f04192dabdb +f77f84af6ad3d588a746da878cc95d8035d7bf22 +ea3fdb6c15bbc0d70308970dad6f7784c970c4b3 +f0eae99a07c022d7b1e6ee37b12133d21c904c5e +aae5ffb6fdca9363c60124bf23f90451996a74fd +e3ba0e50a7dbf628aec9f08f72378d5cb6c9bf27 +429b4ca6809024780316e8465dbda56a2d0aad67 +01c5e11f9fe6277dc3c2fae76e9979c0e13fffa4 +92ccb517f3ecb8ceaf0a6a58c97d27ae055cd40a +71dbe2aaf27057faae8cbd4ef6a3bd5871ba13a3 +4c66908b74b5c3d2799673de759070968b56929d +649e0a2a78c8cffd7cc42d0f7b493c25c6180221 +5b2efe313ff551e06d876b9f9e942df6b18e88d1 +a892f0c0eccaf1026a9bb999f018039a576e5f9b +6b100d4b88ebe77204e9c10b816bcd4e6d506979 +a8dde158cb9b6efe76922d50b8a1edd07cd989de +ff209bbd76a5b37ad0529ce4e402a9635e26ed23 +4f28dfd4fdda639151a8b7558f16fd058e6848ee +4779d076dc44145886627ee63aa36ea9b51bcc65 +b9128002bbdcd151fbbcb6c1e184aa384e2f66d7 +0e97d39c4c1fbc6ae4619e24550baff3d5e0addf +a31b421c1286884bba658f3d2cab6cd5498de34c +f24bd461736162654d186543556fcb91dcfb2674 +21148e3cc5048734e21cd941384beaa2c87b3ae0 +d10779ca3485ec524eed1251c6fef31a64f43437 +230c1a327c39ece654ac3c293b2509c314835896 +326aeb682b06c4f2704a86d7aeb2a9206da988e7 +7e290d3e5281c6580efc25af4d33ed402f74186a +f054180a6218b327ff65d90f7947d057a3f6fe32 +d2f8dc25fd3fa7497d3c3c8b6a7d820f593fc09b +9765b24c1cdc0f82ec0d800be168704c6d1e97d0 +d75b0ae8abfea19c1da69085af58003a653ac547 +1e87cf0eebd447d98676f361829bd05b898013dc +9424ec1bb8ef599ed7b766831b3823718b388219 +45e5ab15c2d11ecd9ad162973d5b3fc5c1c2fe43 +d2624f80707835a451ad74c352482dc8bb89578d +550256890171b0976a2f4d18468247ae035db018 +b2c19eda40e6409257e616ccc9995ed3063733bd +668ec9efad33af2897484f9bba8dc0c1352603aa +3eed3410448c93426af03e2fce23aaa7db986e73 +de605e2b6b089c8aa8c0e429cc343913cc2c622b +0e80244b521bd06fc2aded46c56f268c50d08f0b +3187081ec0473ae6d7bd8b62ef7cf8b88fab78c9 +64cbfd673c0a6022d711fb0d4e10bca6f2683b94 +bdaa646ed62b07c9d7fadd095d91ff5a6b9d5391 +b8dee184a9a9cb2ac16f24b18dd14cbfac6b3a35 +b91a7b9c514f6a380f7cea758325156f095cf8c7 +a77f0f6ab17e7c367fd132b3f423155a2e877e48 +afda8ee1f65d29e523872bbf9974011254b14289 +69132bd6e24a72f2caa0a17647ab68ea02219858 +dadc372ecd629cdeb708727e5abce3b7179eb307 +0d096d7388e90c3fe4fb931d3a2d9147169e010e +eed52d04aeec6174fb18b25ca644e78cc8dda1ed +04202690936201d18ff0e4f58f9d06da9c345589 +ac04abb98c1e021a0ce8aa74ed331f4156a7a8e3 +c32d5249e1152f620c60f1d158bb3b2f065abf55 +5a31b6f5b5f199ad865e1e2dc74ee6728cbe9e26 +7530c42fbe39d2dfb0879e648b6c02c2d2737a48 +ade7e17d5770ceb8dbbc8a616ef2901dc4e07f02 +7c00f9d4517e2bf0179c4ab49e7582a1e17f6c17 +54151d384a7747b72aac8af7b697196e1144f467 +d527797dc3edee0bfdf9656a2435751b93843035 +f970ba0fd0b5d48758eae7105b121a09091bfc9e +74be5261fc7a03dfd72be685453a8452b1d6cb9b +3f3e73a0ecf5654f9e6a30bfb308d61d47b9c82a +854b3b773c5e56804214e3ce7e470459ebf46e55 +615684d12456b608cf797eb7c19cfe34d7e10464 +9dcdb8037e28bdf0ec0b6f0cfb59bd08e996c75b +fa37b4c5d3d0fff9cf3da12fdc1054a8b77814fc +54e813ffefe37985cf7004c21416eb23d6b49f53 +aa7a2c823acec4d805f08e92e34e48f740cd32a5 +5482c27e8b4d7df34c0bc3bd00be0fc385999843 +fc36efaebee23a0bb76d6b9d51c3351af611764b +a03574ba14fa69d8199455925d8d46cfd3f91c3f +4ecc7ae43904b392059fcc60ce56b38ef6f3b78a +6492b1a195e3e8868bce18b3ac13f4f0602820b3 +07add5a10d5b88e138daaaa9c1558da0690db340 +0664f1050f7a7d79740f6ceb4490719d3d98d017 +39c963dd381496c5d97f7016900f376483e02e72 +ac264282dab1fd2d5d930f8aa591564f91c1af18 +e4651ce711d2bd1f53736e3ec0abf77c4b137c04 +d84c7ebf6acb0c7d012f7fe3aa40825e47e02fc3 +a9b0d349f24484aa8a885732eef943c140ad8222 +a8f69821dc5b79c826ae3da09b256b41422abe96 +c0088b7923cc47413dd80797e14e1e4bf24ca2be +cc6bae282f0dcec38822b674b9536cebe20de17a +febd2933742874924e316cb3e40b4c47b5ed5211 +d6057cb92be6fdb07d02da23e13035b90a57d1fc +286b836c3f421c8abf8dc0db3a8288e5fade135b +0f5bb8b893afc4955f608d9a206035f63e2bb6fe +f3e8fe23122bb47e2d8dfb4c8264e920007ee20b +ead87afce1ab0d7179b9012f88df1089e609bc61 +0f09545d9626b7e64c883a721f592847c5fd4367 +216a02fba12fe445e36fae1c019820715379c437 +6366dc4c55076c084caddcbd67f997c17e9360bd +7d4dacf8bdeb24513888719065069102152b9be4 +e431a9675cd7da4b229efe95f92ba44e73ff87a9 +38bae672da42db241eb885eceeb3a6097644eeb5 +3343394c65e4268b0c375fd3024daff9d6f94c60 +7d6d6d2805f34a6de6d6e1400d376e8970886c4d +b21965a98600b4fcd3bbf286ac9b5f9bd9032fa6 +399907c1f4e65ea00f53f1050df6117c75f71e8c +c431cf6f44a221dc6e7971d4977a1575c6d16867 +e18b188e81f64656b45232da8ec15c9954ef2281 +b5f5b35f41208af238d6f6bc2bdb2a4fd1f71110 +f0453c42c0266468bc3cc0dfe1bf527f778dff2d +817e82a80529dd34254643b9786a806eb2b6769c +93d79da903c10ff51ebd1db494165ca99aa4c5ae +4f890799455b11ccd671ebe0785e32d2c7d34d76 +f6a40d36969b24c49b64c912f46f7d33c16cfd99 +552f9b8bd3233ae4d6a123e04e958a99dee8748e +e8772c33401d8e5ba4290678d820b18004f0d6d4 +add135fe0527c2bf29cbfcbcf1639931fee057bb +bdeab82b494d2b7edd1700b5339f7ce3cbe89fa6 +70b8ec7e44ae11e2a9b0f24491f60fbf05429a36 +699a4bd2e8e09035e3b93e0d117ba6874e119a8e +4d8f9390047b7f3dbb1836ad4390c74a33c6632d +94d41b262ab0008ca0110606dac32e418b22f1df +9ae7a93ccb01bcb3570ef7e2c3120ab725720245 +aa5d4b0c90c0292b60421e1944f994b91cfec524 +74e9d56adb1ea4c71fdaa764efbb9e69a363e8af +2433c00dcba8162b8068fbfa2e2029e60646b560 +3625d401c619e5f6cf86efcc8cf1263f0a7a1ffe +0a3c177f4b6083a00f75810340cf7570f373bb15 +51785524a3e28db65d6444777a706dcf144efc1c +05b0a81808c63de36cc90fef489225160522293e +763d77d19d0614eed198c23bc97338eebf61995c +460bbd080f5fd119abacd2cf64c2946dcfd4231c +5f554ab28fa326ddb140a655733d6adcd9a3cc65 +423c3d69992bd803d17f747e18c72a054e4da474 +cae7d2db30eeaeb61b847f35e6178b72b69224e5 +e261c5da31e2f27d2e9538ec3bd785b3d98b233a +58eab3cf2f9b05ba26a8506ef6c8a90871ee3db6 +e83b1b029128c392c6ff9e5c9de30e5ce0f1e141 +7cff6b5e1d8b02014979e3866fb040afa955d7b0 +435732021833ab48172b3005852a1eeb88af4af5 +c24b119928b4b2475f578c9b7f282381bf95ee53 +23c92ca436575bff821d8f15b340a6bf94020596 +36fbc53289da6783d34a61cd0860ce9aef497aa2 +a9b09e4ae4fa03725575c6fcf3ab2ad3a81c992d +3a80ac20277e0d3d72825e88a4872af57f4401af +ce6df71399c9b7175736c2666a5ec94bf3c61291 +340290883434116288b977febdfb0cd5ef408662 +f128cc3e3cfddad4659f3a7b5bd3f53b640f68e1 +f717ce9d3d691ff35849bbe62d9264f9f0c66426 +d45bee628e767b1f28f73f9f16b6bdabda2ad40d +54f0cde4907b6451d289c32a6ff15d3d007095dd +0b57626387a0a61e764de5413362e5389f1b71c2 +27c0d3a2d2988d8e01d3e8d6e14793a7ee5ed843 +8fa29865b6365620513f9a30c7e14b31a9ec1c7a +b74b4239fa387b0222654df93590960e6beb1bee +78a580ddf1ab885006020a9313e66a21534455af +b7c58883b7cf2279f638c450e12bfa920221d785 +9026e0920dc932a0705de81b484eea21bb08abe7 +8db3780fce367aa9643d54eea7ca3e89646273cb +7c7da754bb91acfe6f3eb098cac36239ce908319 +e65ac95a44a5d5199105aa8dd190ebe6657d998b +d0e0880b1965a6769197be1fc9aaa5650e229bca +5ab10eae37cd71a4676478c79c79b520ff048d61 +13769f90af4500948b0376d57df4cceaa13e78b5 +36003c96270f622e67288ed8179e04cbbaa42d84 +b134a395b44d1023f1c9f5cf63e2feffa1faf9c0 +64745e7ec2ef7008d128d9ef2234f5d78a2243b8 +03db47ed5e4b1e02a784dbf0274627ecb0714b73 +9f87099dbbb0d2519235cc19327dbb198776e040 +cb94e5974eddb705f5391cee0fea489e5f573fbb +e6ecbc95d71a47e913680db14f3861daad4dae11 +8bdba5653a51e42ac1651b8def36f1b4a3a0999a +0cf6d47afbac9d273f27e37f84592c5fb4b430db +3c099095dc50ce7d27d080686b005b49248cc216 +acc6ee6bfe25114e9527c3f27de1307ff098490a +94e5f160b0f46764c4c95eed6a7de90ef3d65717 +dcd16e32a88385fb1fe8366da4de9807ce33baf3 +85d01bceb0dd87fb0d02648cd70d753450175f62 +92080b1cd2ca5f123158571481722767b99c2b27 +5d1ef8a64055a039aa4a0834d2d26fe7752fe9a0 diff --git a/.github/workflows/update-fixed-env.yaml b/.github/workflows/update-fixed-env.yaml new file mode 100644 index 00000000..f66890aa --- /dev/null +++ b/.github/workflows/update-fixed-env.yaml @@ -0,0 +1,39 @@ +name: Fixed Environment YAML Monitor + +on: + push: + branches: + - master + paths: + - 'env/environment.yaml' + +jobs: + update_environment_fixed: + runs-on: ubuntu-latest + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: latest + environment-file: envs/environment.yaml + log-level: debug + init-shell: bash + cache-environment: true + cache-downloads: true + + - name: Update environment.fixed.yaml + run: | + mamba env export --file envs/environment.fixed.yaml --no-builds + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: update-environment-fixed + title: Update fixed environment + body: Automatically generated PR to update environment.fixed.yaml + labels: automated diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cb2d98fd..72eabf9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,7 @@ repos: # Do YAML formatting (before the linter checks it for misses) - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.13.0 + rev: v2.14.0 hooks: - id: pretty-format-yaml args: [--autofix, --indent, "2", --preserve-quotes] @@ -87,6 +87,6 @@ repos: # Check for FSFE REUSE compliance (licensing) - repo: https://github.com/fsfe/reuse-tool - rev: v3.1.0a1 + rev: v4.0.3 hooks: - id: reuse diff --git a/config/config.default.yaml b/config/config.default.yaml index fb89878f..67c4db14 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -134,35 +134,25 @@ electricity: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite atlite: - default_cutout: europe-2013-era5 + default_cutout: europe-2013-sarah3-era5 nprocesses: 4 show_progress: false cutouts: # use 'base' to determine geographical bounds and time span from config # base: # module: era5 - europe-2013-era5: - module: era5 # in priority order + europe-2013-sarah3-era5: + module: [sarah, era5] # in priority order x: [-12., 42.] - y: [33., 72] + y: [33., 72.] dx: 0.3 dy: 0.3 time: ['2013', '2013'] - europe-2013-sarah: - module: [sarah, era5] # in priority order - x: [-12., 42.] - y: [33., 65] - dx: 0.2 - dy: 0.2 - time: ['2013', '2013'] - sarah_interpolate: false - sarah_dir: - features: [influx, temperature] # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#renewable renewable: onwind: - cutout: europe-2013-era5 + cutout: europe-2013-sarah3-era5 resource: method: wind turbine: Vestas_V112_3MW @@ -181,7 +171,7 @@ renewable: excluder_resolution: 100 clip_p_max_pu: 1.e-2 offwind-ac: - cutout: europe-2013-era5 + cutout: europe-2013-sarah3-era5 resource: method: wind turbine: NREL_ReferenceTurbine_2020ATB_5.5MW @@ -197,7 +187,7 @@ renewable: excluder_resolution: 200 clip_p_max_pu: 1.e-2 offwind-dc: - cutout: europe-2013-era5 + cutout: europe-2013-sarah3-era5 resource: method: wind turbine: NREL_ReferenceTurbine_2020ATB_5.5MW @@ -213,7 +203,7 @@ renewable: excluder_resolution: 200 clip_p_max_pu: 1.e-2 offwind-float: - cutout: europe-2013-era5 + cutout: europe-2013-sarah3-era5 resource: method: wind turbine: NREL_ReferenceTurbine_5MW_offshore @@ -231,7 +221,7 @@ renewable: max_depth: 1000 clip_p_max_pu: 1.e-2 solar: - cutout: europe-2013-sarah + cutout: europe-2013-sarah3-era5 resource: method: pv panel: CSi @@ -246,7 +236,7 @@ renewable: excluder_resolution: 100 clip_p_max_pu: 1.e-2 solar-hsat: - cutout: europe-2013-sarah + cutout: europe-2013-sarah3-era5 resource: method: pv panel: CSi @@ -261,7 +251,7 @@ renewable: excluder_resolution: 100 clip_p_max_pu: 1.e-2 hydro: - cutout: europe-2013-era5 + cutout: europe-2013-sarah3-era5 carriers: [ror, PHS, hydro] PHS_max_hours: 6 hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float @@ -295,7 +285,7 @@ lines: under_construction: 'keep' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity dynamic_line_rating: activate: false - cutout: europe-2013-era5 + cutout: europe-2013-sarah3-era5 correction_factor: 0.95 max_voltage_difference: false max_line_rating: false @@ -572,12 +562,12 @@ sector: min_part_load_fischer_tropsch: 0.5 min_part_load_methanolisation: 0.3 min_part_load_methanation: 0.3 - use_fischer_tropsch_waste_heat: true - use_haber_bosch_waste_heat: true - use_methanolisation_waste_heat: true - use_methanation_waste_heat: true - use_fuel_cell_waste_heat: true - use_electrolysis_waste_heat: true + use_fischer_tropsch_waste_heat: 0.25 + use_haber_bosch_waste_heat: 0.25 + use_methanolisation_waste_heat: 0.25 + use_methanation_waste_heat: 0.25 + use_fuel_cell_waste_heat: 0.25 + use_electrolysis_waste_heat: 0.25 electricity_transmission_grid: true electricity_distribution_grid: true electricity_distribution_grid_cost_factor: 1.0 @@ -787,6 +777,7 @@ solving: options: clip_p_max_pu: 1.e-2 load_shedding: false + curtailment_mode: false noisy_costs: true skip_iterations: true rolling_horizon: false @@ -831,7 +822,7 @@ solving: solver_options: highs-default: # refer to https://ergo-code.github.io/HiGHS/dev/options/definitions/ - threads: 4 + threads: 1 solver: "ipm" run_crossover: "off" small_matrix_value: 1e-6 @@ -842,7 +833,7 @@ solving: parallel: "on" random_seed: 123 gurobi-default: - threads: 4 + threads: 8 method: 2 # barrier crossover: 0 BarConvTol: 1.e-6 @@ -880,6 +871,13 @@ solving: Threads: 8 LpMethod: 2 Crossover: 0 + RelGap: 1.e-6 + Dualize: 0 + copt-gpu: + LpMethod: 6 + GPUMode: 1 + PDLPTol: 1.e-5 + Crossover: 0 cbc-default: {} # Used in CI glpk-default: {} # Used in CI @@ -1057,7 +1055,7 @@ plotting: V2G: '#e5ffa8' land transport EV: '#baf238' land transport demand: '#38baf2' - Li ion: '#baf238' + EV battery: '#baf238' # hot water storage water tanks: '#e69487' residential rural water tanks: '#f7b7a3' diff --git a/data/GDP_PPP_30arcsec_v3_mapped_default.csv b/data/GDP_PPP_30arcsec_v3_mapped_default.csv deleted file mode 100644 index f0e640b3..00000000 --- a/data/GDP_PPP_30arcsec_v3_mapped_default.csv +++ /dev/null @@ -1,151 +0,0 @@ -name,GDP_PPP,country -3140,632728.0438507323,MD -3139,806541.9318093687,MD -3142,1392454.6690911907,MD -3152,897871.2903553953,MD -3246,645554.8588933202,MD -7049,1150156.4449477682,MD -1924,162285.16792916053,UA -1970,751970.6071848695,UA -2974,368873.75840156944,UA -2977,294847.85539198935,UA -2979,197988.13680768458,UA -2980,301371.2491126519,UA -3031,56925.21878805953,UA -3032,139395.18279351242,UA -3033,145377.8061037629,UA -3035,52282.83655208812,UA -3036,497950.25890516065,UA -3037,1183293.1987702171,UA -3038,255005.98207636533,UA -3039,224711.50098325178,UA -3040,342959.943226467,UA -3044,69119.31486955672,UA -3045,246273.65986119965,UA -3047,146742.08407299497,UA -3049,107265.7028733467,UA -3050,1126147.985259493,UA -3051,69833.56303043803,UA -3052,67230.88206577855,UA -3053,27019.224685201345,UA -3054,260571.47337292184,UA -3055,88760.94152915622,UA -3056,101368.26196568517,UA -3058,55752.92329667119,UA -3059,89024.37880630122,UA -3062,358411.291265149,UA -3064,75081.64142862396,UA -3065,158101.42949135564,UA -3066,83763.89576442329,UA -3068,173474.51218344545,UA -3069,60327.01572375589,UA -3070,18073.687271955278,UA -3071,249069.43314695224,UA -3072,220707.35700825177,UA -3073,61342.30137462664,UA -3074,254235.98867635374,UA -3077,769558.9832370486,UA -3078,132674.2315809836,UA -3079,1388517.1478032232,UA -3080,1861003.8718246964,UA -3082,140123.73854745473,UA -3083,834887.5595419679,UA -3084,1910795.5590558557,UA -3086,93828.36549170096,UA -3088,347197.65113392205,UA -3089,3754718.141734592,UA -3090,521912.69768585655,UA -3093,232818.05269714879,UA -3095,435376.20361377904,UA -3099,345596.5288937008,UA -3100,175689.10947424968,UA -3105,538438.9311459162,UA -3107,88096.86032871014,UA -3108,79847.68447063807,UA -3109,348504.73449373,UA -3144,71657.0165675802,UA -3146,80342.05037424155,UA -3158,74465.12922576343,UA -3164,3102112.2672631275,UA -3165,65215.04081671433,UA -3166,413924.2225725632,UA -3167,135060.0056434935,UA -3168,54980.442979330146,UA -3170,29584.879122227037,UA -3171,142780.68163047134,UA -3172,40436.63814695243,UA -3173,1253342.1790126422,UA -3174,173842.03139155387,UA -3176,65699.76352408895,UA -3177,143591.75419817626,UA -3178,56434.04525832523,UA -3179,389996.1670051216,UA -3180,138452.84503524794,UA -3181,67402.59500436619,UA -3184,51204.293695376415,UA -3185,46867.82356528432,UA -3186,103892.35612417295,UA -3187,193668.91476930346,UA -3189,54584.176457692694,UA -3190,219077.64942830536,UA -3197,88516.52699983507,UA -3198,298166.8272673622,UA -3199,61334.952541812374,UA -3229,175692.61136747137,UA -3230,106722.62773321665,UA -3236,61542.06264321315,UA -3241,83752.90489164277,UA -4301,48419.52825967164,UA -4305,147759.74280349456,UA -4306,53156.905740992224,UA -4315,218025.78516351627,UA -4317,155240.40554731718,UA -4318,1342144.2459407183,UA -4319,91669.1449633853,UA -4321,85852.49282415409,UA -4347,67938.7698430624,UA -4357,20064.979012172935,UA -4360,47840.51245168512,UA -4361,55580.924388032574,UA -4362,165753.82588729708,UA -4363,46390.2448142152,UA -4365,96265.47592938849,UA -4366,272003.25510057947,UA -4367,80878.50229245829,UA -4370,330072.35444044066,UA -4371,7707066.181975477,UA -4373,2019766.7891575783,UA -4374,985354.331818515,UA -4377,230805.08833664874,UA -4382,125670.67125287943,UA -4383,46914.065511740075,UA -4384,48020.804310510954,UA -4385,55612.34707641123,UA -4387,74558.3475791577,UA -4388,245243.33449409154,UA -4389,95696.56767732685,UA -4391,251085.7523045193,UA -4401,66375.82996856027,UA -4403,111954.41038437477,UA -4405,46911.68560148837,UA -4408,150782.51691456966,UA -4409,112776.7399582134,UA -4410,153076.56860965435,UA -4412,192629.31238456024,UA -4413,181295.3120834606,UA -4414,995694.9413199169,UA -4416,157640.7868989174,UA -4418,77580.20674809469,UA -4420,122320.99275223716,UA -4424,184891.10924920067,UA -4425,84486.75974340564,UA -4431,50485.84380961137,UA -4435,231040.45446464577,UA -4436,81222.18707585508,UA -4438,114819.76472988473,UA -4439,76839.1052178896,UA -4440,135337.0313562152,UA -4441,49159.485269198034,UA -7031,42001.73757065917,UA -7059,159790.48382874,UA -7063,39599.10564971086,UA diff --git a/doc/configtables/atlite.csv b/doc/configtables/atlite.csv index 0b01005d..5e0d2bd0 100644 --- a/doc/configtables/atlite.csv +++ b/doc/configtables/atlite.csv @@ -3,7 +3,7 @@ default_cutout,--,str,"Defines a default cutout." nprocesses,--,int,"Number of parallel processes in cutout preparation" show_progress,bool,true/false,"Whether progressbar for atlite conversion processes should be shown. False saves time." cutouts,,, --- {name},--,"Convention is to name cutouts like ``--`` (e.g. ``europe-2013-era5``).","Name of the cutout netcdf file. The user may specify multiple cutouts under configuration ``atlite: cutouts:``. Reference is used in configuration ``renewable: {technology}: cutout:``. The cutout ``base`` may be used to automatically calculate temporal and spatial bounds of the network." +-- {name},--,"Convention is to name cutouts like ``--`` (e.g. ``europe-2013-sarah3-era5``).","Name of the cutout netcdf file. The user may specify multiple cutouts under configuration ``atlite: cutouts:``. Reference is used in configuration ``renewable: {technology}: cutout:``. The cutout ``base`` may be used to automatically calculate temporal and spatial bounds of the network." -- -- module,--,"Subset of {'era5','sarah'}","Source of the reanalysis weather dataset (e.g. `ERA5 `_ or `SARAH-2 `_)" -- -- x,°,"Float interval within [-180, 180]","Range of longitudes to download weather data for. If not defined, it defaults to the spatial bounds of all bus shapes." -- -- y,°,"Float interval within [-90, 90]","Range of latitudes to download weather data for. If not defined, it defaults to the spatial bounds of all bus shapes." diff --git a/doc/configtables/hydro.csv b/doc/configtables/hydro.csv index 790029d1..8903aa0f 100644 --- a/doc/configtables/hydro.csv +++ b/doc/configtables/hydro.csv @@ -1,5 +1,5 @@ ,Unit,Values,Description -cutout,--,Must be 'europe-2013-era5',Specifies the directory where the relevant weather data ist stored. +cutout,--,Must be 'europe-2013-sarah3-era5',Specifies the directory where the relevant weather data ist stored. carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydro power plants to build per-unit availability time series for. 'ror' stands for run-of-river plants, 'PHS' represents pumped-hydro storage, and 'hydro' stands for hydroelectric dams." PHS_max_hours,h,float,Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}",Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation `_. diff --git a/doc/configtables/lines.csv b/doc/configtables/lines.csv index 3707d4a6..79fa2e16 100644 --- a/doc/configtables/lines.csv +++ b/doc/configtables/lines.csv @@ -8,7 +8,7 @@ under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove co reconnect_crimea,--,"true or false","Whether to reconnect Crimea to the Ukrainian grid" dynamic_line_rating,,, -- activate,bool,"true or false","Whether to take dynamic line rating into account" --- cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored." +-- cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-sarah3-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored." -- correction_factor,--,"float","Factor to compensate for overestimation of wind speeds in hourly averaged wind data" -- max_voltage_difference,deg,"float","Maximum voltage angle difference in degrees or 'false' to disable" -- max_line_rating,--,"float","Maximum line rating relative to nominal capacity without DLR, e.g. 1.3 or 'false' to disable" diff --git a/doc/configtables/offwind-ac.csv b/doc/configtables/offwind-ac.csv index b2533f04..9ba2fa7e 100644 --- a/doc/configtables/offwind-ac.csv +++ b/doc/configtables/offwind-ac.csv @@ -1,5 +1,5 @@ ,Unit,Values,Description -cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored." +cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-sarah3-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored." resource,,, -- method,--,"Must be 'wind'","A superordinate technology type." -- turbine,--,"One of turbine types included in `atlite `_. Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve." diff --git a/doc/configtables/offwind-dc.csv b/doc/configtables/offwind-dc.csv index 7c537543..e55d8944 100644 --- a/doc/configtables/offwind-dc.csv +++ b/doc/configtables/offwind-dc.csv @@ -1,5 +1,5 @@ ,Unit,Values,Description -cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored." +cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-sarah3-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored." resource,,, -- method,--,"Must be 'wind'","A superordinate technology type." -- turbine,--,"One of turbine types included in `atlite `_. Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve." diff --git a/doc/configtables/onwind.csv b/doc/configtables/onwind.csv index 3b09214b..a801d83c 100644 --- a/doc/configtables/onwind.csv +++ b/doc/configtables/onwind.csv @@ -1,5 +1,5 @@ ,Unit,Values,Description -cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored." +cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-sarah3-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored." resource,,, -- method,--,"Must be 'wind'","A superordinate technology type." -- turbine,--,"One of turbine types included in `atlite `_. Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve." diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv index 059c4233..5045cecd 100644 --- a/doc/configtables/sector.csv +++ b/doc/configtables/sector.csv @@ -5,7 +5,7 @@ biomass,--,"{true, false}",Flag to include biomass sector. industry,--,"{true, false}",Flag to include industry sector. agriculture,--,"{true, false}",Flag to include agriculture sector. district_heating,--,,`prepare_sector_network.py `_ --- potential,--,float,maximum fraction of urban demand which can be supplied by district heating +-- potential,--,float,maximum fraction of urban demand which can be supplied by district heating. Ignored where below current fraction. -- progress,--,Dictionary with planning horizons as keys., Increase of today's district heating demand to potential maximum district heating share. Progress = 0 means today's district heating share. Progress = 1 means maximum fraction of urban demand is supplied by district heating -- district_heating_loss,--,float,Share increase in district heat demand in urban central due to heat losses cluster_heat_buses,--,"{true, false}",Cluster residential and service heat buses in `prepare_sector_network.py `_ to one to save memory. diff --git a/doc/configtables/solar.csv b/doc/configtables/solar.csv index 18587694..21d7c2e4 100644 --- a/doc/configtables/solar.csv +++ b/doc/configtables/solar.csv @@ -1,5 +1,5 @@ ,Unit,Values,Description -cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module can be ERA5 or SARAH-2.","Specifies the directory where the relevant weather data ist stored that is specified at ``atlite/cutouts`` configuration. Both ``sarah`` and ``era5`` work." +cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-sarah3-era5') or reference an existing folder in the directory ``cutouts``. Source module can be ERA5 or SARAH-2.","Specifies the directory where the relevant weather data ist stored that is specified at ``atlite/cutouts`` configuration. Both ``sarah`` and ``era5`` work." resource,,, -- method,--,"Must be 'pv'","A superordinate technology type." -- panel,--,"One of {'Csi', 'CdTe', 'KANENA'} as defined in `atlite `_ . Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the solar panel technology and its characteristic attributes." diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv index 4cfb9065..d2e22c28 100644 --- a/doc/configtables/solving.csv +++ b/doc/configtables/solving.csv @@ -2,6 +2,7 @@ options,,, -- clip_p_max_pu,p.u.,float,To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero. -- load_shedding,bool/float,"{'true','false', float}","Add generators with very high marginal cost to simulate load shedding and avoid problem infeasibilities. If load shedding is a float, it denotes the marginal cost in EUR/kWh." +-- curtailment_mode,bool/float,"{'true','false'}","Fixes the dispatch profiles of generators with time-varying p_max_pu by setting ``p_min_pu = p_max_pu`` and adds an auxiliary curtailment generator (with negative sign to absorb excess power) at every AC bus. This can speed up the solving process as the curtailment decision is aggregated into a single generator per region. Defaults to ``false``." -- noisy_costs,bool,"{'true','false'}","Add random noise to marginal cost of generators by :math:`\mathcal{U}(0.009,0,011)` and capital cost of lines and links by :math:`\mathcal{U}(0.09,0,11)`." -- skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches. Defaults to true." -- rolling_horizon,bool,"{'true','false'}","Switch for rule :mod:`solve_operations_network` whether to optimize the network in a rolling horizon manner, where the snapshot range is split into slices of size `horizon` which are solved consecutively. This setting has currently no effect on sector-coupled networks." diff --git a/doc/installation.rst b/doc/installation.rst index 45404e1f..2e942978 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -81,7 +81,8 @@ Nevertheless, you can still use open-source solvers for smaller problems. .. note:: The rules :mod:`cluster_network` and :mod:`simplify_network` solve a mixed-integer quadratic optimisation problem for clustering. The open-source solvers HiGHS, Cbc and GlPK cannot handle this. A fallback to SCIP is implemented in this case, which is included in the standard environment specifications. - For an open-source solver setup install in your ``conda`` environment on OSX/Linux. To install the default solver Gurobi, run + For an open-source solver setup install for example HiGHS **and** SCIP in your ``conda`` environment on OSX/Linux. + To install the default solver Gurobi, run .. code:: bash diff --git a/doc/preparation.rst b/doc/preparation.rst index 06e8b19b..669f3392 100644 --- a/doc/preparation.rst +++ b/doc/preparation.rst @@ -16,7 +16,7 @@ using the ``retrieve*`` rules (:ref:`data`). Having downloaded the necessary data, - :mod:`build_shapes` generates GeoJSON files with shapes of the countries, exclusive economic zones and `NUTS3 `__ areas. -- :mod:`build_cutout` prepares smaller weather data portions from `ERA5 `__ for cutout ``europe-2013-era5`` and SARAH for cutout ``europe-2013-sarah``. +- :mod:`build_cutout` prepares smaller weather data portions from `ERA5 `__ for cutout ``europe-2013-sarah3-era5`` and SARAH for cutout ``europe-2013-sarah``. With these and the externally extracted ENTSO-E online map topology (``data/entsoegridkit``), it can build a base PyPSA network with the following rules: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 8388946f..383287a6 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,14 @@ Release Notes Upcoming Release ================ +* Renamed the carrier of batteries in BEVs from `battery storage` to `EV battery` and the corresponding bus carrier from `Li ion` to `EV battery`. This is to avoid confusion with stationary battery storage. + +* Changed default assumptions about waste heat usage from PtX and fuel cells in district heating. + The default value for the link efficiency scaling factor was changed from 100% to 25%. + It can be set to other values in the configuration ``sector: use_TECHNOLOGY_waste_heat``. + +* In simplifying polygons in :mod:`build_shapes` default to no tolerance. + * Set non-zero capital_cost for methanol stores to avoid unrealistic storage sizes * Set p_nom = p_nom_min for generators with baseyear == grouping_year in add_existing_baseyear. This has no effect on the optimization but helps n.statistics to correctly report already installed capacities. @@ -25,6 +33,33 @@ Upcoming Release * Bugfix: Correctly read in threshold capacity below which to remove components from previous planning horizons in :mod:`add_brownfield`. +* For countries not contained in the NUTS3-specific datasets (i.e. MD and UA), the mapping of GDP per capita and population per bus region used to spatially distribute electricity demand is now endogenised in a new rule :mod:`build_gdp_ppp_non_nuts3`. https://github.com/PyPSA/pypsa-eur/pull/1146 + +* The databundle has been updated to release v0.3.0, which includes raw GDP and population data for countries outside the NUTS system (UA, MD). https://github.com/PyPSA/pypsa-eur/pull/1146 + +* Updated filtering in :mod:`determine_availability_matrix_MD_UA.py` to improve speed. https://github.com/PyPSA/pypsa-eur/pull/1146 + +* Bugfix: Impose minimum value of zero for district heating progress between current and future market share in :mod:`build_district_heat_share`. + +* The ``{scope}`` wildcard was removed, since its outputs were not used. + +* Enable parallelism in :mod:`determine_availability_matrix_MD_UA.py` and remove plots. This requires the use of temporary files. + +* Updated pre-built `weather data cutouts + `__. These are now merged cutouts with + solar irradiation from the new SARAH-3 dataset while taking all other + variables from ERA5. Cutouts are now available for multiple years (2010, 2013, + 2019, and 2023). + +* Added option ``solving: curtailment_mode``` which fixes the dispatch profiles + of generators with time-varying p_max_pu by setting ``p_min_pu = p_max_pu`` + and adds an auxiliary curtailment generator with negative sign (to absorb + excess power) at every AC bus. This can speed up the solving process as the + curtailment decision is aggregated into a single generator per region. + +* In :mod:`base_network`, replace own voronoi polygon calculation function with + Geopandas `gdf.voronoi_polygons` method. + PyPSA-Eur 0.11.0 (25th May 2024) ===================================== diff --git a/doc/wildcards.rst b/doc/wildcards.rst index f8e60e20..9ddb7b0a 100644 --- a/doc/wildcards.rst +++ b/doc/wildcards.rst @@ -142,13 +142,6 @@ The ``{sector_opts}`` wildcard is only used for sector-coupling studies. :widths: 10,20,10,10 :file: configtables/sector-opts.csv -.. _scope: - -The ``{scope}`` wildcard -======================== - -Takes values ``residential``, ``urban``, ``total``. - .. _planning_horizons: The ``{planning_horizons}`` wildcard diff --git a/envs/environment.yaml b/envs/environment.yaml index 8f2ecd27..febd6ea2 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -28,7 +28,7 @@ dependencies: - powerplantmatching>=0.5.15 - numpy - pandas>=2.1 -- geopandas>=0.11.0 +- geopandas>=1 - xarray>=2023.11.0 - rioxarray - netcdf4 diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index eff0d9e0..18ff8230 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -202,7 +202,6 @@ rule determine_availability_matrix_MD_UA: + ".nc", output: availability_matrix=resources("availability_matrix_MD-UA_{technology}.nc"), - availability_map=resources("availability_matrix_MD-UA_{technology}.png"), log: logs("determine_availability_matrix_MD_UA_{technology}.log"), threads: config["atlite"].get("nprocesses", 4) @@ -375,6 +374,37 @@ def input_conventional(w): } +# Optional input when having Ukraine (UA) or Moldova (MD) in the countries list +def input_gdp_pop_non_nuts3(w): + countries = set(config_provider("countries")(w)) + if {"UA", "MD"}.intersection(countries): + return {"gdp_pop_non_nuts3": resources("gdp_pop_non_nuts3.geojson")} + return {} + + +rule build_gdp_pop_non_nuts3: + params: + countries=config_provider("countries"), + input: + base_network=resources("networks/base.nc"), + regions=resources("regions_onshore.geojson"), + gdp_non_nuts3="data/bundle/GDP_per_capita_PPP_1990_2015_v2.nc", + pop_non_nuts3="data/bundle/ppp_2013_1km_Aggregated.tif", + output: + resources("gdp_pop_non_nuts3.geojson"), + log: + logs("build_gdp_pop_non_nuts3.log"), + benchmark: + benchmarks("build_gdp_pop_non_nuts3") + threads: 1 + resources: + mem_mb=8000, + conda: + "../envs/environment.yaml" + script: + "../scripts/build_gdp_pop_non_nuts3.py" + + rule add_electricity: params: length_factor=config_provider("lines", "length_factor"), @@ -390,6 +420,7 @@ rule add_electricity: input: unpack(input_profile_tech), unpack(input_conventional), + unpack(input_gdp_pop_non_nuts3), base_network=resources("networks/base.nc"), line_rating=lambda w: ( resources("networks/line_rating.nc") @@ -411,7 +442,6 @@ rule add_electricity: ), load=resources("electricity_demand.csv"), nuts3_shapes=resources("nuts3_shapes.geojson"), - ua_md_gdp="data/GDP_PPP_30arcsec_v3_mapped_default.csv", output: resources("networks/elec.nc"), log: diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 6614b163..139ced1f 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -151,18 +151,18 @@ rule build_daily_heat_demand: snapshots=config_provider("snapshots"), drop_leap_day=config_provider("enable", "drop_leap_day"), input: - pop_layout=resources("pop_layout_{scope}.nc"), + pop_layout=resources("pop_layout_total.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), cutout=heat_demand_cutout, output: - heat_demand=resources("daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"), + heat_demand=resources("daily_heat_demand_total_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, threads: 8 log: - logs("build_daily_heat_demand_{scope}_{simpl}_{clusters}.loc"), + logs("build_daily_heat_demand_total_{simpl}_{clusters}.loc"), benchmark: - benchmarks("build_daily_heat_demand/{scope}_s{simpl}_{clusters}") + benchmarks("build_daily_heat_demand/total_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -175,16 +175,16 @@ rule build_hourly_heat_demand: drop_leap_day=config_provider("enable", "drop_leap_day"), input: heat_profile="data/heat_load_profile_BDEW.csv", - heat_demand=resources("daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"), + heat_demand=resources("daily_heat_demand_total_elec_s{simpl}_{clusters}.nc"), output: - heat_demand=resources("hourly_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"), + heat_demand=resources("hourly_heat_demand_total_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=2000, threads: 8 log: - logs("build_hourly_heat_demand_{scope}_{simpl}_{clusters}.loc"), + logs("build_hourly_heat_demand_total_{simpl}_{clusters}.loc"), benchmark: - benchmarks("build_hourly_heat_demand/{scope}_s{simpl}_{clusters}") + benchmarks("build_hourly_heat_demand/total_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -196,19 +196,19 @@ rule build_temperature_profiles: snapshots=config_provider("snapshots"), drop_leap_day=config_provider("enable", "drop_leap_day"), input: - pop_layout=resources("pop_layout_{scope}.nc"), + pop_layout=resources("pop_layout_total.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), cutout=heat_demand_cutout, output: - temp_soil=resources("temp_soil_{scope}_elec_s{simpl}_{clusters}.nc"), - temp_air=resources("temp_air_{scope}_elec_s{simpl}_{clusters}.nc"), + temp_soil=resources("temp_soil_total_elec_s{simpl}_{clusters}.nc"), + temp_air=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, threads: 8 log: - logs("build_temperature_profiles_{scope}_{simpl}_{clusters}.log"), + logs("build_temperature_profiles_total_{simpl}_{clusters}.log"), benchmark: - benchmarks("build_temperature_profiles/{scope}_s{simpl}_{clusters}") + benchmarks("build_temperature_profiles/total_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -220,18 +220,10 @@ rule build_cop_profiles: heat_pump_sink_T=config_provider("sector", "heat_pump_sink_T"), input: temp_soil_total=resources("temp_soil_total_elec_s{simpl}_{clusters}.nc"), - temp_soil_rural=resources("temp_soil_rural_elec_s{simpl}_{clusters}.nc"), - temp_soil_urban=resources("temp_soil_urban_elec_s{simpl}_{clusters}.nc"), temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), - temp_air_rural=resources("temp_air_rural_elec_s{simpl}_{clusters}.nc"), - temp_air_urban=resources("temp_air_urban_elec_s{simpl}_{clusters}.nc"), output: cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), - cop_soil_rural=resources("cop_soil_rural_elec_s{simpl}_{clusters}.nc"), - cop_soil_urban=resources("cop_soil_urban_elec_s{simpl}_{clusters}.nc"), cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), - cop_air_rural=resources("cop_air_rural_elec_s{simpl}_{clusters}.nc"), - cop_air_urban=resources("cop_air_urban_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, log: @@ -263,18 +255,18 @@ rule build_solar_thermal_profiles: drop_leap_day=config_provider("enable", "drop_leap_day"), solar_thermal=config_provider("solar_thermal"), input: - pop_layout=resources("pop_layout_{scope}.nc"), + pop_layout=resources("pop_layout_total.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), cutout=solar_thermal_cutout, output: - solar_thermal=resources("solar_thermal_{scope}_elec_s{simpl}_{clusters}.nc"), + solar_thermal=resources("solar_thermal_total_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, threads: 16 log: - logs("build_solar_thermal_profiles_{scope}_s{simpl}_{clusters}.log"), + logs("build_solar_thermal_profiles_total_s{simpl}_{clusters}.log"), benchmark: - benchmarks("build_solar_thermal_profiles/{scope}_s{simpl}_{clusters}") + benchmarks("build_solar_thermal_profiles/total_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -1024,32 +1016,14 @@ rule prepare_sector_network: "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), temp_soil_total=resources("temp_soil_total_elec_s{simpl}_{clusters}.nc"), - temp_soil_rural=resources("temp_soil_rural_elec_s{simpl}_{clusters}.nc"), - temp_soil_urban=resources("temp_soil_urban_elec_s{simpl}_{clusters}.nc"), temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), - temp_air_rural=resources("temp_air_rural_elec_s{simpl}_{clusters}.nc"), - temp_air_urban=resources("temp_air_urban_elec_s{simpl}_{clusters}.nc"), cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), - cop_soil_rural=resources("cop_soil_rural_elec_s{simpl}_{clusters}.nc"), - cop_soil_urban=resources("cop_soil_urban_elec_s{simpl}_{clusters}.nc"), cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), - cop_air_rural=resources("cop_air_rural_elec_s{simpl}_{clusters}.nc"), - cop_air_urban=resources("cop_air_urban_elec_s{simpl}_{clusters}.nc"), solar_thermal_total=lambda w: ( resources("solar_thermal_total_elec_s{simpl}_{clusters}.nc") if config_provider("sector", "solar_thermal")(w) else [] ), - solar_thermal_urban=lambda w: ( - resources("solar_thermal_urban_elec_s{simpl}_{clusters}.nc") - if config_provider("sector", "solar_thermal")(w) - else [] - ), - solar_thermal_rural=lambda w: ( - resources("solar_thermal_rural_elec_s{simpl}_{clusters}.nc") - if config_provider("sector", "solar_thermal")(w) - else [] - ), egs_potentials=lambda w: ( resources("egs_potentials_s{simpl}_{clusters}.csv") if config_provider("sector", "enhanced_geothermal", "enable")(w) diff --git a/rules/common.smk b/rules/common.smk index 2b8495e1..ef518beb 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -55,7 +55,7 @@ def dynamic_getter(wildcards, keys, default): scenario_name = wildcards.run if scenario_name not in scenarios: raise ValueError( - f"Scenario {scenario_name} not found in file {config['run']['scenario']['file']}." + f"Scenario {scenario_name} not found in file {config['run']['scenarios']['file']}." ) config_with_scenario = scenario_config(scenario_name) config_with_wildcards = update_config_from_wildcards( @@ -81,7 +81,8 @@ def config_provider(*keys, default=None): def solver_threads(w): solver_options = config_provider("solving", "solver_options")(w) option_set = config_provider("solving", "solver", "options")(w) - threads = solver_options[option_set].get("threads", 4) + solver_option_set = solver_options[option_set] + threads = solver_option_set.get("threads") or solver_option_set.get("Threads") or 4 return threads diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 10ad9684..18b0ddd2 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -29,6 +29,8 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", "h2_salt_caverns_GWh_per_sqkm.geojson", "natura/natura.tiff", "gebco/GEBCO_2014_2D.nc", + "GDP_per_capita_PPP_1990_2015_v2.nc", + "ppp_2013_1km_Aggregated.tif", ] rule retrieve_databundle: @@ -69,7 +71,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True rule retrieve_cutout: input: storage( - "https://zenodo.org/records/6382570/files/{cutout}.nc", + "https://zenodo.org/records/12791128/files/{cutout}.nc", ), output: protected("cutouts/" + CDIR + "{cutout}.nc"), @@ -163,7 +165,7 @@ if config["enable"]["retrieve"]: rule retrieve_ship_raster: input: storage( - "https://zenodo.org/records/10973944/files/shipdensity_global.zip", + "https://zenodo.org/records/12760663/files/shipdensity_global.zip", keep_local=True, ), output: diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 672f9e62..72dd1c88 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -89,10 +89,6 @@ def add_brownfield(n, n_p, year): # deal with gas network pipe_carrier = ["gas pipeline"] if snakemake.params.H2_retrofit: - # drop capacities of previous year to avoid duplicating - to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year) - n.mremove("Link", n.links.loc[to_drop].index) - # subtract the already retrofitted from today's gas grid capacity h2_retrofitted_fixed_i = n.links[ (n.links.carrier == "H2 pipeline retrofitted") @@ -115,10 +111,6 @@ def add_brownfield(n, n_p, year): index=pipe_capacity.index ).fillna(0) n.links.loc[gas_pipes_i, "p_nom"] = remaining_capacity - else: - new_pipes = n.links.carrier.isin(pipe_carrier) & (n.links.build_year == year) - n.links.loc[new_pipes, "p_nom"] = 0.0 - n.links.loc[new_pipes, "p_nom_min"] = 0.0 def disable_grid_expansion_if_limit_hit(n): diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 49d0bdf7..49510953 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -287,26 +287,26 @@ def shapes_to_shapes(orig, dest): transfer = sparse.lil_matrix((len(dest), len(orig)), dtype=float) for i, j in product(range(len(dest)), range(len(orig))): - if orig_prepped[j].intersects(dest[i]): - area = orig[j].intersection(dest[i]).area - transfer[i, j] = area / dest[i].area + if orig_prepped[j].intersects(dest.iloc[i]): + area = orig.iloc[j].intersection(dest.iloc[i]).area + transfer[i, j] = area / dest.iloc[i].area return transfer -def attach_load(n, regions, load, nuts3_shapes, ua_md_gdp, countries, scaling=1.0): +def attach_load( + n, regions, load, nuts3_shapes, gdp_pop_non_nuts3, countries, scaling=1.0 +): substation_lv_i = n.buses.index[n.buses["substation_lv"]] - regions = gpd.read_file(regions).set_index("name").reindex(substation_lv_i) + gdf_regions = gpd.read_file(regions).set_index("name").reindex(substation_lv_i) opsd_load = pd.read_csv(load, index_col=0, parse_dates=True).filter(items=countries) - ua_md_gdp = pd.read_csv(ua_md_gdp, dtype={"name": "str"}).set_index("name") - logger.info(f"Load data scaled by factor {scaling}.") opsd_load *= scaling nuts3 = gpd.read_file(nuts3_shapes).set_index("index") - def upsample(cntry, group): + def upsample(cntry, group, gdp_pop_non_nuts3): load = opsd_load[cntry] if len(group) == 1: @@ -325,7 +325,15 @@ def attach_load(n, regions, load, nuts3_shapes, ua_md_gdp, countries, scaling=1. factors = normed(0.6 * normed(gdp_n) + 0.4 * normed(pop_n)) if cntry in ["UA", "MD"]: # overwrite factor because nuts3 provides no data for UA+MD - factors = normed(ua_md_gdp.loc[group.index, "GDP_PPP"].squeeze()) + gdp_pop_non_nuts3 = gpd.read_file(gdp_pop_non_nuts3).set_index("Bus") + gdp_pop_non_nuts3 = gdp_pop_non_nuts3.loc[ + (gdp_pop_non_nuts3.country == cntry) + & (gdp_pop_non_nuts3.index.isin(substation_lv_i)) + ] + factors = normed( + 0.6 * normed(gdp_pop_non_nuts3["gdp"]) + + 0.4 * normed(gdp_pop_non_nuts3["pop"]) + ) return pd.DataFrame( factors.values * load.values[:, np.newaxis], index=load.index, @@ -334,8 +342,8 @@ def attach_load(n, regions, load, nuts3_shapes, ua_md_gdp, countries, scaling=1. load = pd.concat( [ - upsample(cntry, group) - for cntry, group in regions.geometry.groupby(regions.country) + upsample(cntry, group, gdp_pop_non_nuts3) + for cntry, group in gdf_regions.geometry.groupby(gdf_regions.country) ], axis=1, ) @@ -821,7 +829,7 @@ if __name__ == "__main__": snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.input.ua_md_gdp, + snakemake.input.get("gdp_pop_non_nuts3"), params.countries, params.scaling_factor, ) @@ -844,7 +852,7 @@ if __name__ == "__main__": fuel_price = pd.read_csv( snakemake.input.fuel_price, index_col=0, header=0, parse_dates=True ) - fuel_price = fuel_price.reindex(n.snapshots).fillna(method="ffill") + fuel_price = fuel_price.reindex(n.snapshots).ffill() else: fuel_price = None diff --git a/scripts/base_network.py b/scripts/base_network.py index df3bc2b2..118e7dba 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -72,6 +72,7 @@ Creates the network topology from an ENTSO-E map extract, and create Voronoi sha """ import logging +import warnings from itertools import product import geopandas as gpd @@ -85,9 +86,9 @@ import shapely.wkt import yaml from _helpers import REGION_COLS, configure_logging, get_snapshots, set_scenario_config from packaging.version import Version, parse -from scipy import spatial from scipy.sparse import csgraph -from shapely.geometry import LineString, Point, Polygon +from scipy.spatial import KDTree +from shapely.geometry import LineString, Point PD_GE_2_2 = parse(pd.__version__) >= Version("2.2") @@ -118,7 +119,7 @@ def _find_closest_links(links, new_links, distance_upper_bound=1.5): querycoords = np.vstack( [new_links[["x1", "y1", "x2", "y2"]], new_links[["x2", "y2", "x1", "y1"]]] ) - tree = spatial.KDTree(treecoords) + tree = KDTree(treecoords) dist, ind = tree.query(querycoords, distance_upper_bound=distance_upper_bound) found_b = ind < len(links) found_i = np.arange(len(new_links) * 2)[found_b] % len(new_links) @@ -273,7 +274,7 @@ def _add_links_from_tyndp(buses, links, links_tyndp, europe_shape): return buses, links tree_buses = buses.query("carrier=='AC'") - tree = spatial.KDTree(tree_buses[["x", "y"]]) + tree = KDTree(tree_buses[["x", "y"]]) _, ind0 = tree.query(links_tyndp[["x1", "y1"]]) ind0_b = ind0 < len(tree_buses) links_tyndp.loc[ind0_b, "bus0"] = tree_buses.index[ind0[ind0_b]] @@ -671,7 +672,7 @@ def _set_links_underwater_fraction(n, offshore_shapes): if not hasattr(n.links, "geometry"): n.links["underwater_fraction"] = 0.0 else: - offshore_shape = gpd.read_file(offshore_shapes).unary_union + offshore_shape = gpd.read_file(offshore_shapes).union_all() links = gpd.GeoSeries(n.links.geometry.dropna().map(shapely.wkt.loads)) n.links["underwater_fraction"] = ( links.intersection(offshore_shape).length / links.length @@ -788,59 +789,26 @@ def base_network( return n -def voronoi_partition_pts(points, outline): +def voronoi(points, outline, crs=4326): """ - Compute the polygons of a voronoi partition of `points` within the polygon - `outline`. Taken from - https://github.com/FRESNA/vresutils/blob/master/vresutils/graph.py. - - Attributes - ---------- - points : Nx2 - ndarray[dtype=float] - outline : Polygon - Returns - ------- - polygons : N - ndarray[dtype=Polygon|MultiPolygon] + Create Voronoi polygons from a set of points within an outline. """ - points = np.asarray(points) + pts = gpd.GeoSeries( + gpd.points_from_xy(points.x, points.y), + index=points.index, + crs=crs, + ) + voronoi = pts.voronoi_polygons(extend_to=outline).clip(outline) - if len(points) == 1: - polygons = [outline] - else: - xmin, ymin = np.amin(points, axis=0) - xmax, ymax = np.amax(points, axis=0) - xspan = xmax - xmin - yspan = ymax - ymin + # can be removed with shapely 2.1 where order is preserved + # https://github.com/shapely/shapely/issues/2020 + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + pts = gpd.GeoDataFrame(geometry=pts) + voronoi = gpd.GeoDataFrame(geometry=voronoi) + joined = gpd.sjoin_nearest(pts, voronoi, how="right") - # to avoid any network positions outside all Voronoi cells, append - # the corners of a rectangle framing these points - vor = spatial.Voronoi( - np.vstack( - ( - points, - [ - [xmin - 3.0 * xspan, ymin - 3.0 * yspan], - [xmin - 3.0 * xspan, ymax + 3.0 * yspan], - [xmax + 3.0 * xspan, ymin - 3.0 * yspan], - [xmax + 3.0 * xspan, ymax + 3.0 * yspan], - ], - ) - ) - ) - - polygons = [] - for i in range(len(points)): - poly = Polygon(vor.vertices[vor.regions[vor.point_region[i]]]) - - if not poly.is_valid: - poly = poly.buffer(0) - - with np.errstate(invalid="ignore"): - poly = poly.intersection(outline) - - polygons.append(poly) - - return polygons + return joined.dissolve(by="Bus").squeeze() def build_bus_shapes(n, country_shapes, offshore_shapes, countries): @@ -870,11 +838,10 @@ def build_bus_shapes(n, country_shapes, offshore_shapes, countries): "name": onshore_locs.index, "x": onshore_locs["x"], "y": onshore_locs["y"], - "geometry": voronoi_partition_pts( - onshore_locs.values, onshore_shape - ), + "geometry": voronoi(onshore_locs, onshore_shape), "country": country, - } + }, + crs=n.crs, ) ) @@ -887,14 +854,16 @@ def build_bus_shapes(n, country_shapes, offshore_shapes, countries): "name": offshore_locs.index, "x": offshore_locs["x"], "y": offshore_locs["y"], - "geometry": voronoi_partition_pts(offshore_locs.values, offshore_shape), + "geometry": voronoi(offshore_locs, offshore_shape), "country": country, - } + }, + crs=n.crs, ) - offshore_regions_c = offshore_regions_c.loc[offshore_regions_c.area > 1e-2] + sel = offshore_regions_c.to_crs(3035).area > 10 # m2 + offshore_regions_c = offshore_regions_c.loc[sel] offshore_regions.append(offshore_regions_c) - shapes = pd.concat(onshore_regions, ignore_index=True) + shapes = pd.concat(onshore_regions, ignore_index=True).set_crs(n.crs) return onshore_regions, offshore_regions, shapes, offshore_shapes diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index 2a47198b..a6d99947 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -21,20 +21,12 @@ Relevant Settings Inputs: ------- - ``resources//temp_soil_total_elec_s_.nc``: Soil temperature (total) time series. -- ``resources//temp_soil_rural_elec_s_.nc``: Soil temperature (rural) time series. -- ``resources//temp_soil_urban_elec_s_.nc``: Soil temperature (urban) time series. - ``resources//temp_air_total_elec_s_.nc``: Ambient air temperature (total) time series. -- ``resources//temp_air_rural_elec_s_.nc``: Ambient air temperature (rural) time series. -- ``resources//temp_air_urban_elec_s_.nc``: Ambient air temperature (urban) time series. Outputs: -------- - ``resources/cop_soil_total_elec_s_.nc``: COP (ground-sourced) time series (total). -- ``resources/cop_soil_rural_elec_s_.nc``: COP (ground-sourced) time series (rural). -- ``resources/cop_soil_urban_elec_s_.nc``: COP (ground-sourced) time series (urban). - ``resources/cop_air_total_elec_s_.nc``: COP (air-sourced) time series (total). -- ``resources/cop_air_rural_elec_s_.nc``: COP (air-sourced) time series (rural). -- ``resources/cop_air_urban_elec_s_.nc``: COP (air-sourced) time series (urban). References @@ -67,12 +59,11 @@ if __name__ == "__main__": set_scenario_config(snakemake) - for area in ["total", "urban", "rural"]: - for source in ["air", "soil"]: - source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"]) + for source in ["air", "soil"]: + source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_total"]) - delta_T = snakemake.params.heat_pump_sink_T - source_T + delta_T = snakemake.params.heat_pump_sink_T - source_T - cop = coefficient_of_performance(delta_T, source) + cop = coefficient_of_performance(delta_T, source) - cop.to_netcdf(snakemake.output[f"cop_{source}_{area}"]) + cop.to_netcdf(snakemake.output[f"cop_{source}_total"]) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 1edb18ce..e8d6207c 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -103,7 +103,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5") + snakemake = mock_snakemake("build_cutout", cutout="europe-2013-sarah3-era5") configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/build_district_heat_share.py b/scripts/build_district_heat_share.py index d62d2ab0..7e8497d6 100644 --- a/scripts/build_district_heat_share.py +++ b/scripts/build_district_heat_share.py @@ -86,7 +86,7 @@ if __name__ == "__main__": urban_fraction = pd.concat([urban_fraction, dist_fraction_node], axis=1).max(axis=1) # difference of max potential and today's share of district heating - diff = (urban_fraction * central_fraction) - dist_fraction_node + diff = ((urban_fraction * central_fraction) - dist_fraction_node).clip(lower=0) progress = get( snakemake.config["sector"]["district_heating"]["progress"], investment_year ) diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index 67dbc986..ca43db3c 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -7,6 +7,7 @@ Build import locations for fossil gas from entry-points, LNG terminals and production sites with data from SciGRID_gas and Global Energy Monitor. """ +import json import logging import geopandas as gpd @@ -19,7 +20,8 @@ logger = logging.getLogger(__name__) def read_scigrid_gas(fn): df = gpd.read_file(fn) - df = pd.concat([df, df.param.apply(pd.Series)], axis=1) + expanded_param = df.param.apply(json.loads).apply(pd.Series) + df = pd.concat([df, expanded_param], axis=1) df.drop(["param", "uncertainty", "method"], axis=1, inplace=True) return df @@ -97,11 +99,11 @@ def build_gas_input_locations(gem_fn, entry_fn, sto_fn, countries): ~(entry.from_country.isin(countries) & entry.to_country.isin(countries)) & ~entry.name.str.contains("Tegelen") # only take non-EU entries | (entry.from_country == "NO") # malformed datapoint # entries from NO to GB - ] + ].copy() sto = read_scigrid_gas(sto_fn) remove_country = ["RU", "UA", "TR", "BY"] # noqa: F841 - sto = sto.query("country_code not in @remove_country") + sto = sto.query("country_code not in @remove_country").copy() # production sites inside the model scope prod = build_gem_prod_data(gem_fn) @@ -132,7 +134,8 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_gas_input_locations", simpl="", - clusters="128", + clusters="5", + configfiles="config/test/config.overnight.yaml", ) configure_logging(snakemake) @@ -162,7 +165,7 @@ if __name__ == "__main__": gas_input_nodes = gpd.sjoin(gas_input_locations, regions, how="left") - gas_input_nodes.rename(columns={"index_right": "bus"}, inplace=True) + gas_input_nodes.rename(columns={"name": "bus"}, inplace=True) gas_input_nodes.to_file(snakemake.output.gas_input_nodes, driver="GeoJSON") diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index 5e9a5c9a..e16096d3 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -7,6 +7,7 @@ Preprocess gas network based on data from bthe SciGRID_gas project (https://www.gas.scigrid.de/). """ +import json import logging import geopandas as gpd @@ -54,8 +55,9 @@ def diameter_to_capacity(pipe_diameter_mm): def load_dataset(fn): df = gpd.read_file(fn) - param = df.param.apply(pd.Series) - method = df.method.apply(pd.Series)[["diameter_mm", "max_cap_M_m3_per_d"]] + param = df.param.apply(json.loads).apply(pd.Series) + cols = ["diameter_mm", "max_cap_M_m3_per_d"] + method = df.method.apply(json.loads).apply(pd.Series)[cols] method.columns = method.columns + "_method" df = pd.concat([df, param, method], axis=1) to_drop = ["param", "uncertainty", "method", "tags"] diff --git a/scripts/build_gdp_pop_non_nuts3.py b/scripts/build_gdp_pop_non_nuts3.py new file mode 100644 index 00000000..d475aec9 --- /dev/null +++ b/scripts/build_gdp_pop_non_nuts3.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +Maps the per-capita GDP and population values to non-NUTS3 regions. + +The script takes as input the country code, a GeoDataFrame containing +the regions, and the file paths to the datasets containing the GDP and +POP values for non-NUTS3 countries. +""" + +import logging + +import geopandas as gpd +import numpy as np +import pandas as pd +import pypsa +import rasterio +import xarray as xr +from _helpers import configure_logging, set_scenario_config +from rasterio.mask import mask +from shapely.geometry import box + +logger = logging.getLogger(__name__) + + +def calc_gdp_pop(country, regions, gdp_non_nuts3, pop_non_nuts3): + """ + Calculate the GDP p.c. and population values for non NUTS3 regions. + + Parameters: + country (str): The two-letter country code of the non-NUTS3 region. + regions (GeoDataFrame): A GeoDataFrame containing the regions. + gdp_non_nuts3 (str): The file path to the dataset containing the GDP p.c values + for non NUTS3 countries (e.g. MD, UA) + pop_non_nuts3 (str): The file path to the dataset containing the POP values + for non NUTS3 countries (e.g. MD, UA) + + Returns: + tuple: A tuple containing two GeoDataFrames: + - gdp: A GeoDataFrame with the mean GDP p.c. values mapped to each bus. + - pop: A GeoDataFrame with the summed POP values mapped to each bus. + """ + regions = ( + regions.rename(columns={"name": "Bus"}) + .drop(columns=["x", "y"]) + .set_index("Bus") + ) + regions = regions[regions.country == country] + # Create a bounding box for UA, MD from region shape, including a buffer of 10000 metres + bounding_box = ( + gpd.GeoDataFrame(geometry=[box(*regions.total_bounds)], crs=regions.crs) + .to_crs(epsg=3857) + .buffer(10000) + .to_crs(regions.crs) + ) + + # GDP Mapping + logger.info(f"Mapping mean GDP p.c. to non-NUTS3 region: {country}") + with xr.open_dataset(gdp_non_nuts3) as src_gdp: + src_gdp = src_gdp.where( + (src_gdp.longitude >= bounding_box.bounds.minx.min()) + & (src_gdp.longitude <= bounding_box.bounds.maxx.max()) + & (src_gdp.latitude >= bounding_box.bounds.miny.min()) + & (src_gdp.latitude <= bounding_box.bounds.maxy.max()), + drop=True, + ) + gdp = src_gdp.to_dataframe().reset_index() + gdp = gdp.rename(columns={"GDP_per_capita_PPP": "gdp"}) + gdp = gdp[gdp.time == gdp.time.max()] + gdp_raster = gpd.GeoDataFrame( + gdp, + geometry=gpd.points_from_xy(gdp.longitude, gdp.latitude), + crs="EPSG:4326", + ) + gdp_mapped = gpd.sjoin(gdp_raster, regions, predicate="within") + gdp = ( + gdp_mapped.copy() + .groupby(["Bus", "country"]) + .agg({"gdp": "mean"}) + .reset_index(level=["country"]) + ) + + # Population Mapping + logger.info(f"Mapping summed population to non-NUTS3 region: {country}") + with rasterio.open(pop_non_nuts3) as src_pop: + # Mask the raster with the bounding box + out_image, out_transform = mask(src_pop, bounding_box, crop=True) + out_meta = src_pop.meta.copy() + out_meta.update( + { + "driver": "GTiff", + "height": out_image.shape[1], + "width": out_image.shape[2], + "transform": out_transform, + } + ) + masked_data = out_image[0] # Use the first band (rest is empty) + row_indices, col_indices = np.where(masked_data != src_pop.nodata) + values = masked_data[row_indices, col_indices] + + # Affine transformation from pixel coordinates to geo coordinates + x_coords, y_coords = rasterio.transform.xy(out_transform, row_indices, col_indices) + pop_raster = pd.DataFrame({"x": x_coords, "y": y_coords, "pop": values}) + pop_raster = gpd.GeoDataFrame( + pop_raster, + geometry=gpd.points_from_xy(pop_raster.x, pop_raster.y), + crs=src_pop.crs, + ) + pop_mapped = gpd.sjoin(pop_raster, regions, predicate="within") + pop = ( + pop_mapped.groupby(["Bus", "country"]) + .agg({"pop": "sum"}) + .reset_index() + .set_index("Bus") + ) + gdp_pop = regions.join(gdp.drop(columns="country"), on="Bus").join( + pop.drop(columns="country"), on="Bus" + ) + gdp_pop.fillna(0, inplace=True) + + return gdp_pop + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("build_gdp_pop_non_nuts3") + configure_logging(snakemake) + set_scenario_config(snakemake) + + n = pypsa.Network(snakemake.input.base_network) + regions = gpd.read_file(snakemake.input.regions) + + gdp_non_nuts3 = snakemake.input.gdp_non_nuts3 + pop_non_nuts3 = snakemake.input.pop_non_nuts3 + + subset = {"MD", "UA"}.intersection(snakemake.params.countries) + + gdp_pop = pd.concat( + [ + calc_gdp_pop(country, regions, gdp_non_nuts3, pop_non_nuts3) + for country in subset + ], + axis=0, + ) + + logger.info( + f"Exporting GDP and POP values for non-NUTS3 regions {snakemake.output}" + ) + gdp_pop.reset_index().to_file(snakemake.output, driver="GeoJSON") diff --git a/scripts/build_hourly_heat_demand.py b/scripts/build_hourly_heat_demand.py index 0dcf3524..8573a198 100644 --- a/scripts/build_hourly_heat_demand.py +++ b/scripts/build_hourly_heat_demand.py @@ -22,12 +22,12 @@ Inputs ------ - ``data/heat_load_profile_BDEW.csv``: Intraday heat profile for water and space heating demand for the residential and services sectors for weekends and weekdays. -- ``resources/daily_heat_demand__elec_s_.nc``: Daily heat demand per cluster. +- ``resources/daily_heat_demand_total_elec_s_.nc``: Daily heat demand per cluster. Outputs ------- -- ``resources/hourly_heat_demand__elec_s_.nc``: +- ``resources/hourly_heat_demand_total_elec_s_.nc``: """ from itertools import product @@ -41,10 +41,10 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake( - "build_hourly_heat_demands", + "build_hourly_heat_demand", scope="total", simpl="", - clusters=48, + clusters=5, ) set_scenario_config(snakemake) @@ -85,6 +85,6 @@ if __name__ == "__main__": heat_demand.index.name = "snapshots" - ds = heat_demand.stack().to_xarray() + ds = heat_demand.stack(future_stack=True).to_xarray() ds.to_netcdf(snakemake.output.heat_demand) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index bfbba35e..e6654728 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -116,7 +116,7 @@ def prepare_hotmaps_database(regions): gdf = gpd.sjoin(gdf, regions, how="inner", predicate="within") - gdf.rename(columns={"index_right": "bus"}, inplace=True) + gdf.rename(columns={"name": "bus"}, inplace=True) gdf["country"] = gdf.bus.str[:2] # the .sjoin can lead to duplicates if a geom is in two overlapping regions diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 9c8f2e98..b77ba8d6 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -184,7 +184,7 @@ def separate_basic_chemicals(demand, production): demand.drop(columns="Basic chemicals", inplace=True) - demand["HVC"].clip(lower=0, inplace=True) + demand["HVC"] = demand["HVC"].clip(lower=0) return demand @@ -248,7 +248,7 @@ if __name__ == "__main__": demand = add_non_eu28_industrial_energy_demand(countries, demand, production) # for format compatibility - demand = demand.stack(dropna=False).unstack(level=[0, 2]) + demand = demand.stack(future_stack=True).unstack(level=[0, 2]) # style and annotation demand.index.name = "TWh/a" diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 45806205..ec86a78d 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -301,7 +301,8 @@ def separate_basic_chemicals(demand, year): demand["Basic chemicals"] -= demand["Ammonia"] # EE, HR and LT got negative demand through subtraction - poor data - demand["Basic chemicals"].clip(lower=0.0, inplace=True) + col = "Basic chemicals" + demand[col] = demand[col].clip(lower=0.0) # assume HVC, methanol, chlorine production proportional to non-ammonia basic chemicals distribution_key = ( diff --git a/scripts/build_industry_sector_ratios_intermediate.py b/scripts/build_industry_sector_ratios_intermediate.py index 5fe042ab..ebbabdb2 100644 --- a/scripts/build_industry_sector_ratios_intermediate.py +++ b/scripts/build_industry_sector_ratios_intermediate.py @@ -129,11 +129,12 @@ def build_industry_sector_ratios_intermediate(): ] today_sector_ratios_ct.loc[:, ~missing_mask] = today_sector_ratios_ct.loc[ :, ~missing_mask - ].fillna(0) + ].fillna(future_sector_ratios) intermediate_sector_ratios[ct] = ( today_sector_ratios_ct * (1 - fraction_future) + future_sector_ratios * fraction_future ) + intermediate_sector_ratios = pd.concat(intermediate_sector_ratios, axis=1) intermediate_sector_ratios.to_csv(snakemake.output.industry_sector_ratios) diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index dc4cf2f8..ca664ed0 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -92,7 +92,9 @@ if __name__ == "__main__": # The first low density grid cells to reach rural fraction are rural asc_density_i = density_cells_ct.sort_values().index - asc_density_cumsum = pop_cells_ct[asc_density_i].cumsum() / pop_cells_ct.sum() + asc_density_cumsum = ( + pop_cells_ct.iloc[asc_density_i].cumsum() / pop_cells_ct.sum() + ) rural_fraction_ct = 1 - urban_fraction[ct] pop_ct_rural_b = asc_density_cumsum < rural_fraction_ct pop_ct_urban_b = ~pop_ct_rural_b diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 0aef89bc..57568f24 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -406,7 +406,7 @@ if __name__ == "__main__": if snakemake.wildcards.technology.startswith("offwind"): logger.info("Calculate underwater fraction of connections.") - offshore_shape = gpd.read_file(snakemake.input["offshore_shapes"]).unary_union + offshore_shape = gpd.read_file(snakemake.input["offshore_shapes"]).union_all() underwater_fraction = [] for bus in buses: p = centre_of_mass.sel(bus=bus).data diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index 52f545e9..44f4a738 100755 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -890,7 +890,7 @@ def calculate_gain_utilisation_factor(heat_transfer_perm2, Q_ht, Q_gain): Calculates gain utilisation factor nu. """ # time constant of the building tau [h] = c_m [Wh/(m^2K)] * 1 /(H_tr_e+H_tb*H_ve) [m^2 K /W] - tau = c_m / heat_transfer_perm2.T.groupby(axis=1).sum().T + tau = c_m / heat_transfer_perm2.groupby().sum() alpha = alpha_H_0 + (tau / tau_H_0) # heat balance ratio gamma = (1 / Q_ht).mul(Q_gain.sum(axis=1), axis=0) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 85afdaea..93a73858 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -91,7 +91,7 @@ def _get_country(target, **keys): return np.nan -def _simplify_polys(polys, minarea=0.1, tolerance=0.01, filterremote=True): +def _simplify_polys(polys, minarea=0.1, tolerance=None, filterremote=True): if isinstance(polys, MultiPolygon): polys = sorted(polys.geoms, key=attrgetter("area"), reverse=True) mainpoly = polys[0] @@ -106,7 +106,9 @@ def _simplify_polys(polys, minarea=0.1, tolerance=0.01, filterremote=True): ) else: polys = mainpoly - return polys.simplify(tolerance=tolerance) + if tolerance is not None: + polys = polys.simplify(tolerance=tolerance) + return polys def countries(naturalearth, country_list): @@ -124,7 +126,7 @@ def countries(naturalearth, country_list): df = df.loc[ df.name.isin(country_list) & ((df["scalerank"] == 0) | (df["scalerank"] == 5)) ] - s = df.set_index("name")["geometry"].map(_simplify_polys) + s = df.set_index("name")["geometry"].map(_simplify_polys).set_crs(df.crs) if "RS" in country_list: s["RS"] = s["RS"].union(s.pop("KV")) # cleanup shape union @@ -145,7 +147,8 @@ def eez(country_shapes, eez, country_list): lambda s: _simplify_polys(s, filterremote=False) ) s = gpd.GeoSeries( - {k: v for k, v in s.items() if v.distance(country_shapes[k]) < 1e-3} + {k: v for k, v in s.items() if v.distance(country_shapes[k]) < 1e-3}, + crs=df.crs, ) s = s.to_frame("geometry") s.index.name = "name" @@ -156,7 +159,7 @@ def country_cover(country_shapes, eez_shapes=None): shapes = country_shapes if eez_shapes is not None: shapes = pd.concat([shapes, eez_shapes]) - europe_shape = shapes.unary_union + europe_shape = shapes.union_all() if isinstance(europe_shape, MultiPolygon): europe_shape = max(europe_shape.geoms, key=attrgetter("area")) return Polygon(shell=europe_shape.exterior) @@ -235,11 +238,11 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp): [["BA1", "BA", 3871.0], ["RS1", "RS", 7210.0], ["AL1", "AL", 2893.0]], columns=["NUTS_ID", "country", "pop"], geometry=gpd.GeoSeries(), + crs=df.crs, ) - manual["geometry"] = manual["country"].map(country_shapes) + manual["geometry"] = manual["country"].map(country_shapes.to_crs(df.crs)) manual = manual.dropna() manual = manual.set_index("NUTS_ID") - manual = manual.set_crs("ETRS89") df = pd.concat([df, manual], sort=False) @@ -265,7 +268,8 @@ if __name__ == "__main__": offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes) europe_shape = gpd.GeoDataFrame( - geometry=[country_cover(country_shapes, offshore_shapes.geometry)] + geometry=[country_cover(country_shapes, offshore_shapes.geometry)], + crs=country_shapes.crs, ) europe_shape.reset_index().to_file(snakemake.output.europe_shape) diff --git a/scripts/build_shipping_demand.py b/scripts/build_shipping_demand.py index b50cd316..d8c960ae 100644 --- a/scripts/build_shipping_demand.py +++ b/scripts/build_shipping_demand.py @@ -45,9 +45,7 @@ if __name__ == "__main__": # assign ports to nearest region p = european_ports.to_crs(3857) r = regions.to_crs(3857) - outflows = ( - p.sjoin_nearest(r).groupby("index_right").properties_outflows.sum().div(1e3) - ) + outflows = p.sjoin_nearest(r).groupby("name").properties_outflows.sum().div(1e3) # calculate fraction of each country's port outflows countries = outflows.index.str[:2] diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index 493bd08f..8e07ee87 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -25,15 +25,15 @@ Relevant Settings Inputs ------ -- ``resources//pop_layout_.nc``: +- ``resources//pop_layout_total.nc``: - ``resources//regions_onshore_elec_s_.geojson``: - ``cutout``: Weather data cutout, as specified in config Outputs ------- -- ``resources/temp_soil__elec_s_.nc``: -- ``resources/temp_air__elec_s_.nc` +- ``resources/temp_soil_total_elec_s_.nc``: +- ``resources/temp_air_total_elec_s_.nc` """ import atlite diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index 19585aa9..b95c4580 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -41,9 +41,9 @@ def build_clustered_gas_network(df, bus_regions, length_factor=1.25): for i in [0, 1]: gdf = gpd.GeoDataFrame(geometry=df[f"point{i}"], crs="EPSG:4326") - bus_mapping = gpd.sjoin( - gdf, bus_regions, how="left", predicate="within" - ).index_right + bus_mapping = gpd.sjoin(gdf, bus_regions, how="left", predicate="within")[ + "name" + ] bus_mapping = bus_mapping.groupby(bus_mapping.index).first() df[f"bus{i}"] = bus_mapping @@ -58,6 +58,9 @@ def build_clustered_gas_network(df, bus_regions, length_factor=1.25): # drop pipes within the same region df = df.loc[df.bus1 != df.bus0] + if df.empty: + return df + # recalculate lengths as center to center * length factor df["length"] = df.apply( lambda p: length_factor diff --git a/scripts/determine_availability_matrix_MD_UA.py b/scripts/determine_availability_matrix_MD_UA.py index 80c04083..0e7962ab 100644 --- a/scripts/determine_availability_matrix_MD_UA.py +++ b/scripts/determine_availability_matrix_MD_UA.py @@ -8,16 +8,15 @@ Create land elibility analysis for Ukraine and Moldova with different datasets. import functools import logging +import os import time +from tempfile import NamedTemporaryFile import atlite import fiona import geopandas as gpd -import matplotlib.pyplot as plt import numpy as np from _helpers import configure_logging, set_scenario_config -from atlite.gis import shape_availability -from rasterio.plot import show logger = logging.getLogger(__name__) @@ -40,7 +39,7 @@ if __name__ == "__main__": configure_logging(snakemake) set_scenario_config(snakemake) - nprocesses = None # snakemake.config["atlite"].get("nprocesses") + nprocesses = int(snakemake.threads) noprogress = not snakemake.config["atlite"].get("show_progress", True) config = snakemake.config["renewable"][snakemake.wildcards.technology] @@ -48,7 +47,9 @@ if __name__ == "__main__": regions = ( gpd.read_file(snakemake.input.regions).set_index("name").rename_axis("bus") ) - buses = regions.index + # Limit to "UA" and "MD" regions + buses = regions.loc[regions["country"].isin(["UA", "MD"])].index.values + regions = regions.loc[buses] excluder = atlite.ExclusionContainer(crs=3035, res=100) @@ -93,8 +94,15 @@ if __name__ == "__main__": bbox=regions.geometry, layer=layer, ).to_crs(3035) + + # temporary file needed for parallelization + with NamedTemporaryFile(suffix=".geojson", delete=False) as f: + plg_tmp_fn = f.name if not wdpa.empty: - excluder.add_geometry(wdpa.geometry) + wdpa[["geometry"]].to_file(plg_tmp_fn) + while not os.path.exists(plg_tmp_fn): + time.sleep(1) + excluder.add_geometry(plg_tmp_fn) layer = get_wdpa_layer_name(wdpa_fn, "points") wdpa_pts = gpd.read_file( @@ -107,8 +115,15 @@ if __name__ == "__main__": wdpa_pts = wdpa_pts.set_geometry( wdpa_pts["geometry"].buffer(wdpa_pts["buffer_radius"]) ) + + # temporary file needed for parallelization + with NamedTemporaryFile(suffix=".geojson", delete=False) as f: + pts_tmp_fn = f.name if not wdpa_pts.empty: - excluder.add_geometry(wdpa_pts.geometry) + wdpa_pts[["geometry"]].to_file(pts_tmp_fn) + while not os.path.exists(pts_tmp_fn): + time.sleep(1) + excluder.add_geometry(pts_tmp_fn) if "max_depth" in config: # lambda not supported for atlite + multiprocessing @@ -144,16 +159,10 @@ if __name__ == "__main__": else: availability = cutout.availabilitymatrix(regions, excluder, **kwargs) - regions_geometry = regions.to_crs(3035).geometry - band, transform = shape_availability(regions_geometry, excluder) - fig, ax = plt.subplots(figsize=(4, 8)) - gpd.GeoSeries(regions_geometry.unary_union).plot(ax=ax, color="none") - show(band, transform=transform, cmap="Greens", ax=ax) - plt.axis("off") - plt.savefig(snakemake.output.availability_map, bbox_inches="tight", dpi=500) + for fn in [pts_tmp_fn, plg_tmp_fn]: + if os.path.exists(fn): + os.remove(fn) - # Limit results only to buses for UA and MD - buses = regions.loc[regions["country"].isin(["UA", "MD"])].index.values availability = availability.sel(bus=buses) # Save and plot for verification diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py index 76bd4ad0..8e56e5d4 100644 --- a/scripts/make_summary_perfect.py +++ b/scripts/make_summary_perfect.py @@ -631,7 +631,7 @@ def calculate_co2_emissions(n, label, df): weightings = n.snapshot_weightings.generators.mul( n.investment_period_weightings["years"] .reindex(n.snapshots) - .fillna(method="bfill") + .bfill() .fillna(1.0), axis=0, ) diff --git a/scripts/plot_validation_electricity_production.py b/scripts/plot_validation_electricity_production.py index 5a68cfa5..f842bea3 100644 --- a/scripts/plot_validation_electricity_production.py +++ b/scripts/plot_validation_electricity_production.py @@ -70,7 +70,7 @@ if __name__ == "__main__": optimized = optimized[["Generator", "StorageUnit"]].droplevel(0, axis=1) optimized = optimized.rename(columns=n.buses.country, level=0) optimized = optimized.rename(columns=carrier_groups, level=1) - optimized = optimized.groupby(axis=1, level=[0, 1]).sum() + optimized = optimized.T.groupby(level=[0, 1]).sum().T data = pd.concat([historic, optimized], keys=["Historic", "Optimized"], axis=1) data.columns.names = ["Kind", "Country", "Carrier"] diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 00cb00bf..382e633d 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -137,9 +137,7 @@ def add_emission_prices(n, emission_prices={"co2": 0.0}, exclude_co2=False): def add_dynamic_emission_prices(n): co2_price = pd.read_csv(snakemake.input.co2_price, index_col=0, parse_dates=True) co2_price = co2_price[~co2_price.index.duplicated()] - co2_price = ( - co2_price.reindex(n.snapshots).fillna(method="ffill").fillna(method="bfill") - ) + co2_price = co2_price.reindex(n.snapshots).ffill().bfill() emissions = ( n.generators.carrier.map(n.carriers.co2_emissions) / n.generators.efficiency diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index c5e4caf9..efc95700 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -250,7 +250,7 @@ def adjust_stores(n): n.stores.loc[cyclic_i, "e_cyclic_per_period"] = True n.stores.loc[cyclic_i, "e_cyclic"] = False # non cyclic store assumptions - non_cyclic_store = ["co2", "co2 stored", "solid biomass", "biogas", "Li ion"] + non_cyclic_store = ["co2", "co2 stored", "solid biomass", "biogas", "EV battery"] co2_i = n.stores[n.stores.carrier.isin(non_cyclic_store)].index n.stores.loc[co2_i, "e_cyclic_per_period"] = False n.stores.loc[co2_i, "e_cyclic"] = False diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index e4fc3299..35de1fe7 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -696,6 +696,7 @@ def add_co2_tracking(n, costs, options): e_nom_extendable=True, e_nom_max=e_nom_max, capital_cost=options["co2_sequestration_cost"], + marginal_cost=-0.1, bus=sequestration_buses, lifetime=options["co2_sequestration_lifetime"], carrier="co2 sequestered", @@ -1245,12 +1246,14 @@ def add_storage_and_grids(n, costs): gas_pipes["p_nom_min"] = 0.0 # 0.1 EUR/MWkm/a to prefer decommissioning to address degeneracy gas_pipes["capital_cost"] = 0.1 * gas_pipes.length + gas_pipes["p_nom_extendable"] = True else: gas_pipes["p_nom_max"] = np.inf gas_pipes["p_nom_min"] = gas_pipes.p_nom gas_pipes["capital_cost"] = ( gas_pipes.length * costs.at["CH4 (g) pipeline", "fixed"] ) + gas_pipes["p_nom_extendable"] = False n.madd( "Link", @@ -1259,14 +1262,14 @@ def add_storage_and_grids(n, costs): bus1=gas_pipes.bus1 + " gas", p_min_pu=gas_pipes.p_min_pu, p_nom=gas_pipes.p_nom, - p_nom_extendable=True, + p_nom_extendable=gas_pipes.p_nom_extendable, p_nom_max=gas_pipes.p_nom_max, p_nom_min=gas_pipes.p_nom_min, length=gas_pipes.length, capital_cost=gas_pipes.capital_cost, tags=gas_pipes.name, carrier="gas pipeline", - lifetime=costs.at["CH4 (g) pipeline", "lifetime"], + lifetime=np.inf, ) # remove fossil generators where there is neither @@ -1548,14 +1551,14 @@ def add_EVs( temperature, ): - n.add("Carrier", "Li ion") + n.add("Carrier", "EV battery") n.madd( "Bus", spatial.nodes, suffix=" EV battery", location=spatial.nodes, - carrier="Li ion", + carrier="EV battery", unit="MWh_el", ) @@ -1628,9 +1631,9 @@ def add_EVs( n.madd( "Store", spatial.nodes, - suffix=" battery storage", + suffix=" EV battery", bus=spatial.nodes + " EV battery", - carrier="battery storage", + carrier="EV battery", e_cyclic=True, e_nom=e_nom, e_max_pu=1, @@ -2780,10 +2783,11 @@ def add_industry(n, costs): ) domestic_navigation = pop_weighted_energy_totals.loc[ - nodes, "total domestic navigation" + nodes, ["total domestic navigation"] ].squeeze() international_navigation = ( - pd.read_csv(snakemake.input.shipping_demand, index_col=0).squeeze() * nyears + pd.read_csv(snakemake.input.shipping_demand, index_col=0).squeeze(axis=1) + * nyears ) all_navigation = domestic_navigation + international_navigation p_set = all_navigation * 1e6 / nhours @@ -3363,7 +3367,7 @@ def add_waste_heat(n): ) n.links.loc[urban_central + " Fischer-Tropsch", "efficiency3"] = ( 0.95 - n.links.loc[urban_central + " Fischer-Tropsch", "efficiency"] - ) + ) * options["use_fischer_tropsch_waste_heat"] if options["use_methanation_waste_heat"] and "Sabatier" in link_carriers: n.links.loc[urban_central + " Sabatier", "bus3"] = ( @@ -3371,7 +3375,7 @@ def add_waste_heat(n): ) n.links.loc[urban_central + " Sabatier", "efficiency3"] = ( 0.95 - n.links.loc[urban_central + " Sabatier", "efficiency"] - ) + ) * options["use_methanation_waste_heat"] # DEA quotes 15% of total input (11% of which are high-value heat) if options["use_haber_bosch_waste_heat"] and "Haber-Bosch" in link_carriers: @@ -3388,7 +3392,7 @@ def add_waste_heat(n): ) n.links.loc[urban_central + " Haber-Bosch", "efficiency3"] = ( 0.15 * total_energy_input / electricity_input - ) + ) * options["use_haber_bosch_waste_heat"] if ( options["use_methanolisation_waste_heat"] @@ -3400,11 +3404,11 @@ def add_waste_heat(n): n.links.loc[urban_central + " methanolisation", "efficiency4"] = ( costs.at["methanolisation", "heat-output"] / costs.at["methanolisation", "hydrogen-input"] - ) + ) * options["use_methanolisation_waste_heat"] # TODO integrate usable waste heat efficiency into technology-data from DEA if ( - options.get("use_electrolysis_waste_heat", False) + options["use_electrolysis_waste_heat"] and "H2 Electrolysis" in link_carriers ): n.links.loc[urban_central + " H2 Electrolysis", "bus2"] = ( @@ -3412,7 +3416,7 @@ def add_waste_heat(n): ) n.links.loc[urban_central + " H2 Electrolysis", "efficiency2"] = ( 0.84 - n.links.loc[urban_central + " H2 Electrolysis", "efficiency"] - ) + ) * options["use_electrolysis_waste_heat"] if options["use_fuel_cell_waste_heat"] and "H2 Fuel Cell" in link_carriers: n.links.loc[urban_central + " H2 Fuel Cell", "bus2"] = ( @@ -3420,7 +3424,7 @@ def add_waste_heat(n): ) n.links.loc[urban_central + " H2 Fuel Cell", "efficiency2"] = ( 0.95 - n.links.loc[urban_central + " H2 Fuel Cell", "efficiency"] - ) + ) * options["use_fuel_cell_waste_heat"] def add_agriculture(n, costs): @@ -3731,7 +3735,7 @@ def lossy_bidirectional_links(n, carrier, efficiencies={}): rev_links.index = rev_links.index.map(lambda x: x + "-reversed") n.links = pd.concat([n.links, rev_links], sort=False) - n.links["reversed"] = n.links["reversed"].fillna(False) + n.links["reversed"] = n.links["reversed"].fillna(False).infer_objects(copy=False) n.links["length_original"] = n.links["length_original"].fillna(n.links.length) # do compression losses after concatenation to take electricity consumption at bus0 in either direction @@ -3956,12 +3960,11 @@ if __name__ == "__main__": snakemake = mock_snakemake( "prepare_sector_network", - # configfiles="test/config.overnight.yaml", simpl="", opts="", - clusters="37", - ll="v1.0", - sector_opts="730H-T-H-B-I-A-dist1", + clusters="1", + ll="vopt", + sector_opts="", planning_horizons="2050", ) diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index e2736f63..63b71dd5 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -48,7 +48,7 @@ if __name__ == "__main__": configure_logging(snakemake) set_scenario_config(snakemake) - url = "https://zenodo.org/records/10973944/files/bundle.tar.xz" + url = "https://zenodo.org/records/12760663/files/bundle.tar.xz" tarball_fn = Path(f"{rootpath}/bundle.tar.xz") to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent diff --git a/scripts/retrieve_eurostat_data.py b/scripts/retrieve_eurostat_data.py index 085da064..9ee32e6a 100644 --- a/scripts/retrieve_eurostat_data.py +++ b/scripts/retrieve_eurostat_data.py @@ -28,7 +28,8 @@ if __name__ == "__main__": disable_progress = snakemake.config["run"].get("disable_progressbar", False) url_eurostat = ( - "https://ec.europa.eu/eurostat/documents/38154/4956218/Balances-April2023.zip" + # "https://ec.europa.eu/eurostat/documents/38154/4956218/Balances-April2023.zip" # link down + "https://tubcloud.tu-berlin.de/s/prkJpL7B9M3cDPb/download/Balances-April2023.zip" ) tarball_fn = Path(f"{rootpath}/data/eurostat/eurostat_2023.zip") to_fn = Path(f"{rootpath}/data/eurostat/Balances-April2023/") diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ba2fac7f..a2391ea2 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -155,7 +155,7 @@ def _add_land_use_constraint(n): existing_large, "p_nom_min" ] - n.generators.p_nom_max.clip(lower=0, inplace=True) + n.generators["p_nom_max"] = n.generators["p_nom_max"].clip(lower=0) def _add_land_use_constraint_m(n, planning_horizons, config): @@ -207,7 +207,7 @@ def _add_land_use_constraint_m(n, planning_horizons, config): existing_large, "p_nom_min" ] - n.generators.p_nom_max.clip(lower=0, inplace=True) + n.generators["p_nom_max"] = n.generators["p_nom_max"].clip(lower=0) def add_solar_potential_constraints(n, config): @@ -471,6 +471,22 @@ def prepare_network( p_nom=1e9, # kW ) + if solve_opts.get("curtailment_mode"): + n.add("Carrier", "curtailment", color="#fedfed", nice_name="Curtailment") + n.generators_t.p_min_pu = n.generators_t.p_max_pu + buses_i = n.buses.query("carrier == 'AC'").index + n.madd( + "Generator", + buses_i, + suffix=" curtailment", + bus=buses_i, + p_min_pu=-1, + p_max_pu=0, + marginal_cost=-0.1, + carrier="curtailment", + p_nom=1e6, + ) + if solve_opts.get("noisy_costs"): for t in n.iterate_components(): # if 'capital_cost' in t.df: