diff --git a/flupdt/cli.py b/flupdt/cli.py index 1a94c0c..a99e9b4 100644 --- a/flupdt/cli.py +++ b/flupdt/cli.py @@ -33,6 +33,17 @@ def parse_inputs() -> argparse.Namespace: default=None, help="location of post.json for comparison. defaults to /post.json", ) + parser.add_argument( + "--compare-output-to-file", + action="store_true", + help="whether to output comparison output to a file", + ) + parser.add_argument( + "--compare-output-file", + metavar="compare-output-file-path", + default=None, + help="location of comparison output. defaults to /post-diff. implies --compare-output-to-file.", + ) args = parser.parse_args() if args.compare_pre_json is None: @@ -41,4 +52,10 @@ def parse_inputs() -> argparse.Namespace: if args.compare_post_json is None: args.compare_post_json = args.flake_path + "/post.json" + if args.compare_output_to_file and args.compare_output_file is None: + args.compare_output_file = args.flake_path + "/post-diff" + + if args.compare_output_file is not None: + args.compare_output_to_file = True + return args diff --git a/flupdt/flake_diff.py b/flupdt/flake_diff.py index f72a25d..fa16c80 100644 --- a/flupdt/flake_diff.py +++ b/flupdt/flake_diff.py @@ -7,8 +7,8 @@ from flupdt.common import bash_wrapper def compare_derivations( - path_to_flake: str, path_to_pre_drv: str, path_to_post_drv: str -) -> list[str]: + path_to_flake: str, path_to_pre_drv: str, path_to_post_drv: str, *, soft_failure: bool = True +) -> str: """Gets all derivations present in a flake. :param path_to_flake: path to flake to be checked @@ -19,12 +19,15 @@ def compare_derivations( if nvd_path is None: status_msg = "nvd is not available in the PATH, please verify that it is installed" raise RuntimeError(status_msg) - diff_output = bash_wrapper( + stdout, stderr, returncode = bash_wrapper( f"{nvd_path} diff {path_to_pre_drv} {path_to_post_drv}", path=path_to_flake ) - logging.debug(diff_output[0]) - logging.debug(diff_output[1]) - logging.debug(diff_output[2]) + if returncode != 0: + log_func = logging.error + if soft_failure: + log_func = logging.warning + logging.warning(f"diff failed for {path_to_pre_drv} and {path_to_post_drv}") + log_func(stderr) - return diff_output + return stdout diff --git a/flupdt/main.py b/flupdt/main.py index 46f018c..edff707 100755 --- a/flupdt/main.py +++ b/flupdt/main.py @@ -3,6 +3,7 @@ """Default processing of flake outputs for evaluating flake updates.""" import logging +import os from argparse import Namespace from pathlib import Path @@ -35,6 +36,76 @@ def batch_eval(args: Namespace, flake_path: str, derivations: list[str]) -> None dump(drv_map, f) +def compare_drvs(args: Namespace) -> None: + """Compares derivation jsons using nvd. + + param args: argparse namespace + """ + pre_json_dict = {} + post_json_dict = {} + from json import load + + with ( + Path.open(args.compare_pre_json, "r") as pre, + Path.open(args.compare_post_json, "r") as post, + ): + pre_json_dict = load(pre) + post_json_dict = load(post) + + logging.debug(f"pre-snapshot derivations: {pre_json_dict}") + logging.debug(f"post-snapshot derivations: {post_json_dict}") + + pre_json_keys = set(pre_json_dict.keys()) + post_json_keys = set(post_json_dict.keys()) + + common_keys_to_eval = pre_json_keys.union(post_json_keys) + + missing_post_keys = pre_json_keys.difference(common_keys_to_eval) + missing_pre_keys = post_json_keys.difference(common_keys_to_eval) + + if missing_pre_keys: + logging.warning(f"Following outputs are missing from pre-snapshot: {missing_pre_keys}") + if missing_post_keys: + logging.warning(f"Following outputs are missing from post-snapshot: {missing_post_keys}") + + logging.info(f"Evaluating the following outputs for differences: {common_keys_to_eval}") + + out_file: str = os.devnull + if args.compare_output_to_file: + out_file = args.compare_output_file + + out_file_path = Path(out_file) + with out_file_path.open("w") as f: + for output_key in common_keys_to_eval: + comp_out = compare_derivations( + args.flake_path, pre_json_dict[output_key], post_json_dict[output_key] + ) + f.write(f"comparing {output_key}:" + "\n") + if comp_out: + f.write(comp_out + "\n\n") + else: + f.write("comparison output is empty, please check script logs\n\n") + + +def build_or_eval(args: Namespace) -> None: + """Builds or evaluates all outputs in a flake. + + param args: argparse namespace + """ + flake_path = args.flake_path + derivations, hydra_jobs = partition( + lambda s: s.startswith("hydraJobs"), get_derivations(flake_path) + ) + derivations, hydra_jobs = list(derivations), list(hydra_jobs) + logging.info(f"derivations: {list(derivations)}") + batch_eval(args, flake_path, derivations) + + if not args.keep_hydra: + logging.info("--keep-hydra flag is not specified, removing Hydra jobs") + else: + batch_eval(args, flake_path, hydra_jobs) + + def main() -> None: """Sets up logging, parses args, and runs evaluation routine. @@ -44,53 +115,9 @@ def main() -> None: configure_logger("DEBUG") args = parse_inputs() if args.compare_drvs: - pre_json_dict = {} - post_json_dict = {} - from json import load - - with ( - Path.open(args.compare_pre_json, "r") as pre, - Path.open(args.compare_post_json, "r") as post, - ): - pre_json_dict = load(pre) - post_json_dict = load(post) - - logging.debug(f"pre-snapshot derivations: {pre_json_dict}") - logging.debug(f"post-snapshot derivations: {post_json_dict}") - - pre_json_keys = set(pre_json_dict.keys()) - post_json_keys = set(post_json_dict.keys()) - - common_keys_to_eval = pre_json_keys.union(post_json_keys) - - missing_post_keys = pre_json_keys.difference(common_keys_to_eval) - missing_pre_keys = post_json_keys.difference(common_keys_to_eval) - - if missing_pre_keys: - logging.warning(f"Following outputs are missing from pre-snapshot: {missing_pre_keys}") - if missing_post_keys: - logging.warning(f"Following outputs are missing from post-snapshot: {missing_post_keys}") - - logging.info(f"Evaluating the following outputs for differences: {common_keys_to_eval}") - - for output_key in common_keys_to_eval: - compare_derivations(args.flake_path, pre_json_dict[output_key], post_json_dict[output_key]) - - - + compare_drvs(args) else: - flake_path = args.flake_path - derivations, hydra_jobs = partition( - lambda s: s.startswith("hydraJobs"), get_derivations(flake_path) - ) - derivations, hydra_jobs = list(derivations), list(hydra_jobs) - logging.info(f"derivations: {list(derivations)}") - batch_eval(args, flake_path, derivations) - - if not args.keep_hydra: - logging.info("--keep-hydra flag is not specified, removing Hydra jobs") - else: - batch_eval(args, flake_path, hydra_jobs) + build_or_eval(args) if __name__ == "__main__": diff --git a/poetry.lock b/poetry.lock index fd7137d..8d7961f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "greenlet" @@ -6,6 +6,8 @@ version = "3.1.1" description = "Lightweight in-process concurrent programming" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.14\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")" files = [ {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"}, {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"}, @@ -88,29 +90,30 @@ test = ["objgraph", "psutil"] [[package]] name = "ruff" -version = "0.5.1" +version = "0.8.6" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ - {file = "ruff-0.5.1-py3-none-linux_armv6l.whl", hash = "sha256:6ecf968fcf94d942d42b700af18ede94b07521bd188aaf2cd7bc898dd8cb63b6"}, - {file = "ruff-0.5.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:204fb0a472f00f2e6280a7c8c7c066e11e20e23a37557d63045bf27a616ba61c"}, - {file = "ruff-0.5.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d235968460e8758d1e1297e1de59a38d94102f60cafb4d5382033c324404ee9d"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38beace10b8d5f9b6bdc91619310af6d63dd2019f3fb2d17a2da26360d7962fa"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e478d2f09cf06add143cf8c4540ef77b6599191e0c50ed976582f06e588c994"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0368d765eec8247b8550251c49ebb20554cc4e812f383ff9f5bf0d5d94190b0"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:3a9a9a1b582e37669b0138b7c1d9d60b9edac880b80eb2baba6d0e566bdeca4d"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bdd9f723e16003623423affabcc0a807a66552ee6a29f90eddad87a40c750b78"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be9fd62c1e99539da05fcdc1e90d20f74aec1b7a1613463ed77870057cd6bd96"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e216fc75a80ea1fbd96af94a6233d90190d5b65cc3d5dfacf2bd48c3e067d3e1"}, - {file = "ruff-0.5.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c4c2112e9883a40967827d5c24803525145e7dab315497fae149764979ac7929"}, - {file = "ruff-0.5.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dfaf11c8a116394da3b65cd4b36de30d8552fa45b8119b9ef5ca6638ab964fa3"}, - {file = "ruff-0.5.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d7ceb9b2fe700ee09a0c6b192c5ef03c56eb82a0514218d8ff700f6ade004108"}, - {file = "ruff-0.5.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bac6288e82f6296f82ed5285f597713acb2a6ae26618ffc6b429c597b392535c"}, - {file = "ruff-0.5.1-py3-none-win32.whl", hash = "sha256:5c441d9c24ec09e1cb190a04535c5379b36b73c4bc20aa180c54812c27d1cca4"}, - {file = "ruff-0.5.1-py3-none-win_amd64.whl", hash = "sha256:b1789bf2cd3d1b5a7d38397cac1398ddf3ad7f73f4de01b1e913e2abc7dfc51d"}, - {file = "ruff-0.5.1-py3-none-win_arm64.whl", hash = "sha256:2875b7596a740cbbd492f32d24be73e545a4ce0a3daf51e4f4e609962bfd3cd2"}, - {file = "ruff-0.5.1.tar.gz", hash = "sha256:3164488aebd89b1745b47fd00604fb4358d774465f20d1fcd907f9c0fc1b0655"}, + {file = "ruff-0.8.6-py3-none-linux_armv6l.whl", hash = "sha256:defed167955d42c68b407e8f2e6f56ba52520e790aba4ca707a9c88619e580e3"}, + {file = "ruff-0.8.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:54799ca3d67ae5e0b7a7ac234baa657a9c1784b48ec954a094da7c206e0365b1"}, + {file = "ruff-0.8.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e88b8f6d901477c41559ba540beeb5a671e14cd29ebd5683903572f4b40a9807"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0509e8da430228236a18a677fcdb0c1f102dd26d5520f71f79b094963322ed25"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:91a7ddb221779871cf226100e677b5ea38c2d54e9e2c8ed847450ebbdf99b32d"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:248b1fb3f739d01d528cc50b35ee9c4812aa58cc5935998e776bf8ed5b251e75"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:bc3c083c50390cf69e7e1b5a5a7303898966be973664ec0c4a4acea82c1d4315"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52d587092ab8df308635762386f45f4638badb0866355b2b86760f6d3c076188"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:61323159cf21bc3897674e5adb27cd9e7700bab6b84de40d7be28c3d46dc67cf"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ae4478b1471fc0c44ed52a6fb787e641a2ac58b1c1f91763bafbc2faddc5117"}, + {file = "ruff-0.8.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0c000a471d519b3e6cfc9c6680025d923b4ca140ce3e4612d1a2ef58e11f11fe"}, + {file = "ruff-0.8.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9257aa841e9e8d9b727423086f0fa9a86b6b420fbf4bf9e1465d1250ce8e4d8d"}, + {file = "ruff-0.8.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:45a56f61b24682f6f6709636949ae8cc82ae229d8d773b4c76c09ec83964a95a"}, + {file = "ruff-0.8.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:496dd38a53aa173481a7d8866bcd6451bd934d06976a2505028a50583e001b76"}, + {file = "ruff-0.8.6-py3-none-win32.whl", hash = "sha256:e169ea1b9eae61c99b257dc83b9ee6c76f89042752cb2d83486a7d6e48e8f764"}, + {file = "ruff-0.8.6-py3-none-win_amd64.whl", hash = "sha256:f1d70bef3d16fdc897ee290d7d20da3cbe4e26349f62e8a0274e7a3f4ce7a905"}, + {file = "ruff-0.8.6-py3-none-win_arm64.whl", hash = "sha256:7d7fc2377a04b6e04ffe588caad613d0c460eb2ecba4c0ccbbfe2bc973cbc162"}, + {file = "ruff-0.8.6.tar.gz", hash = "sha256:dcad24b81b62650b0eb8814f576fc65cfee8674772a6e24c9b747911801eeaa5"}, ] [[package]] @@ -119,6 +122,7 @@ version = "2.0.38" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "SQLAlchemy-2.0.38-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5e1d9e429028ce04f187a9f522818386c8b076723cdbe9345708384f49ebcec6"}, {file = "SQLAlchemy-2.0.38-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b87a90f14c68c925817423b0424381f0e16d80fc9a1a1046ef202ab25b19a444"}, @@ -214,12 +218,13 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] [metadata] -lock-version = "2.0" -python-versions = "^3.11" -content-hash = "6ebb0770da484a771f8c11e7ba8100b3180b80c107e8b56d240036e7d2689506" +lock-version = "2.1" +python-versions = "^3.12" +content-hash = "ff4f7ce6f8bf20026c9f29da5eabc2f4e5de408fee8f39a40e8d8c25c9e13b74" diff --git a/pyproject.toml b/pyproject.toml index a8e4a66..b2340e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ readme = "README.md" packages = [{ include = "flupdt" }] [tool.poetry.dependencies] -python = "^3.11" +python = "^3.12" sqlalchemy = "^2.0.31" [tool.poetry.scripts] @@ -20,7 +20,7 @@ flupdt = "flupdt.main:main" flake-update-diff = "flupdt.main:main" [tool.poetry.group.dev.dependencies] -ruff = "0.5.1" +ruff = "0.8.6" [build-system] @@ -32,13 +32,11 @@ build-backend = "poetry.core.masonry.api" line-length = 100 indent-width = 4 -target-version = "py39" +target-version = "py312" [tool.ruff.lint] select = ["ALL"] ignore = [ - "ANN101", # (perm) this rule is deprecated - "ANN102", # (perm) this rule is deprecated "G004", # (perm) this is a performance nit "COM812", # https://docs.astral.sh/ruff/rules/missing-trailing-comma/ "ISC001", # https://docs.astral.sh/ruff/rules/single-line-implicit-string-concatenation/