ahuston-0/flake-update-diff
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feature/diff_output #5

Merged
ahuston-0 merged 1 commits from feature/diff_output into main 2025-03-08 14:50:51 -05:00
Conversation 0 Commits 1 Files Changed 5 +131 -81
5 changed files with 131 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
flupdt/cli.py
View File

@ -33,6 +33,17 @@ def parse_inputs() -> argparse.Namespace:
default=None, default=None,
help="location of post.json for comparison. defaults to <flake_path>/post.json", help="location of post.json for comparison. defaults to <flake_path>/post.json",
) )
parser.add_argument(
"--compare-output-to-file",
action="store_true",
help="whether to output comparison output to a file",
)
parser.add_argument(
"--compare-output-file",
metavar="compare-output-file-path",
default=None,
help="location of comparison output. defaults to <flake_path>/post-diff. implies --compare-output-to-file.",
)
args = parser.parse_args() args = parser.parse_args()
if args.compare_pre_json is None: if args.compare_pre_json is None:
@ -41,4 +52,10 @@ def parse_inputs() -> argparse.Namespace:
if args.compare_post_json is None: if args.compare_post_json is None:
args.compare_post_json = args.flake_path + "/post.json" args.compare_post_json = args.flake_path + "/post.json"
if args.compare_output_to_file and args.compare_output_file is None:
args.compare_output_file = args.flake_path + "/post-diff"
if args.compare_output_file is not None:
args.compare_output_to_file = True
return args return args

17
flupdt/flake_diff.py
View File

@ -7,8 +7,8 @@ from flupdt.common import bash_wrapper
def compare_derivations( def compare_derivations(
path_to_flake: str, path_to_pre_drv: str, path_to_post_drv: str path_to_flake: str, path_to_pre_drv: str, path_to_post_drv: str, *, soft_failure: bool = True
) -> list[str]: ) -> str:
"""Gets all derivations present in a flake. """Gets all derivations present in a flake.
:param path_to_flake: path to flake to be checked :param path_to_flake: path to flake to be checked
@ -19,12 +19,15 @@ def compare_derivations(
if nvd_path is None: if nvd_path is None:
status_msg = "nvd is not available in the PATH, please verify that it is installed" status_msg = "nvd is not available in the PATH, please verify that it is installed"
raise RuntimeError(status_msg) raise RuntimeError(status_msg)
diff_output = bash_wrapper( stdout, stderr, returncode = bash_wrapper(
f"{nvd_path} diff {path_to_pre_drv} {path_to_post_drv}", path=path_to_flake f"{nvd_path} diff {path_to_pre_drv} {path_to_post_drv}", path=path_to_flake
) )
logging.debug(diff_output[0]) if returncode != 0:
logging.debug(diff_output[1]) log_func = logging.error
logging.debug(diff_output[2]) if soft_failure:
log_func = logging.warning
logging.warning(f"diff failed for {path_to_pre_drv} and {path_to_post_drv}")
log_func(stderr)
return diff_output return stdout

119
flupdt/main.py
View File

@ -3,6 +3,7 @@
"""Default processing of flake outputs for evaluating flake updates.""" """Default processing of flake outputs for evaluating flake updates."""
import logging import logging
import os
from argparse import Namespace from argparse import Namespace
from pathlib import Path from pathlib import Path
@ -35,6 +36,76 @@ def batch_eval(args: Namespace, flake_path: str, derivations: list[str]) -> None
dump(drv_map, f) dump(drv_map, f)
def compare_drvs(args: Namespace) -> None:
"""Compares derivation jsons using nvd.
param args: argparse namespace
"""
pre_json_dict = {}
post_json_dict = {}
from json import load
with (
Path.open(args.compare_pre_json, "r") as pre,
Path.open(args.compare_post_json, "r") as post,
):
pre_json_dict = load(pre)
post_json_dict = load(post)
logging.debug(f"pre-snapshot derivations: {pre_json_dict}")
logging.debug(f"post-snapshot derivations: {post_json_dict}")
pre_json_keys = set(pre_json_dict.keys())
post_json_keys = set(post_json_dict.keys())
common_keys_to_eval = pre_json_keys.union(post_json_keys)
missing_post_keys = pre_json_keys.difference(common_keys_to_eval)
missing_pre_keys = post_json_keys.difference(common_keys_to_eval)
if missing_pre_keys:
logging.warning(f"Following outputs are missing from pre-snapshot: {missing_pre_keys}")
if missing_post_keys:
logging.warning(f"Following outputs are missing from post-snapshot: {missing_post_keys}")
logging.info(f"Evaluating the following outputs for differences: {common_keys_to_eval}")
out_file: str = os.devnull
if args.compare_output_to_file:
out_file = args.compare_output_file
out_file_path = Path(out_file)
with out_file_path.open("w") as f:
for output_key in common_keys_to_eval:
comp_out = compare_derivations(
args.flake_path, pre_json_dict[output_key], post_json_dict[output_key]
)
f.write(f"comparing {output_key}:" + "\n")
if comp_out:
f.write(comp_out + "\n\n")
else:
f.write("comparison output is empty, please check script logs\n\n")
def build_or_eval(args: Namespace) -> None:
"""Builds or evaluates all outputs in a flake.
param args: argparse namespace
"""
flake_path = args.flake_path
derivations, hydra_jobs = partition(
lambda s: s.startswith("hydraJobs"), get_derivations(flake_path)
)
derivations, hydra_jobs = list(derivations), list(hydra_jobs)
logging.info(f"derivations: {list(derivations)}")
batch_eval(args, flake_path, derivations)
if not args.keep_hydra:
logging.info("--keep-hydra flag is not specified, removing Hydra jobs")
else:
batch_eval(args, flake_path, hydra_jobs)
def main() -> None: def main() -> None:
"""Sets up logging, parses args, and runs evaluation routine. """Sets up logging, parses args, and runs evaluation routine.
@ -44,53 +115,9 @@ def main() -> None:
configure_logger("DEBUG") configure_logger("DEBUG")
args = parse_inputs() args = parse_inputs()
if args.compare_drvs: if args.compare_drvs:
pre_json_dict = {} compare_drvs(args)
post_json_dict = {}
from json import load
with (
Path.open(args.compare_pre_json, "r") as pre,
Path.open(args.compare_post_json, "r") as post,
):
pre_json_dict = load(pre)
post_json_dict = load(post)
logging.debug(f"pre-snapshot derivations: {pre_json_dict}")
logging.debug(f"post-snapshot derivations: {post_json_dict}")
pre_json_keys = set(pre_json_dict.keys())
post_json_keys = set(post_json_dict.keys())
common_keys_to_eval = pre_json_keys.union(post_json_keys)
missing_post_keys = pre_json_keys.difference(common_keys_to_eval)
missing_pre_keys = post_json_keys.difference(common_keys_to_eval)
if missing_pre_keys:
logging.warning(f"Following outputs are missing from pre-snapshot: {missing_pre_keys}")
if missing_post_keys:
logging.warning(f"Following outputs are missing from post-snapshot: {missing_post_keys}")
logging.info(f"Evaluating the following outputs for differences: {common_keys_to_eval}")
for output_key in common_keys_to_eval:
compare_derivations(args.flake_path, pre_json_dict[output_key], post_json_dict[output_key])
else: else:
flake_path = args.flake_path build_or_eval(args)
derivations, hydra_jobs = partition(
lambda s: s.startswith("hydraJobs"), get_derivations(flake_path)
)
derivations, hydra_jobs = list(derivations), list(hydra_jobs)
logging.info(f"derivations: {list(derivations)}")
batch_eval(args, flake_path, derivations)
if not args.keep_hydra:
logging.info("--keep-hydra flag is not specified, removing Hydra jobs")
else:
batch_eval(args, flake_path, hydra_jobs)
if __name__ == "__main__": if __name__ == "__main__":

51
poetry.lock generated
View File

@ -1,4 +1,4 @@
# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. # This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.
[[package]] [[package]]
name = "greenlet" name = "greenlet"
@ -6,6 +6,8 @@ version = "3.1.1"
description = "Lightweight in-process concurrent programming" description = "Lightweight in-process concurrent programming"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
groups = ["main"]
markers = "python_version < \"3.14\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"
files = [ files = [
{file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"}, {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"},
{file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"}, {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"},
@ -88,29 +90,30 @@ test = ["objgraph", "psutil"]
[[package]] [[package]]
name = "ruff" name = "ruff"
version = "0.5.1" version = "0.8.6"
description = "An extremely fast Python linter and code formatter, written in Rust." description = "An extremely fast Python linter and code formatter, written in Rust."
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
groups = ["dev"]
files = [ files = [
{file = "ruff-0.5.1-py3-none-linux_armv6l.whl", hash = "sha256:6ecf968fcf94d942d42b700af18ede94b07521bd188aaf2cd7bc898dd8cb63b6"}, {file = "ruff-0.8.6-py3-none-linux_armv6l.whl", hash = "sha256:defed167955d42c68b407e8f2e6f56ba52520e790aba4ca707a9c88619e580e3"},
{file = "ruff-0.5.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:204fb0a472f00f2e6280a7c8c7c066e11e20e23a37557d63045bf27a616ba61c"}, {file = "ruff-0.8.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:54799ca3d67ae5e0b7a7ac234baa657a9c1784b48ec954a094da7c206e0365b1"},
{file = "ruff-0.5.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d235968460e8758d1e1297e1de59a38d94102f60cafb4d5382033c324404ee9d"}, {file = "ruff-0.8.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e88b8f6d901477c41559ba540beeb5a671e14cd29ebd5683903572f4b40a9807"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38beace10b8d5f9b6bdc91619310af6d63dd2019f3fb2d17a2da26360d7962fa"}, {file = "ruff-0.8.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0509e8da430228236a18a677fcdb0c1f102dd26d5520f71f79b094963322ed25"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e478d2f09cf06add143cf8c4540ef77b6599191e0c50ed976582f06e588c994"}, {file = "ruff-0.8.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:91a7ddb221779871cf226100e677b5ea38c2d54e9e2c8ed847450ebbdf99b32d"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0368d765eec8247b8550251c49ebb20554cc4e812f383ff9f5bf0d5d94190b0"}, {file = "ruff-0.8.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:248b1fb3f739d01d528cc50b35ee9c4812aa58cc5935998e776bf8ed5b251e75"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:3a9a9a1b582e37669b0138b7c1d9d60b9edac880b80eb2baba6d0e566bdeca4d"}, {file = "ruff-0.8.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:bc3c083c50390cf69e7e1b5a5a7303898966be973664ec0c4a4acea82c1d4315"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bdd9f723e16003623423affabcc0a807a66552ee6a29f90eddad87a40c750b78"}, {file = "ruff-0.8.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52d587092ab8df308635762386f45f4638badb0866355b2b86760f6d3c076188"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be9fd62c1e99539da05fcdc1e90d20f74aec1b7a1613463ed77870057cd6bd96"}, {file = "ruff-0.8.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:61323159cf21bc3897674e5adb27cd9e7700bab6b84de40d7be28c3d46dc67cf"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e216fc75a80ea1fbd96af94a6233d90190d5b65cc3d5dfacf2bd48c3e067d3e1"}, {file = "ruff-0.8.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ae4478b1471fc0c44ed52a6fb787e641a2ac58b1c1f91763bafbc2faddc5117"},
{file = "ruff-0.5.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c4c2112e9883a40967827d5c24803525145e7dab315497fae149764979ac7929"}, {file = "ruff-0.8.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0c000a471d519b3e6cfc9c6680025d923b4ca140ce3e4612d1a2ef58e11f11fe"},
{file = "ruff-0.5.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dfaf11c8a116394da3b65cd4b36de30d8552fa45b8119b9ef5ca6638ab964fa3"}, {file = "ruff-0.8.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9257aa841e9e8d9b727423086f0fa9a86b6b420fbf4bf9e1465d1250ce8e4d8d"},
{file = "ruff-0.5.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d7ceb9b2fe700ee09a0c6b192c5ef03c56eb82a0514218d8ff700f6ade004108"}, {file = "ruff-0.8.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:45a56f61b24682f6f6709636949ae8cc82ae229d8d773b4c76c09ec83964a95a"},
{file = "ruff-0.5.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bac6288e82f6296f82ed5285f597713acb2a6ae26618ffc6b429c597b392535c"}, {file = "ruff-0.8.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:496dd38a53aa173481a7d8866bcd6451bd934d06976a2505028a50583e001b76"},
{file = "ruff-0.5.1-py3-none-win32.whl", hash = "sha256:5c441d9c24ec09e1cb190a04535c5379b36b73c4bc20aa180c54812c27d1cca4"}, {file = "ruff-0.8.6-py3-none-win32.whl", hash = "sha256:e169ea1b9eae61c99b257dc83b9ee6c76f89042752cb2d83486a7d6e48e8f764"},
{file = "ruff-0.5.1-py3-none-win_amd64.whl", hash = "sha256:b1789bf2cd3d1b5a7d38397cac1398ddf3ad7f73f4de01b1e913e2abc7dfc51d"}, {file = "ruff-0.8.6-py3-none-win_amd64.whl", hash = "sha256:f1d70bef3d16fdc897ee290d7d20da3cbe4e26349f62e8a0274e7a3f4ce7a905"},
{file = "ruff-0.5.1-py3-none-win_arm64.whl", hash = "sha256:2875b7596a740cbbd492f32d24be73e545a4ce0a3daf51e4f4e609962bfd3cd2"}, {file = "ruff-0.8.6-py3-none-win_arm64.whl", hash = "sha256:7d7fc2377a04b6e04ffe588caad613d0c460eb2ecba4c0ccbbfe2bc973cbc162"},
{file = "ruff-0.5.1.tar.gz", hash = "sha256:3164488aebd89b1745b47fd00604fb4358d774465f20d1fcd907f9c0fc1b0655"}, {file = "ruff-0.8.6.tar.gz", hash = "sha256:dcad24b81b62650b0eb8814f576fc65cfee8674772a6e24c9b747911801eeaa5"},
] ]
[[package]] [[package]]
@ -119,6 +122,7 @@ version = "2.0.38"
description = "Database Abstraction Library" description = "Database Abstraction Library"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
groups = ["main"]
files = [ files = [
{file = "SQLAlchemy-2.0.38-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5e1d9e429028ce04f187a9f522818386c8b076723cdbe9345708384f49ebcec6"}, {file = "SQLAlchemy-2.0.38-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5e1d9e429028ce04f187a9f522818386c8b076723cdbe9345708384f49ebcec6"},
{file = "SQLAlchemy-2.0.38-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b87a90f14c68c925817423b0424381f0e16d80fc9a1a1046ef202ab25b19a444"}, {file = "SQLAlchemy-2.0.38-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b87a90f14c68c925817423b0424381f0e16d80fc9a1a1046ef202ab25b19a444"},
@ -214,12 +218,13 @@ version = "4.12.2"
description = "Backported and Experimental Type Hints for Python 3.8+" description = "Backported and Experimental Type Hints for Python 3.8+"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
groups = ["main"]
files = [ files = [
{file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
{file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
] ]
[metadata] [metadata]
lock-version = "2.0" lock-version = "2.1"
python-versions = "^3.11" python-versions = "^3.12"
content-hash = "6ebb0770da484a771f8c11e7ba8100b3180b80c107e8b56d240036e7d2689506" content-hash = "ff4f7ce6f8bf20026c9f29da5eabc2f4e5de408fee8f39a40e8d8c25c9e13b74"

8
pyproject.toml
View File

@ -12,7 +12,7 @@ readme = "README.md"
packages = [{ include = "flupdt" }] packages = [{ include = "flupdt" }]
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "^3.11" python = "^3.12"
sqlalchemy = "^2.0.31" sqlalchemy = "^2.0.31"
[tool.poetry.scripts] [tool.poetry.scripts]
@ -20,7 +20,7 @@ flupdt = "flupdt.main:main"
flake-update-diff = "flupdt.main:main" flake-update-diff = "flupdt.main:main"
[tool.poetry.group.dev.dependencies] [tool.poetry.group.dev.dependencies]
ruff = "0.5.1" ruff = "0.8.6"
[build-system] [build-system]
@ -32,13 +32,11 @@ build-backend = "poetry.core.masonry.api"
line-length = 100 line-length = 100
indent-width = 4 indent-width = 4
target-version = "py39" target-version = "py312"
[tool.ruff.lint] [tool.ruff.lint]
select = ["ALL"] select = ["ALL"]
ignore = [ ignore = [
"ANN101", # (perm) this rule is deprecated
"ANN102", # (perm) this rule is deprecated
"G004", # (perm) this is a performance nit "G004", # (perm) this is a performance nit
"COM812", # https://docs.astral.sh/ruff/rules/missing-trailing-comma/ "COM812", # https://docs.astral.sh/ruff/rules/missing-trailing-comma/
"ISC001", # https://docs.astral.sh/ruff/rules/single-line-implicit-string-concatenation/ "ISC001", # https://docs.astral.sh/ruff/rules/single-line-implicit-string-concatenation/