ahuston-0/flake-update-diff
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feature/diff_output #5

Merged
ahuston-0 merged 1 commits from feature/diff_output into main 2025-03-08 14:50:51 -05:00
Conversation 0 Commits 1 Files Changed 5 +131 -81
5 changed files with 131 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
flupdt/cli.py
View File

@ -33,6 +33,17 @@ def parse_inputs() -> argparse.Namespace:
default=None,
help="location of post.json for comparison. defaults to <flake_path>/post.json",
)
parser.add_argument(
"--compare-output-to-file",
action="store_true",
help="whether to output comparison output to a file",
)
parser.add_argument(
"--compare-output-file",
metavar="compare-output-file-path",
default=None,
help="location of comparison output. defaults to <flake_path>/post-diff. implies --compare-output-to-file.",
)
args = parser.parse_args()
if args.compare_pre_json is None:
@ -41,4 +52,10 @@ def parse_inputs() -> argparse.Namespace:
if args.compare_post_json is None:
args.compare_post_json = args.flake_path + "/post.json"
if args.compare_output_to_file and args.compare_output_file is None:
args.compare_output_file = args.flake_path + "/post-diff"
if args.compare_output_file is not None:
args.compare_output_to_file = True
return args

17
flupdt/flake_diff.py
View File

@ -7,8 +7,8 @@ from flupdt.common import bash_wrapper
def compare_derivations(
path_to_flake: str, path_to_pre_drv: str, path_to_post_drv: str
) -> list[str]:
path_to_flake: str, path_to_pre_drv: str, path_to_post_drv: str, *, soft_failure: bool = True
) -> str:
"""Gets all derivations present in a flake.
:param path_to_flake: path to flake to be checked
@ -19,12 +19,15 @@ def compare_derivations(
if nvd_path is None:
status_msg = "nvd is not available in the PATH, please verify that it is installed"
raise RuntimeError(status_msg)
diff_output = bash_wrapper(
stdout, stderr, returncode = bash_wrapper(
f"{nvd_path} diff {path_to_pre_drv} {path_to_post_drv}", path=path_to_flake
)
logging.debug(diff_output[0])
logging.debug(diff_output[1])
logging.debug(diff_output[2])
if returncode != 0:
log_func = logging.error
if soft_failure:
log_func = logging.warning
logging.warning(f"diff failed for {path_to_pre_drv} and {path_to_post_drv}")
log_func(stderr)
return diff_output
return stdout

119
flupdt/main.py
View File

@ -3,6 +3,7 @@
"""Default processing of flake outputs for evaluating flake updates."""
import logging
import os
from argparse import Namespace
from pathlib import Path
@ -35,6 +36,76 @@ def batch_eval(args: Namespace, flake_path: str, derivations: list[str]) -> None
dump(drv_map, f)
def compare_drvs(args: Namespace) -> None:
"""Compares derivation jsons using nvd.
param args: argparse namespace
"""
pre_json_dict = {}
post_json_dict = {}
from json import load
with (
Path.open(args.compare_pre_json, "r") as pre,
Path.open(args.compare_post_json, "r") as post,
):
pre_json_dict = load(pre)
post_json_dict = load(post)
logging.debug(f"pre-snapshot derivations: {pre_json_dict}")
logging.debug(f"post-snapshot derivations: {post_json_dict}")
pre_json_keys = set(pre_json_dict.keys())
post_json_keys = set(post_json_dict.keys())
common_keys_to_eval = pre_json_keys.union(post_json_keys)
missing_post_keys = pre_json_keys.difference(common_keys_to_eval)
missing_pre_keys = post_json_keys.difference(common_keys_to_eval)
if missing_pre_keys:
logging.warning(f"Following outputs are missing from pre-snapshot: {missing_pre_keys}")
if missing_post_keys:
logging.warning(f"Following outputs are missing from post-snapshot: {missing_post_keys}")
logging.info(f"Evaluating the following outputs for differences: {common_keys_to_eval}")
out_file: str = os.devnull
if args.compare_output_to_file:
out_file = args.compare_output_file
out_file_path = Path(out_file)
with out_file_path.open("w") as f:
for output_key in common_keys_to_eval:
comp_out = compare_derivations(
args.flake_path, pre_json_dict[output_key], post_json_dict[output_key]
)
f.write(f"comparing {output_key}:" + "\n")
if comp_out:
f.write(comp_out + "\n\n")
else:
f.write("comparison output is empty, please check script logs\n\n")
def build_or_eval(args: Namespace) -> None:
"""Builds or evaluates all outputs in a flake.
param args: argparse namespace
"""
flake_path = args.flake_path
derivations, hydra_jobs = partition(
lambda s: s.startswith("hydraJobs"), get_derivations(flake_path)
)
derivations, hydra_jobs = list(derivations), list(hydra_jobs)
logging.info(f"derivations: {list(derivations)}")
batch_eval(args, flake_path, derivations)
if not args.keep_hydra:
logging.info("--keep-hydra flag is not specified, removing Hydra jobs")
else:
batch_eval(args, flake_path, hydra_jobs)
def main() -> None:
"""Sets up logging, parses args, and runs evaluation routine.
@ -44,53 +115,9 @@ def main() -> None:
configure_logger("DEBUG")
args = parse_inputs()
if args.compare_drvs:
pre_json_dict = {}
post_json_dict = {}
from json import load
with (
Path.open(args.compare_pre_json, "r") as pre,
Path.open(args.compare_post_json, "r") as post,
):
pre_json_dict = load(pre)
post_json_dict = load(post)
logging.debug(f"pre-snapshot derivations: {pre_json_dict}")
logging.debug(f"post-snapshot derivations: {post_json_dict}")
pre_json_keys = set(pre_json_dict.keys())
post_json_keys = set(post_json_dict.keys())
common_keys_to_eval = pre_json_keys.union(post_json_keys)
missing_post_keys = pre_json_keys.difference(common_keys_to_eval)
missing_pre_keys = post_json_keys.difference(common_keys_to_eval)
if missing_pre_keys:
logging.warning(f"Following outputs are missing from pre-snapshot: {missing_pre_keys}")
if missing_post_keys:
logging.warning(f"Following outputs are missing from post-snapshot: {missing_post_keys}")
logging.info(f"Evaluating the following outputs for differences: {common_keys_to_eval}")
for output_key in common_keys_to_eval:
compare_derivations(args.flake_path, pre_json_dict[output_key], post_json_dict[output_key])
compare_drvs(args)
else:
flake_path = args.flake_path
derivations, hydra_jobs = partition(
lambda s: s.startswith("hydraJobs"), get_derivations(flake_path)
)
derivations, hydra_jobs = list(derivations), list(hydra_jobs)
logging.info(f"derivations: {list(derivations)}")
batch_eval(args, flake_path, derivations)
if not args.keep_hydra:
logging.info("--keep-hydra flag is not specified, removing Hydra jobs")
else:
batch_eval(args, flake_path, hydra_jobs)
build_or_eval(args)
if __name__ == "__main__":

51
poetry.lock generated
View File

@ -1,4 +1,4 @@
# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.
[[package]]
name = "greenlet"
@ -6,6 +6,8 @@ version = "3.1.1"
description = "Lightweight in-process concurrent programming"
optional = false
python-versions = ">=3.7"
groups = ["main"]
markers = "python_version < \"3.14\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"
files = [
{file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"},
{file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"},
@ -88,29 +90,30 @@ test = ["objgraph", "psutil"]
[[package]]
name = "ruff"
version = "0.5.1"
version = "0.8.6"
description = "An extremely fast Python linter and code formatter, written in Rust."
optional = false
python-versions = ">=3.7"
groups = ["dev"]
files = [
{file = "ruff-0.5.1-py3-none-linux_armv6l.whl", hash = "sha256:6ecf968fcf94d942d42b700af18ede94b07521bd188aaf2cd7bc898dd8cb63b6"},
{file = "ruff-0.5.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:204fb0a472f00f2e6280a7c8c7c066e11e20e23a37557d63045bf27a616ba61c"},
{file = "ruff-0.5.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d235968460e8758d1e1297e1de59a38d94102f60cafb4d5382033c324404ee9d"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38beace10b8d5f9b6bdc91619310af6d63dd2019f3fb2d17a2da26360d7962fa"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e478d2f09cf06add143cf8c4540ef77b6599191e0c50ed976582f06e588c994"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0368d765eec8247b8550251c49ebb20554cc4e812f383ff9f5bf0d5d94190b0"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:3a9a9a1b582e37669b0138b7c1d9d60b9edac880b80eb2baba6d0e566bdeca4d"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bdd9f723e16003623423affabcc0a807a66552ee6a29f90eddad87a40c750b78"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be9fd62c1e99539da05fcdc1e90d20f74aec1b7a1613463ed77870057cd6bd96"},
{file = "ruff-0.5.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e216fc75a80ea1fbd96af94a6233d90190d5b65cc3d5dfacf2bd48c3e067d3e1"},
{file = "ruff-0.5.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c4c2112e9883a40967827d5c24803525145e7dab315497fae149764979ac7929"},
{file = "ruff-0.5.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dfaf11c8a116394da3b65cd4b36de30d8552fa45b8119b9ef5ca6638ab964fa3"},
{file = "ruff-0.5.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d7ceb9b2fe700ee09a0c6b192c5ef03c56eb82a0514218d8ff700f6ade004108"},
{file = "ruff-0.5.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bac6288e82f6296f82ed5285f597713acb2a6ae26618ffc6b429c597b392535c"},
{file = "ruff-0.5.1-py3-none-win32.whl", hash = "sha256:5c441d9c24ec09e1cb190a04535c5379b36b73c4bc20aa180c54812c27d1cca4"},
{file = "ruff-0.5.1-py3-none-win_amd64.whl", hash = "sha256:b1789bf2cd3d1b5a7d38397cac1398ddf3ad7f73f4de01b1e913e2abc7dfc51d"},
{file = "ruff-0.5.1-py3-none-win_arm64.whl", hash = "sha256:2875b7596a740cbbd492f32d24be73e545a4ce0a3daf51e4f4e609962bfd3cd2"},
{file = "ruff-0.5.1.tar.gz", hash = "sha256:3164488aebd89b1745b47fd00604fb4358d774465f20d1fcd907f9c0fc1b0655"},
{file = "ruff-0.8.6-py3-none-linux_armv6l.whl", hash = "sha256:defed167955d42c68b407e8f2e6f56ba52520e790aba4ca707a9c88619e580e3"},
{file = "ruff-0.8.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:54799ca3d67ae5e0b7a7ac234baa657a9c1784b48ec954a094da7c206e0365b1"},
{file = "ruff-0.8.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e88b8f6d901477c41559ba540beeb5a671e14cd29ebd5683903572f4b40a9807"},
{file = "ruff-0.8.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0509e8da430228236a18a677fcdb0c1f102dd26d5520f71f79b094963322ed25"},
{file = "ruff-0.8.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:91a7ddb221779871cf226100e677b5ea38c2d54e9e2c8ed847450ebbdf99b32d"},
{file = "ruff-0.8.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:248b1fb3f739d01d528cc50b35ee9c4812aa58cc5935998e776bf8ed5b251e75"},
{file = "ruff-0.8.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:bc3c083c50390cf69e7e1b5a5a7303898966be973664ec0c4a4acea82c1d4315"},
{file = "ruff-0.8.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52d587092ab8df308635762386f45f4638badb0866355b2b86760f6d3c076188"},
{file = "ruff-0.8.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:61323159cf21bc3897674e5adb27cd9e7700bab6b84de40d7be28c3d46dc67cf"},
{file = "ruff-0.8.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ae4478b1471fc0c44ed52a6fb787e641a2ac58b1c1f91763bafbc2faddc5117"},
{file = "ruff-0.8.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0c000a471d519b3e6cfc9c6680025d923b4ca140ce3e4612d1a2ef58e11f11fe"},
{file = "ruff-0.8.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9257aa841e9e8d9b727423086f0fa9a86b6b420fbf4bf9e1465d1250ce8e4d8d"},
{file = "ruff-0.8.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:45a56f61b24682f6f6709636949ae8cc82ae229d8d773b4c76c09ec83964a95a"},
{file = "ruff-0.8.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:496dd38a53aa173481a7d8866bcd6451bd934d06976a2505028a50583e001b76"},
{file = "ruff-0.8.6-py3-none-win32.whl", hash = "sha256:e169ea1b9eae61c99b257dc83b9ee6c76f89042752cb2d83486a7d6e48e8f764"},
{file = "ruff-0.8.6-py3-none-win_amd64.whl", hash = "sha256:f1d70bef3d16fdc897ee290d7d20da3cbe4e26349f62e8a0274e7a3f4ce7a905"},
{file = "ruff-0.8.6-py3-none-win_arm64.whl", hash = "sha256:7d7fc2377a04b6e04ffe588caad613d0c460eb2ecba4c0ccbbfe2bc973cbc162"},
{file = "ruff-0.8.6.tar.gz", hash = "sha256:dcad24b81b62650b0eb8814f576fc65cfee8674772a6e24c9b747911801eeaa5"},
]
[[package]]
@ -119,6 +122,7 @@ version = "2.0.38"
description = "Database Abstraction Library"
optional = false
python-versions = ">=3.7"
groups = ["main"]
files = [
{file = "SQLAlchemy-2.0.38-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5e1d9e429028ce04f187a9f522818386c8b076723cdbe9345708384f49ebcec6"},
{file = "SQLAlchemy-2.0.38-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b87a90f14c68c925817423b0424381f0e16d80fc9a1a1046ef202ab25b19a444"},
@ -214,12 +218,13 @@ version = "4.12.2"
description = "Backported and Experimental Type Hints for Python 3.8+"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
{file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
]
[metadata]
lock-version = "2.0"
python-versions = "^3.11"
content-hash = "6ebb0770da484a771f8c11e7ba8100b3180b80c107e8b56d240036e7d2689506"
lock-version = "2.1"
python-versions = "^3.12"
content-hash = "ff4f7ce6f8bf20026c9f29da5eabc2f4e5de408fee8f39a40e8d8c25c9e13b74"

8
pyproject.toml
View File

@ -12,7 +12,7 @@ readme = "README.md"
packages = [{ include = "flupdt" }]
[tool.poetry.dependencies]
python = "^3.11"
python = "^3.12"
sqlalchemy = "^2.0.31"
[tool.poetry.scripts]
@ -20,7 +20,7 @@ flupdt = "flupdt.main:main"
flake-update-diff = "flupdt.main:main"
[tool.poetry.group.dev.dependencies]
ruff = "0.5.1"
ruff = "0.8.6"
[build-system]
@ -32,13 +32,11 @@ build-backend = "poetry.core.masonry.api"
line-length = 100
indent-width = 4
target-version = "py39"
target-version = "py312"
[tool.ruff.lint]
select = ["ALL"]
ignore = [
"ANN101", # (perm) this rule is deprecated
"ANN102", # (perm) this rule is deprecated
"G004", # (perm) this is a performance nit
"COM812", # https://docs.astral.sh/ruff/rules/missing-trailing-comma/
"ISC001", # https://docs.astral.sh/ruff/rules/single-line-implicit-string-concatenation/