diff --git a/garak/cli.py b/garak/cli.py index 67ed67853..43540ffec 100644 --- a/garak/cli.py +++ b/garak/cli.py @@ -42,7 +42,7 @@ def main(arguments=None) -> None: from garak import __description__ from garak import _config, _plugins - from garak.exception import GarakException + from garak.exception import GarakException, GarakExitCodes _config.transient.starttime = datetime.datetime.now() _config.transient.starttime_iso = _config.transient.starttime.isoformat() @@ -491,7 +491,8 @@ def worker_count_validation(workers): except Exception as e: logging.error(e) print(e) - sys.exit(1) + # exit code per issue #1221 + sys.exit(GarakExitCodes.UNSPECIFIED_EXCEPTION) finally: command.end_run() @@ -687,8 +688,12 @@ def worker_count_validation(workers): logging.exception(e) logging.info(msg) print(msg) + # exit code per issue #1221 + sys.exit(GarakExitCodes.INTERRUPTED) except (ValueError, GarakException) as e: logging.exception(e) print(e) + # exit code per issue #1221 + sys.exit(GarakExitCodes.UNSPECIFIED_EXCEPTION) _config.set_http_lib_agents(prior_user_agents) diff --git a/garak/exception.py b/garak/exception.py index 9891a9c5b..7dca1f8aa 100644 --- a/garak/exception.py +++ b/garak/exception.py @@ -1,6 +1,32 @@ # SPDX-FileCopyrightText: Portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from enum import IntEnum + + +class GarakExitCodes(IntEnum): + """Standardized exit codes for garak CLI exits. + + These codes allow wrapping tools such as CI/CD pipelines and garak ms to + detect precisely what went wrong during a garak run, rather than relying + on a generic non-zero exit or inspecting log output. + + Negative values are used to avoid collisions with conventional UNIX exit + codes (0 = success, 1 = general error, 2 = misuse of shell builtins, etc.). + """ + + INTERRUPTED = -1 + PROBE_EXCEPTION = -2 + GENERATOR_EXCEPTION = -3 + DETECTOR_EXCEPTION = -4 + BUFF_EXCEPTION = -5 + EVALUATOR_EXCEPTION = -6 + HARNESS_EXCEPTION = -7 + LANGPROVIDER_EXCEPTION = -8 + REPORTING_EXCEPTION = -9 + RESOURCE_EXHAUSTED = -10 + UNSPECIFIED_EXCEPTION = -127 + class GarakException(Exception): """Base class for all garak exceptions"""