fix: add standardized exit codes on improper exit (issue #1221)

garak/cli.py

@@ -42,7 +42,7 @@ def main(arguments=None) -> None:
 
     from garak import __description__
     from garak import _config, _plugins
-    from garak.exception import GarakException
+    from garak.exception import GarakException, GarakExitCodes
 
     _config.transient.starttime = datetime.datetime.now()
     _config.transient.starttime_iso = _config.transient.starttime.isoformat()
@@ -491,7 +491,8 @@ def worker_count_validation(workers):
             except Exception as e:
                 logging.error(e)
                 print(e)
-                sys.exit(1)
+                # exit code per issue #1221
+                sys.exit(GarakExitCodes.UNSPECIFIED_EXCEPTION)
             finally:
                 command.end_run()
 
@@ -687,8 +688,12 @@ def worker_count_validation(workers):
         logging.exception(e)
         logging.info(msg)
         print(msg)
+        # exit code per issue #1221
+        sys.exit(GarakExitCodes.INTERRUPTED)
     except (ValueError, GarakException) as e:
         logging.exception(e)
         print(e)
+        # exit code per issue #1221
+        sys.exit(GarakExitCodes.UNSPECIFIED_EXCEPTION)
 
     _config.set_http_lib_agents(prior_user_agents)

garak/exception.py

@@ -1,6 +1,32 @@
 # SPDX-FileCopyrightText: Portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+from enum import IntEnum
+
+
+class GarakExitCodes(IntEnum):
+    """Standardized exit codes for garak CLI exits.
+
+    These codes allow wrapping tools such as CI/CD pipelines and garak ms to
+    detect precisely what went wrong during a garak run, rather than relying
+    on a generic non-zero exit or inspecting log output.
+
+    Negative values are used to avoid collisions with conventional UNIX exit
+    codes (0 = success, 1 = general error, 2 = misuse of shell builtins, etc.).
+    """
+
+    INTERRUPTED = -1
+    PROBE_EXCEPTION = -2
+    GENERATOR_EXCEPTION = -3
+    DETECTOR_EXCEPTION = -4
+    BUFF_EXCEPTION = -5
+    EVALUATOR_EXCEPTION = -6
+    HARNESS_EXCEPTION = -7
+    LANGPROVIDER_EXCEPTION = -8
+    REPORTING_EXCEPTION = -9
+    RESOURCE_EXHAUSTED = -10
+    UNSPECIFIED_EXCEPTION = -127
+
 
 class GarakException(Exception):
     """Base class for all  garak exceptions"""