pipelines: extend command group

acceptance/pipelines/databricks-cli-help-disabled/output.txt

@@ -0,0 +1,49 @@
+The Lakeflow Spark Declarative Pipelines API allows you to create, edit,
+  delete, start, and view details about pipelines.
+
+  Spark Declarative Pipelines is a framework for building reliable,
+  maintainable, and testable data processing pipelines. You define the
+  transformations to perform on your data, and Spark Declarative Pipelines
+  manages task orchestration, cluster management, monitoring, data quality, and
+  error handling.
+
+  Instead of defining your data pipelines using a series of separate Apache
+  Spark tasks, Spark Declarative Pipelines manages how your data is transformed
+  based on a target schema you define for each processing step. You can also
+  enforce data quality with Spark Declarative Pipelines expectations.
+  Expectations allow you to define expected data quality and specify how to
+  handle records that fail those expectations.
+
+Usage:
+  databricks pipelines [flags]
+  databricks pipelines [command]
+
+Available Commands
+  clone                 Clone a pipeline.
+  create                Create a pipeline.
+  delete                Delete a pipeline.
+  get                   Get a pipeline.
+  get-update            Get a pipeline update.
+  list-pipeline-events  List pipeline events.
+  list-pipelines        List pipelines.
+  list-updates          List pipeline updates.
+  start-update          Start a pipeline.
+  stop                  Stop a pipeline.
+  update                Edit a pipeline.
+
+Permission Commands
+  get-permission-levels Get pipeline permission levels.
+  get-permissions       Get pipeline permissions.
+  set-permissions       Set pipeline permissions.
+  update-permissions    Update pipeline permissions.
+
+Flags:
+  -h, --help   help for pipelines
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+
+Use "databricks pipelines [command] --help" for more information about a command.

acceptance/pipelines/databricks-cli-help-disabled/script

@@ -0,0 +1 @@
+$CLI pipelines

acceptance/pipelines/databricks-cli-help/output.txt

@@ -0,0 +1,60 @@
+The Lakeflow Spark Declarative Pipelines API allows you to create, edit,
+  delete, start, and view details about pipelines.
+
+  Spark Declarative Pipelines is a framework for building reliable,
+  maintainable, and testable data processing pipelines. You define the
+  transformations to perform on your data, and Spark Declarative Pipelines
+  manages task orchestration, cluster management, monitoring, data quality, and
+  error handling.
+
+  Instead of defining your data pipelines using a series of separate Apache
+  Spark tasks, Spark Declarative Pipelines manages how your data is transformed
+  based on a target schema you define for each processing step. You can also
+  enforce data quality with Spark Declarative Pipelines expectations.
+  Expectations allow you to define expected data quality and specify how to
+  handle records that fail those expectations.
+
+Usage:
+  databricks pipelines [flags]
+  databricks pipelines [command]
+
+Available Commands
+  deploy                Deploy pipelines
+  destroy               Destroy a pipelines project
+  dry-run               Validate correctness of the pipeline's graph
+  history               Retrieve past runs for a pipeline
+  init                  Initialize a new pipelines project
+  logs                  Retrieve events for a pipeline
+  open                  Open a pipeline in the browser
+  run                   Run a pipeline
+  stop                  Stop a pipeline
+
+Management Commands
+  clone                 Clone a pipeline.
+  create                Create a pipeline.
+  delete                Delete a pipeline.
+  get                   Get a pipeline.
+  get-update            Get a pipeline update.
+  list-pipeline-events  List pipeline events.
+  list-pipelines        List pipelines.
+  list-updates          List pipeline updates.
+  start-update          Start a pipeline.
+  update                Edit a pipeline.
+
+Permission Commands
+  get-permission-levels Get pipeline permission levels.
+  get-permissions       Get pipeline permissions.
+  set-permissions       Set pipeline permissions.
+  update-permissions    Update pipeline permissions.
+
+Flags:
+  -h, --help          help for pipelines
+      --var strings   set values for variables defined in project config. Example: --var="foo=bar"
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+
+Use "databricks pipelines [command] --help" for more information about a command.

acceptance/pipelines/databricks-cli-help/script

@@ -0,0 +1 @@
+ENABLE_PIPELINES_CLI=1 $CLI pipelines

cmd/cmd.go

@@ -57,6 +57,11 @@ func New(ctx context.Context) *cobra.Command {
 	for _, cmd := range workspaceCommands {
 		// Order the permissions subcommands after the main commands.
 		for _, sub := range cmd.Commands() {
+			// some commands override groups in overrides.go, leave them as-is
+			if sub.GroupID != "" {
+				continue
+			}
+
 			switch {
 			case strings.HasSuffix(sub.Name(), "-permissions"), strings.HasSuffix(sub.Name(), "-permission-levels"):
 				sub.GroupID = permissionsGroup
@@ -73,6 +78,10 @@ func New(ctx context.Context) *cobra.Command {
 				ID:    mainGroup,
 				Title: "Available Commands",
 			},
+			{
+				ID:    pipelines.ManagementGroupID,
+				Title: "Management Commands",
+			},
 			{
 				ID:    permissionsGroup,
 				Title: "Permission Commands",

cmd/pipelines/pipelines.go

@@ -2,6 +2,8 @@ package pipelines
 
 import (
 	"context"
+	"os"
+	"slices"
 
 	"github.com/databricks/cli/cmd/pipelines/root"
 	"github.com/spf13/cobra"
@@ -17,9 +19,39 @@ func New(ctx context.Context) *cobra.Command {
 	cli.AddCommand(dryRunCommand())
 	cli.AddCommand(authCommand())
 	cli.AddCommand(destroyCommand())
-	cli.AddCommand(stopCommand())
+	cli.AddCommand(StopCommand())
 	cli.AddCommand(historyCommand())
 	cli.AddCommand(logsCommand())
 	cli.AddCommand(versionCommand())
 	return cli
 }
+
+// ManagementGroupID contains auto-generated CLI commands for Pipelines API,
+// that are separate from main CLI commands defined in Commands.
+const ManagementGroupID = "management"
+
+// Enabled disables Pipelines CLI in Databricks CLI until it's ready
+func Enabled() bool {
+	value, ok := os.LookupEnv("ENABLE_PIPELINES_CLI")
+	if !ok {
+		return false
+	}
+
+	return slices.Contains([]string{"1", "true", "t", "yes"}, value)
+}
+
+// Commands returns the list of commands that are shared between
+// the standalone pipelines CLI and databricks pipelines.
+// Note: auth and version are excluded as they are only for standalone CLI.
+func Commands() []*cobra.Command {
+	return []*cobra.Command{
+		initCommand(),
+		deployCommand(),
+		destroyCommand(),
+		runCommand(),
+		dryRunCommand(),
+		historyCommand(),
+		logsCommand(),
+		openCommand(),
+	}
+}

cmd/pipelines/stop.go

@@ -36,7 +36,7 @@ func resolveStopArgument(ctx context.Context, b *bundle.Bundle, args []string) (
 	return "", errors.New("expected a KEY of the pipeline to stop")
 }
 
-func stopCommand() *cobra.Command {
+func StopCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "stop [KEY]",
 		Short: "Stop a pipeline",

cmd/workspace/pipelines/overrides.go

@@ -0,0 +1,78 @@
+package pipelines
+
+import (
+	"regexp"
+	"slices"
+
+	pipelinesCli "github.com/databricks/cli/cmd/pipelines"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	if !pipelinesCli.Enabled() {
+		return
+	}
+
+	cmdOverrides = append(cmdOverrides, func(cli *cobra.Command) {
+		// all auto-generated commands apart from nonManagementCommands go into 'management' group
+		nonManagementCommands := []string{
+			// 'stop' command is overloaded as Pipelines API and Pipelines CLI command
+			"stop",
+			// permission commands are assigned into "permission" group in cmd/cmd.go
+			// only if they don't have GroupID set
+			"get-permission-levels",
+			"get-permissions",
+			"set-permissions",
+			"update-permissions",
+		}
+
+		for _, subCmd := range cli.Commands() {
+			if slices.Contains(nonManagementCommands, subCmd.Name()) {
+				continue
+			}
+
+			if subCmd.GroupID == "" {
+				subCmd.GroupID = pipelinesCli.ManagementGroupID
+			}
+		}
+
+		// main section is populated with commands from Pipelines CLI
+		for _, pipelinesCmd := range pipelinesCli.Commands() {
+			cli.AddCommand(pipelinesCmd)
+		}
+
+		// Add --var flag support (from cli/pipelines/variables.go)
+		cli.PersistentFlags().StringSlice("var", []string{}, `set values for variables defined in project config. Example: --var="foo=bar"`)
+	})
+
+	// 'stop' command is different in context of bundle vs. management command
+	stopOverrides = append(stopOverrides, func(cmd *cobra.Command, req *pipelines.StopRequest) {
+		originalRunE := cmd.RunE
+		cmd.RunE = func(cmd *cobra.Command, args []string) error {
+			// For compatibility, if argument looks like pipeline ID, use API
+			if len(args) > 0 && looksLikeUUID(args[0]) {
+				return originalRunE(cmd, args)
+			}
+			// Looks like a bundle key or no args - use Lakeflow stop
+			return pipelinesCli.StopCommand().RunE(cmd, args)
+		}
+
+		// Update usage to reflect dual nature
+		cmd.Use = "stop [KEY|PIPELINE_ID]"
+		cmd.Short = "Stop a pipeline"
+		cmd.Long = `Stop a pipeline.
+
+With a bundle KEY: Stops the pipeline identified by KEY from your databricks.yml.
+If there is only one pipeline in the bundle, KEY is optional.
+
+With a PIPELINE_ID: Stops the pipeline identified by the UUID using the API.`
+	})
+}
+
+var uuidRegex = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
+
+// looksLikeUUID checks if a string matches the UUID format with lowercase hex digits
+func looksLikeUUID(s string) bool {
+	return uuidRegex.MatchString(s)
+}

cmd/workspace/pipelines/overrides_test.go

@@ -0,0 +1,15 @@
+package pipelines
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLooksLikeUUID(t *testing.T) {
+	assert.True(t, looksLikeUUID("a12cd3e4-0ab1-1abc-1a2b-1a2bcd3e4f05"))
+}
+
+func TestLooksLikeUUID_resourceName(t *testing.T) {
+	assert.False(t, looksLikeUUID("my-pipeline-key"))
+}