implement to retrieve autorest configuration in modules

kairu-ms · kairu-ms · commit bdb091b5a7f3 · 2024-12-27T11:12:38.000+08:00
diff --git a/src/aaz_dev/app/app.py b/src/aaz_dev/app/app.py
@@ -27,12 +27,13 @@ def invalid_api_usage(e):
         return jsonify(e.to_dict()), e.status_code
 
     # register url converters
-    from .url_converters import Base64Converter, NameConverter, NameWithCapitalConverter, NamesPathConverter, ListPathConvertor
+    from .url_converters import Base64Converter, NameConverter, NameWithCapitalConverter, NamesPathConverter, ListPathConvertor, PSNamesPathConverter
     app.url_map.converters['base64'] = Base64Converter
     app.url_map.converters['name'] = NameConverter
     app.url_map.converters['Name'] = NameWithCapitalConverter
     app.url_map.converters['names_path'] = NamesPathConverter
     app.url_map.converters['list_path'] = ListPathConvertor
+    app.url_map.converters['PSNamesPath'] = PSNamesPathConverter
 
     # register routes of swagger module
     from swagger.api import register_blueprints
diff --git a/src/aaz_dev/app/url_converters.py b/src/aaz_dev/app/url_converters.py
@@ -32,7 +32,6 @@ def to_url(self, values):
             return super(NamesPathConverter, self).to_url(values)
         return '/'.join(super(NamesPathConverter, self).to_url(value) for value in values)
 
-
 class ListPathConvertor(PathConverter):
 
     def to_python(self, value):
@@ -43,5 +42,17 @@ def to_url(self, values):
             return super(ListPathConvertor, self).to_url(values)
         return '/'.join(super(ListPathConvertor, self).to_url(value) for value in values)
 
+class PSNamesPathConverter(PathConverter):
+    regex = r"([A-Z][a-zA-Z0-9]*)/([A-Z][a-zA-Z0-9]*\.Autorest)"
+    weight = 200
+
+    def to_python(self, value):
+        return value.split('/')
+
+    def to_url(self, values):
+        if isinstance(values, str):
+            return super(PSNamesPathConverter, self).to_url(values)
+        return '/'.join(super(PSNamesPathConverter, self).to_url(value) for value in values)
+
 
-__all__ = ["Base64Converter", "NameConverter", "NamesPathConverter", "ListPathConvertor"]
+__all__ = ["Base64Converter", "NameConverter", "NamesPathConverter", "ListPathConvertor", "PSNamesPathConverter"]
diff --git a/src/aaz_dev/ps/api/powershell.py b/src/aaz_dev/ps/api/powershell.py
@@ -2,23 +2,65 @@
 
 from utils.config import Config
 from utils import exceptions
-from command.controller.specs_manager import AAZSpecsManager
+from ps.controller.ps_module_manager import PSModuleManager
+from app.url_converters import PSNamesPathConverter
+# from command.controller.specs_manager import AAZSpecsManager
 import logging
+import re
 
 logging.basicConfig(level="INFO")
 
 
 bp = Blueprint('powershell', __name__, url_prefix='/PS/Powershell')
 
 
-@bp.route("/Path", methods=("GET", "PUT"))
+@bp.route("/Path", methods=("GET", ))
 def powershell_path():
+    if Config.POWERSHELL_PATH is None:
+        raise exceptions.InvalidAPIUsage("PowerShell path is not set, please add `--ps` option to `aaz-dev run` command or set up `AAZ_POWERSHELL_PATH` environment variable")
+    return jsonify({"path": Config.POWERSHELL_PATH})
+
+
+@bp.route("/Modules", methods=("GET", "POST"))
+def powershell_modules():
+    manager = PSModuleManager()
+    if request.method == "GET":
+        modules = manager.list_modules()
+        result = []
+        for module in modules:
+            result.append({
+                **module,
+                'url': url_for('powershell.powershell_module', module_names=module['name']),
+            })
+        return jsonify(result)
+    elif request.method == "POST":
+        # create a new module in powershell
+        data = request.get_json()
+        if not data or not isinstance(data, dict) or 'name' not in data:
+            raise exceptions.InvalidAPIUsage("Invalid request body")
+        if not re.match(PSNamesPathConverter.regex, data['name'].split('/')):
+            raise exceptions.InvalidAPIUsage("Invalid module name")
+        module_names = data['name'].split('/')
+        # make sure the name is follow the PSNamesPathConverter.regex
+        module = manager.create_new_mod(module_names)
+        result = module.to_primitive()
+        result['url'] = url_for('powershell.powershell_module', module_names=module.name)
+    else:
+        raise NotImplementedError()
+    return jsonify(result)
+
+
+@bp.route("/Modules/<PSNamesPath:module_names>", methods=("GET", "PUT", "PATCH"))
+def powershell_module(module_names):
+    manager = PSModuleManager()
     if request.method == "GET":
-        return jsonify({"path": Config.POWERSHELL_PATH})
+        result = manager.load_module(module_names)
+        # result = module.to_primitive()
+        result['url'] = url_for('powershell.powershell_module', module_names=result['name'])
     elif request.method == "PUT":
-        data = request.json
-        try:
-            Config.validate_and_setup_powershell_path(None, None, data["path"])
-        except ValueError as e:
-            raise exceptions.InvalidAPIUsage(str(e))
-        return jsonify({"path": Config.POWERSHELL_PATH})
+        raise NotImplementedError()
+    elif request.method == "PATCH":
+        raise NotImplementedError()
+    else:
+        raise NotImplementedError()
+    return jsonify(result)
diff --git a/src/aaz_dev/ps/controller/ps_module_manager.py b/src/aaz_dev/ps/controller/ps_module_manager.py
@@ -1,17 +1,94 @@
-import ast
-import glob
-import json
 import logging
 import os
-import pkgutil
-import re
+import yaml
 
-from utils import exceptions
 from utils.config import Config
-from collections import deque
 
 logger = logging.getLogger('backend')
 
 
 class PSModuleManager:
-    pass
+
+    def __init__(self):
+        module_folder = self._find_module_folder()
+        self.folder = module_folder
+
+    def _find_module_folder(self):
+        powershell_folder = Config.POWERSHELL_PATH
+        if not os.path.exists(powershell_folder) or not os.path.isdir(powershell_folder):
+            raise ValueError(f"Invalid PowerShell folder: '{powershell_folder}'")
+        module_folder = os.path.join(powershell_folder, "src")
+        if not os.path.exists(module_folder):
+            raise ValueError(f"Invalid PowerShell folder: cannot find modules in: '{module_folder}'")
+        return module_folder
+
+    def list_modules(self):
+        modules = []
+        for folder_name in os.listdir(self.folder):
+            path = os.path.join(self.folder, folder_name)
+            if os.path.isdir(path):
+                for sub_folder in os.listdir(path):
+                    if os.path.isdir(os.path.join(path, sub_folder)) and sub_folder.endswith(".Autorest"):
+                        name = f"{folder_name}/{sub_folder}"
+                        modules.append({
+                            "name": name,
+                            "folder": os.path.join(path, sub_folder)
+                        })
+        return sorted(modules, key=lambda a: a['name'])
+
+    def create_new_mod(self, module_names):
+        if isinstance(module_names, str):
+            module_names = module_names.split('/')
+        folder = os.path.join(self.folder, *module_names)
+        os.makedirs(folder, exist_ok=True)
+
+    def load_module(self, module_names):
+        if isinstance(module_names, str):
+            module_names = module_names.split('/')
+        folder = os.path.join(self.folder, *module_names)
+        if not os.path.exists(folder):
+            raise ValueError(f"Module folder not found: '{folder}'")
+        autorest_config = self.load_autorest_config(module_names)
+        return {
+            **autorest_config,
+            "name": "/".join(module_names),
+            "folder": folder
+        }
+
+    def load_autorest_config(self, module_names):
+        if isinstance(module_names, str):
+            module_names = module_names.split('/')
+        folder = os.path.join(self.folder, *module_names)
+        readme_file = os.path.join(folder, "README.md")
+        if not os.path.exists(readme_file):
+            raise ValueError(f"README.md not found in: '{readme_file}'")
+        with open(readme_file, "r") as f:
+            content = f.readlines()
+        autorest_config = []
+        in_autorest_config_section = False
+        in_yaml_section = False
+        for line in content:
+            if line.strip().startswith("### AutoRest Configuration"):
+                in_autorest_config_section = True
+            elif in_autorest_config_section:
+                if line.strip().startswith("###"):
+                    break
+                if line.strip().startswith("```") and 'yaml' in line:
+                    in_yaml_section = True
+                elif in_yaml_section:
+                    if line.strip().startswith("```"):
+                        in_yaml_section = False
+                    else:
+                        if line.strip():
+                            autorest_config.append(line)
+                        else:
+                            autorest_config.append("")
+        autorest_config_raw = "\n".join(autorest_config)
+        try:
+            yaml_config = yaml.load(autorest_config_raw, Loader=yaml.FullLoader)
+        except Exception as e:
+            raise ValueError(f"Failed to parse autorest config: {e} for readme_file: {readme_file}")
+        return {
+            "autorest_config": yaml_config,
+            # "raw": autorest_config_raw  # can be used for directive merging
+        }
diff --git a/src/aaz_dev/ps/templates/autorest/config_common_used_props.yaml b/src/aaz_dev/ps/templates/autorest/config_common_used_props.yaml
@@ -0,0 +1,67 @@
+metadata:  # display to the users
+  dict:
+    authors: Microsoft Corporation
+    companyName: Microsoft Corporation
+    copyright: Microsoft Corporation. All rights reserved.
+    description: 'Microsoft Azure PowerShell: Portal Dashboard cmdlets'
+    licenseUri: https://aka.ms/azps-license
+    owners: Microsoft Corporation
+    projectUri: https://github.com/Azure/azure-powershell
+    releaseNotes: Initial release of Portal Dashboard cmdlets.
+    requireLicenseAcceptance: true
+    scriptsToProcess:
+    - ./custom/Helpers.ps1
+    tags: Azure ResourceManager ARM PSModule Portal Dashboard
+prefix:  # the variable
+  basic:
+  - Az  # the default value
+service-name:  # by default it's calculated
+  basic:
+  - ADDomainServices
+  - CostManagement
+module-name:
+  basic:
+  - $(prefix).$(service-name)  # the default value
+  - Az.SignalR
+root-module-name: # used for sub module to generate the code in root module if there are multiple sub modules
+  basic:
+  - $(prefix).AlertsManagement
+  - $(prefix).AppConfiguration
+namespace:  # used for sub module to define the powershell class namespace
+  basic:
+  - Microsoft.Azure.PowerShell.Cmdlets.$(service-name)  # the default value
+  - Microsoft.Azure.PowerShell.Cmdlets.Metric
+subject-prefix: # the subject prefix is used in the commandlet name, which will be added after {action}-AZ{subprefix}
+  basic:
+  - '' # no subject prefix included
+  - $(service-name)  # the default value
+  - $Informatica
+  - $elastic
+  - AD
+repo:  # the variable, used to find the required files, it's used for input-files
+  basic:
+  - https://github.com/Azure/azure-rest-api-specs/tree/$(commit)  # the default value, and also support for other repos
+require:  # required readme.md files for autorest consume
+  list:
+  - $(this-folder)/../../readme.azure.noprofile.md  # always needed
+  - $(repo)/specification/appcomplianceautomation/resource-manager/readme.md
+  - $(repo)/specification/azurefleet/resource-manager/readme.md
+  - $(this-folder)/../../helpers/KeyVault/readme.noprofile.md
+  - $(this-folder)/../../helpers/ManagedIdentity/readme.noprofile.md
+endpoint-resource-id-key-name: # data-plane used, the resource id key name should be registered in azure powershell account before. It will control the client endpoint and auth.
+inlining-threshold: # flatten threshold, 40, which used to control the max allowed flatten properties into outer layer
+sanitize-names:  # TODO: need code for detail how to check the sanitize rules for the argument names
+  basic:
+  - true  # the default value
+support-json-input: # default value is true
+tag:  # used to filter out the input apis, the tag should be in the swagger readme.md file
+  basic:
+  - package-2022-10-12-preview
+  - package-2023-01-01
+try-require:  # using the directive configuration if it exists as prefix
+  list:
+  - $(repo)/specification/azurefleet/resource-manager/readme.powershell.md
+  - $(repo)/specification/containerregistry/resource-manager/readme.powershell.md
+use-extension:
+  dict:
+    '@autorest/powershell': 3.x  # always this value
diff --git a/src/aaz_dev/ps/tests/api_tests/__init__.py b/src/aaz_dev/ps/tests/api_tests/__init__.py
diff --git a/src/aaz_dev/ps/tests/api_tests/test_powershell.py b/src/aaz_dev/ps/tests/api_tests/test_powershell.py
@@ -0,0 +1,63 @@
+from ps.tests.common import CommandTestCase
+from utils.config import Config
+from utils.base64 import b64encode_str
+from utils.stage import AAZStageEnum
+# from cli.controller.az_module_manager import AzMainManager, AzExtensionManager
+import os
+import shutil
+import yaml
+
+
+class APIPowerShellTest(CommandTestCase):
+
+    def test_get_powershell_path(self):
+        with self.app.test_client() as c:
+            print(Config.POWERSHELL_PATH)
+            rv = c.get("/PS/Powershell/Path")
+            self.assertTrue(rv.status_code == 200)
+            data = rv.get_json()
+            self.assertTrue(data["path"] == Config.POWERSHELL_PATH)
+
+    def test_list_powershell_modules(self):
+        config_dict = {}
+        with self.app.test_client() as c:
+            rv = c.get("/PS/Powershell/Modules")
+            self.assertTrue(rv.status_code == 200)
+            data = rv.get_json()
+            self.assertTrue(len(data) > 100)
+            self.assertTrue(all(module["name"].endswith(".Autorest") for module in data))
+            for module in data:
+                request_url = module["url"]
+                rv = c.get(request_url)
+                self.assertTrue(rv.status_code == 200)
+                data = rv.get_json()
+                if data["autorest_config"] is None:
+                    continue
+                for key, value in data["autorest_config"].items():
+                    if key in ["directive", "commit", "input-file", "title", "module-version"]:
+                        continue
+                    if key not in config_dict:
+                        config_dict[key] = {
+                            "list": set(),
+                            "dict": {},
+                            "basic": set(),
+                        }
+                    if isinstance(value, list):
+                        config_dict[key]["list"].update(value)
+                    elif isinstance(value, dict):
+                        config_dict[key]["dict"].update(value)
+                    else:
+                        config_dict[key]["basic"].add(value)
+        for key, value in config_dict.items():
+            if not len(value["list"]):
+                del value["list"]
+            else:
+                value["list"] = sorted(list(value["list"]))
+            if not len(value["dict"]):
+                del value["dict"]
+            if not len(value["basic"]):
+                del value["basic"]
+            else:
+                value["basic"] = sorted(list(value["basic"]))
+        # with open("ps/templates/autorest/config_common_used_props.yaml", "w") as f:
+        #     yaml.dump(config_dict, f)
diff --git a/src/aaz_dev/ps/tests/common.py b/src/aaz_dev/ps/tests/common.py
@@ -0,0 +1,14 @@
+from app.tests.common import ApiTestCase
+from utils.config import Config
+# from command.tests.common import workspace_name
+# from swagger.utils.tools import swagger_resource_path_to_resource_id
+# from swagger.utils.source import SourceTypeEnum
+# from utils.plane import PlaneEnum
+# from utils.stage import AAZStageEnum
+# from utils.client import CloudEnum
+
+
+class CommandTestCase(ApiTestCase):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
