Handling license on subdirs

src/fileLoaders.py

@@ -27,6 +27,38 @@
 class BaseLoader:
     """Base class for File Loaders"""
 
+    def _getLicenseFileFromPath(self, filepath):
+        """Returns the URL of a license file in the specified path, or None if not found."""
+        try:
+            files = self._fetchFilesFromPath(filepath)
+            for f in files:
+                if f["name"].lower() == "license" or f["name"].lower() == "licence":
+                    return f["download_url"]
+        except Exception:
+            pass
+        return None
+
+    def _fetchFilesFromPath(self, filepath):
+        """To be implemented by sub-classes. Returns a list of file items from the specified path."""
+        raise NotImplementedError("Subclasses must override _fetchFilesFromPath()!")
+
+    def getLicenceURL(self):
+        """Returns the URL of the license file in this repository if one exists.
+        Default implementation for loaders that support subdirectories."""
+        # Check subdirectory first (if subdir is set)
+        if hasattr(self, "subdir") and self.subdir:
+            licence_url = self._getLicenseFileFromPath(self.subdir.strip("/"))
+            if licence_url:
+                return licence_url
+            else:
+                glogger.debug("FileLoader -- No Subdir...")
+                # If no license found in subdirectory, check root folder
+                return self._getLicenseFileFromPath("")
+        else:
+            # If no license found in subdirectory, check root folder
+            return self._getLicenseFileFromPath("")
+        return None
+
     def getTextForName(self, query_name):
         """Return the query text and query type for the given query name.
         Note that file extention is not part of the query name. For example,
@@ -92,7 +124,12 @@ def __init__(self, user, repo, subdir=None, sha=None, prov=None):
 
     def fetchFiles(self):
         """Returns a list of file items contained on the github repo."""
-        contents = self.gh_repo.get_contents(self.subdir.strip("/"), ref=self.sha)
+        return self._fetchFilesFromPath(self.subdir)
+
+    def _fetchFilesFromPath(self, filepath):
+        """Returns a list of file items from the specified path in the github repo."""
+        filepath = filepath.strip("/")
+        contents = self.gh_repo.get_contents(filepath, ref=self.sha)
         files = []
         for content_file in contents:
             if content_file.type == "file":
@@ -161,13 +198,6 @@ def getEndpointText(self):
         """Return content of endpoint file (endpoint.txt)"""
         return self._getText("endpoint.txt")
 
-    def getLicenceURL(self):
-        """Returns the URL of the license file in this repository if one exists."""
-        for f in self.fetchFiles():
-            if f["name"].lower() == "license" or f["name"].lower() == "licence":
-                return f["download_url"]
-        return None
-
     def getRepoDescription(self):
         """Return the description of the repository"""
         return self.gh_repo.description
@@ -206,19 +236,22 @@ def __init__(self, user, repo, subdir=None, sha=None, prov=None, branch=None):
             raise Exception("Repo not found: " + user + "/" + repo)
 
     def fetchFiles(self):
-        """Returns a list of file items contained on the github repo."""
+        """Returns a list of file items contained on the gitlab repo."""
+        return self._fetchFilesFromPath(self.subdir)
+
+    def _fetchFilesFromPath(self, filepath):
+        """Returns a list of file items from the specified path in the gitlab repo."""
+        filepath = filepath.strip("/")
         gitlab_files = self.gl_repo.repository_tree(
-            path=self.subdir.strip("/"), ref=self.branch, all=True
+            path=filepath, ref=self.branch, all=True
         )
         files = []
         for gitlab_file in gitlab_files:
             if gitlab_file["type"] == "blob":
                 name = gitlab_file["name"]
                 files.append(
                     {
-                        "download_url": path.join(
-                            self.getRawRepoUri(), self.subdir, name
-                        ),
+                        "download_url": path.join(self.getRawRepoUri(), filepath, name),
                         "name": name,
                         "decoded_content": str.encode(
                             self._getText(gitlab_file["name"])
@@ -281,13 +314,6 @@ def getEndpointText(self):
         """Return content of endpoint file (endpoint.txt)"""
         return self._getText("endpoint.txt")
 
-    def getLicenceURL(self):
-        """Returns the URL of the license file in this repository if one exists."""
-        for f in self.fetchFiles():
-            if f["name"].lower() == "license" or f["name"].lower() == "licence":
-                return f["download_url"]
-        return None
-
     def getRepoDescription(self):
         """Return the description of the repository"""
         return self.gl_repo.description

src/swagger.py

@@ -60,7 +60,7 @@ def get_repo_info(loader, sha, prov_g):
         basePath = "/api-git/" + user_repo + "/"
         basePath += ("subdir/" + loader.subdir + "/") if loader.subdir else ""
         basePath += ("commit/" + sha + "/") if sha else ""
-    if type(loader) is GitlabLoader:
+    elif type(loader) is GitlabLoader:
         basePath = "/api-gitlab/" + user_repo + "/query/"
         basePath += ("branch/" + loader.branch + "/") if loader.branch else ""
         basePath += (
@@ -73,7 +73,7 @@ def get_repo_info(loader, sha, prov_g):
         basePath = "/api-url/"
     else:
         # TODO: raise error
-        glogger.error("Cannot set basePath, loader type unkown")
+        glogger.error("Cannot set basePath, loader type unknown")
 
     return prev_commit, next_commit, info, basePath
 

tests/mock_data.py

@@ -21,6 +21,9 @@
 
 def buildGHEntry(entryName):
     entryName = entryName.replace(base_url, "")
+    entryName = entryName.strip(
+        path.sep
+    )  # filenames contain extra leading / -- removed here
 
     # Named tuple containing properties of mocked github ContentFile
     MockGithubContentFile = namedtuple(
@@ -46,12 +49,17 @@ def buildGLEntry(entryName):
 
 
 class MockGithubRepo:
+    def __init__(self, subdir=False) -> None:
+        self.is_subdir = subdir
+
     def get_contents(self, filename, ref=None):
+        if self.is_subdir:
+            filename = filename.strip("subdir")
         if filename == "":
             return mock_gh_files
         else:
             for f in mock_gh_files:
-                if filename in f.name:  # filenames contain extra /
+                if filename in f.name:
                     return f
             return None
 

tests/repo/License

@@ -0,0 +1 @@
+Fake license

tests/test_loaders.py

@@ -29,8 +29,8 @@ def test_fetchFiles(self):
         # Should return a list of file items
         self.assertIsInstance(files, list, "Should return a list of file items")
 
-        # Should have N files (where N=10)
-        self.assertEqual(len(files), 10, "Should return correct number of files")
+        # Should have N files (where N=11)
+        self.assertEqual(len(files), 11, "Should return correct number of files")
 
         # File items should have a download_url
         for fItem in files:
@@ -87,6 +87,25 @@ def test_getEndpointText(self):
         # Should be some text
         self.assertIsInstance(endpoint, six.string_types, "Should be some text")
 
+    def test_getLicenceURL(self):
+        licenceURL = self.loader.getLicenceURL()
+        self.assertIsNot(licenceURL, None, "License should not be None")
+
+
+class TestGithubLoaderSubdir(unittest.TestCase):
+    @classmethod
+    @patch("grlc.fileLoaders.Github.get_repo", return_value=MockGithubRepo(subdir=True))
+    def setUpClass(self, mocked_repo):
+        self.user = "fakeuser"
+        self.repo = "fakerepo"
+        self.loader = GithubLoader(
+            self.user, self.repo, subdir="subdir", sha=None, prov=None
+        )
+
+    def test_getLicenceURL(self):
+        licenceURL = self.loader.getLicenceURL()
+        self.assertIsNot(licenceURL, None, "License should not None")
+
 
 class TestGitlabLoader(unittest.TestCase):
     @classmethod
@@ -104,8 +123,8 @@ def test_fetchFiles(self):
         # Should return a list of file items
         self.assertIsInstance(files, list, "Should return a list of file items")
 
-        # Should have N files (where N=10)
-        self.assertEqual(len(files), 10, "Should return correct number of files")
+        # Should have N files (where N=11)
+        self.assertEqual(len(files), 11, "Should return correct number of files")
 
         # File items should have a download_url
         for fItem in files:
@@ -174,8 +193,8 @@ def test_fetchFiles(self):
         # Should return a list of file items
         self.assertIsInstance(files, list, "Should return a list of file items")
 
-        # Should have N files (where N=10)
-        self.assertEqual(len(files), 10, "Should return correct number of files")
+        # Should have N files (where N=11)
+        self.assertEqual(len(files), 11, "Should return correct number of files")
 
         # File items should have a download_url
         for fItem in files: