Update to official name (#19)

* Use official name

* Rename directory

* Update pyproject toml

* Update import for new elm version

* Init compass logger

* More name updates

* Init compass logger also

* Docstring updates

* Minor formatting

* Docstring updates

* Update exceptions

* Drop dep on `TEST_DATA_DIR`

* Add back in missing imports

Author: ppinchuk

.github/workflows/lint-python.yml

@@ -5,12 +5,12 @@ on:
     branches-ignore:
       - 'gh-pages'
     paths:
-      - scraper/**
+      - compass/**
       - pyproject.toml
       - .github/workflows/lint-python.yml
   pull_request:
     paths:
-      - scraper/**
+      - compass/**
       - pyproject.toml
       - .github/workflows/lint-python.yml
 
@@ -24,9 +24,9 @@ jobs:
         with:
           version: "latest"
           args: "check"
-          src: "./scraper"
+          src: "./compass"
       - uses: astral-sh/ruff-action@v3
         with:
           version: "latest"
           args: "format --check"
-          src: "./scraper"
+          src: "./compass"

.gitignore

@@ -24,7 +24,7 @@ docs/source/jupyter/*.png
 docs/source/jupyter/*_*.rst
 
 # Dynamic version
-scraper/_version.py
+compass/_version.py
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

scraper/__init__.py compass/__init__.pyscraper/__init__.py renamed to compass/__init__.py

@@ -8,8 +8,8 @@
 import multiprocessing
 from pathlib import Path
 
-from scraper import __version__
-from scraper.process import process_counties_with_openai
+from compass import __version__
+from compass.process import process_counties_with_openai
 
 
 @click.group()
@@ -28,7 +28,7 @@ def main(ctx):
     type=click.Path(exists=True),
     help="Path to ordinance configuration JSON file. This file "
     "should contain any/all the arguments to pass to "
-    ":func:`scraper.process.process_counties_with_openai`.",
+    ":func:`compass.process.process_counties_with_openai`.",
 )
 @click.option(
     "-v",
@@ -43,9 +43,10 @@ def process(config, verbose):
         config = json.load(fh)
 
     if verbose:
-        logger = logging.getLogger("scraper")
-        logger.addHandler(logging.StreamHandler(stream=sys.stdout))
-        logger.setLevel(config.get("log_level", "INFO"))
+        for lib in ["compass", "elm"]:
+            logger = logging.getLogger(lib)
+            logger.addHandler(logging.StreamHandler(stream=sys.stdout))
+            logger.setLevel(config.get("log_level", "INFO"))
 
     # Need to set start method to "spawn" instead of "fork" for unix
     # systems. If this call is not present, software hangs when process

scraper/cli.py compass/cli.pyscraper/cli.py renamed to compass/cli.py

@@ -3,12 +3,13 @@
 import logging
 
 from elm.web.document import PDFDocument
-from elm.web.google_search import google_results_as_docs, filter_documents
+from elm.web.search.run import web_search_links_as_docs
+from elm.web.utilities import filter_documents
 
-from scraper.llm import StructuredLLMCaller
-from scraper.extraction import check_for_ordinance_info
-from scraper.services.threaded import TempFileCache
-from scraper.validation.location import CountyValidator
+from compass.llm import StructuredLLMCaller
+from compass.extraction import check_for_ordinance_info
+from compass.services.threaded import TempFileCache
+from compass.validation.location import CountyValidator
 
 
 logger = logging.getLogger(__name__)
@@ -34,7 +35,7 @@ async def download_county_ordinance(
 
     Parameters
     ----------
-    location : :class:`scraper.utilities.location.Location`
+    location : :class:`compass.utilities.location.Location`
         Location objects representing the county.
     text_splitter : obj, optional
         Instance of an object that implements a `split_text` method.
@@ -50,15 +51,15 @@ async def download_county_ordinance(
         Dictionary of keyword-argument pairs to initialize
         :class:`elm.web.file_loader.AsyncFileLoader` with. If found, the
         "pw_launch_kwargs" key in these will also be used to initialize
-        the :class:`elm.web.google_search.PlaywrightGoogleLinkSearch`
+        the :class:`elm.web.search.google.PlaywrightGoogleLinkSearch`
         used for the google URL search. By default, ``None``.
     browser_semaphore : :class:`asyncio.Semaphore`, optional
         Semaphore instance that can be used to limit the number of
         playwright browsers open concurrently. If ``None``, no limits
         are applied. By default, ``None``.
     **kwargs
         Keyword-value pairs used to initialize an
-        `scraper.llm.LLMCaller` instance.
+        `compass.llm.LLMCaller` instance.
 
     Returns
     -------
@@ -101,7 +102,7 @@ async def _docs_from_google_search(
             "file_cache_coroutine": TempFileCache.call,
         }
     )
-    return await google_results_as_docs(
+    return await web_search_links_as_docs(
         queries,
         num_urls=num_urls,
         browser_semaphore=browser_semaphore,

scraper/data/conus_counties.csv compass/data/conus_counties.csvscraper/data/conus_counties.csv renamed to compass/data/conus_counties.csv

@@ -0,0 +1,24 @@
+"""Custom Exceptions and Errors for COMPASS"""
+
+import logging
+
+
+logger = logging.getLogger("compass")
+
+
+class COMPASSError(Exception):
+    """Generic COMPASS Error"""
+
+    def __init__(self, *args, **kwargs):
+        """Init exception and broadcast message to logger"""
+        super().__init__(*args, **kwargs)
+        if args:
+            logger.error(str(args[0]), stacklevel=2)
+
+
+class COMPASSNotInitializedError(COMPASSError):
+    """COMPASS not initialized error"""
+
+
+class COMPASSValueError(COMPASSError, ValueError):
+    """COMPASS ValueError"""

scraper/download.py compass/download.pyscraper/download.py renamed to compass/download.py

@@ -3,13 +3,13 @@
 import logging
 from warnings import warn
 
-from scraper.llm import LLMCaller, StructuredLLMCaller
-from scraper.extraction.date import DateExtractor
-from scraper.extraction.ordinance import (
+from compass.llm import LLMCaller, StructuredLLMCaller
+from compass.extraction.date import DateExtractor
+from compass.extraction.ordinance import (
     OrdinanceValidator,
     OrdinanceExtractor,
 )
-from scraper.extraction.parse import StructuredOrdinanceParser
+from compass.extraction.parse import StructuredOrdinanceParser
 
 
 logger = logging.getLogger(__name__)
@@ -33,7 +33,7 @@ async def check_for_ordinance_info(doc, text_splitter, **kwargs):
         input.
     **kwargs
         Keyword-value pairs used to initialize an
-        `scraper.llm.LLMCaller` instance.
+        `compass.llm.LLMCaller` instance.
 
     Returns
     -------
@@ -72,7 +72,7 @@ async def extract_ordinance_text_with_llm(doc, text_splitter, extractor):
         A document known to contain ordinance information. This means it
         must contain an ``"ordinance_text"`` key in the metadata. You
         can run
-        :func:`~scraper.extraction.apply.check_for_ordinance_info`
+        :func:`~compass.extraction.apply.check_for_ordinance_info`
         to have this attribute populated automatically for documents
         that are found to contain ordinance data. Note that if the
         document's metadata does not contain the ``"ordinance_text"``
@@ -82,8 +82,8 @@ async def extract_ordinance_text_with_llm(doc, text_splitter, extractor):
         The method should take text as input (str) and return a list
         of text chunks. Langchain's text splitters should work for this
         input.
-    extractor : scraper.extraction.ordinance.OrdinanceExtractor
-        Instance of `~scraper.extraction.ordinance.OrdinanceExtractor`
+    extractor : compass.extraction.ordinance.OrdinanceExtractor
+        Instance of `~compass.extraction.ordinance.OrdinanceExtractor`
         used for ordinance text extraction.
 
     Returns
@@ -130,7 +130,7 @@ async def extract_ordinance_text_with_ngram_validation(
         A document known to contain ordinance information. This means it
         must contain an ``"ordinance_text"`` key in the metadata. You
         can run
-        :func:`~scraper.extraction.apply.check_for_ordinance_info`
+        :func:`~compass.extraction.apply.check_for_ordinance_info`
         to have this attribute populated automatically for documents
         that are found to contain ordinance data. Note that if the
         document's metadata does not contain the ``"ordinance_text"``
@@ -156,7 +156,7 @@ async def extract_ordinance_text_with_ngram_validation(
         By default, ``0.95``.
     **kwargs
         Keyword-value pairs used to initialize an
-        `scraper.llm.LLMCaller` instance.
+        `compass.llm.LLMCaller` instance.
 
     Returns
     -------
@@ -199,7 +199,7 @@ async def _extract_with_ngram_check(
     ngram_fraction_threshold=0.95,
 ):
     """Extract ordinance info from doc and validate using ngrams."""
-    from scraper.extraction.ngrams import sentence_ngram_containment  # noqa
+    from compass.extraction.ngrams import sentence_ngram_containment  # noqa
 
     source = doc.metadata.get("source", "Unknown")
     og_text = doc.metadata["ordinance_text"]
@@ -277,14 +277,14 @@ async def extract_ordinance_values(doc, **kwargs):
         A document known to contain ordinance text. This means it must
         contain an ``"cleaned_ordinance_text"`` key in the metadata. You
         can run
-        :func:`~scraper.extraction.apply.extract_ordinance_text_with_llm`
+        :func:`~compass.extraction.apply.extract_ordinance_text_with_llm`
         to have this attribute populated automatically for documents
         that are found to contain ordinance data. Note that if the
         document's metadata does not contain the
         ``"cleaned_ordinance_text"`` key, it will not be processed.
     **kwargs
         Keyword-value pairs used to initialize an
-        `scraper.llm.LLMCaller` instance.
+        `compass.llm.LLMCaller` instance.
 
     Returns
     -------

compass/exceptions.py

@@ -30,7 +30,7 @@ def __init__(self, structured_llm_caller):
 
         Parameters
         ----------
-        structured_llm_caller : scraper.llm.StructuredLLMCaller
+        structured_llm_caller : compass.llm.StructuredLLMCaller
             StructuredLLMCaller instance. Used for structured validation
             queries.
         """