✨Added Datamate vector knowledge base core v2

Author: Zhi-a

test/backend/agents/test_create_agent_info.py

@@ -5,9 +5,9 @@
 from pathlib import Path
 from unittest.mock import AsyncMock, MagicMock, patch, Mock, PropertyMock
 
-from test.common.env_test_utils import bootstrap_env
+from test.common.test_mocks import bootstrap_test_env
 
-env_state = bootstrap_env()
+env_state = bootstrap_test_env()
 consts_const = env_state["mock_const"]
 TEST_ROOT = Path(__file__).resolve().parents[2]
 PROJECT_ROOT = TEST_ROOT.parent

test/backend/app/test_image_app.py

@@ -8,9 +8,9 @@
 if str(TEST_ROOT) not in sys.path:
     sys.path.append(str(TEST_ROOT))
 
-from test.common.env_test_utils import bootstrap_env
+from test.common.test_mocks import bootstrap_test_env
 
-helpers_env = bootstrap_env()
+helpers_env = bootstrap_test_env()
 
 
 helpers_env["mock_const"].DATA_PROCESS_SERVICE = "http://mock-data-process-service"

test/backend/app/test_knowledge_summary_app.py

@@ -12,12 +12,7 @@
     if path not in sys.path:
         sys.path.insert(0, path)
 
-# Patch environment variables before any imports that might use them
-os.environ.setdefault('MINIO_ENDPOINT', 'http://localhost:9000')
-os.environ.setdefault('MINIO_ACCESS_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_SECRET_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_REGION', 'us-east-1')
-os.environ.setdefault('MINIO_DEFAULT_BUCKET', 'test-bucket')
+# Environment variables are now configured in conftest.py
 
 # Mock external dependencies
 sys.modules['boto3'] = MagicMock()

test/backend/app/test_mock_user_management_app.py

@@ -7,12 +7,7 @@
 current_dir = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, os.path.join(current_dir, "../../../backend"))
 
-# Patch environment variables before any imports that might use them
-os.environ.setdefault('MINIO_ENDPOINT', 'http://localhost:9000')
-os.environ.setdefault('MINIO_ACCESS_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_SECRET_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_REGION', 'us-east-1')
-os.environ.setdefault('MINIO_DEFAULT_BUCKET', 'test-bucket')
+# Environment variables are now configured in conftest.py
 
 boto3_mock = MagicMock()
 minio_client_mock = MagicMock()

test/backend/app/test_model_managment_app.py

@@ -17,11 +17,7 @@
     sys.path.insert(0, BACKEND_ROOT)
 
 # Patch environment variables before any imports that might use them
-os.environ.setdefault('MINIO_ENDPOINT', 'http://localhost:9000')
-os.environ.setdefault('MINIO_ACCESS_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_SECRET_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_REGION', 'us-east-1')
-os.environ.setdefault('MINIO_DEFAULT_BUCKET', 'test-bucket')
+# Environment variables are now configured in conftest.py
 
 # Patch storage factory and MinIO config validation to avoid errors during initialization
 # These patches must be started before any imports that use MinioClient

test/backend/app/test_vectordatabase_app.py

@@ -18,12 +18,7 @@
 backend_dir = os.path.abspath(os.path.join(current_dir, "../../../backend"))
 sys.path.insert(0, backend_dir)
 
-# Patch environment variables before any imports that might use them
-os.environ.setdefault('MINIO_ENDPOINT', 'http://localhost:9000')
-os.environ.setdefault('MINIO_ACCESS_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_SECRET_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_REGION', 'us-east-1')
-os.environ.setdefault('MINIO_DEFAULT_BUCKET', 'test-bucket')
+# Environment variables are now configured in conftest.py
 
 boto3_mock = MagicMock()
 minio_client_mock = MagicMock()

test/backend/database/test_attachment_db.py

@@ -14,21 +14,10 @@
 sys.path.insert(0, os.path.abspath(os.path.join(
     os.path.dirname(__file__), '..', '..', '..')))
 
-# Mock environment variables before imports
-os.environ.setdefault('MINIO_ENDPOINT', 'http://localhost:9000')
-os.environ.setdefault('MINIO_ACCESS_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_SECRET_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_REGION', 'us-east-1')
-os.environ.setdefault('MINIO_DEFAULT_BUCKET', 'test-bucket')
-
 # Mock consts module
 consts_mock = MagicMock()
 consts_mock.const = MagicMock()
-consts_mock.const.MINIO_ENDPOINT = os.environ.get('MINIO_ENDPOINT', 'http://localhost:9000')
-consts_mock.const.MINIO_ACCESS_KEY = os.environ.get('MINIO_ACCESS_KEY', 'minioadmin')
-consts_mock.const.MINIO_SECRET_KEY = os.environ.get('MINIO_SECRET_KEY', 'minioadmin')
-consts_mock.const.MINIO_REGION = os.environ.get('MINIO_REGION', 'us-east-1')
-consts_mock.const.MINIO_DEFAULT_BUCKET = os.environ.get('MINIO_DEFAULT_BUCKET', 'test-bucket')
+# Environment variables are now configured in conftest.py
 
 sys.modules['consts'] = consts_mock
 sys.modules['consts.const'] = consts_mock.const

test/backend/database/test_client.py

@@ -13,31 +13,10 @@
 sys.path.insert(0, os.path.abspath(os.path.join(
     os.path.dirname(__file__), '..', '..', '..')))
 
-# Mock environment variables before imports
-os.environ.setdefault('MINIO_ENDPOINT', 'http://localhost:9000')
-os.environ.setdefault('MINIO_ACCESS_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_SECRET_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_REGION', 'us-east-1')
-os.environ.setdefault('MINIO_DEFAULT_BUCKET', 'test-bucket')
-os.environ.setdefault('POSTGRES_HOST', 'localhost')
-os.environ.setdefault('POSTGRES_USER', 'test_user')
-os.environ.setdefault('NEXENT_POSTGRES_PASSWORD', 'test_password')
-os.environ.setdefault('POSTGRES_DB', 'test_db')
-os.environ.setdefault('POSTGRES_PORT', '5432')
-
 # Mock consts module
 consts_mock = MagicMock()
 consts_mock.const = MagicMock()
-consts_mock.const.MINIO_ENDPOINT = os.environ.get('MINIO_ENDPOINT', 'http://localhost:9000')
-consts_mock.const.MINIO_ACCESS_KEY = os.environ.get('MINIO_ACCESS_KEY', 'minioadmin')
-consts_mock.const.MINIO_SECRET_KEY = os.environ.get('MINIO_SECRET_KEY', 'minioadmin')
-consts_mock.const.MINIO_REGION = os.environ.get('MINIO_REGION', 'us-east-1')
-consts_mock.const.MINIO_DEFAULT_BUCKET = os.environ.get('MINIO_DEFAULT_BUCKET', 'test-bucket')
-consts_mock.const.POSTGRES_HOST = os.environ.get('POSTGRES_HOST', 'localhost')
-consts_mock.const.POSTGRES_USER = os.environ.get('POSTGRES_USER', 'test_user')
-consts_mock.const.NEXENT_POSTGRES_PASSWORD = os.environ.get('NEXENT_POSTGRES_PASSWORD', 'test_password')
-consts_mock.const.POSTGRES_DB = os.environ.get('POSTGRES_DB', 'test_db')
-consts_mock.const.POSTGRES_PORT = int(os.environ.get('POSTGRES_PORT', '5432'))
+# Environment variables are now configured in conftest.py
 
 sys.modules['consts'] = consts_mock
 sys.modules['consts.const'] = consts_mock.const
@@ -51,7 +30,8 @@
 nexent_storage_mock = MagicMock()
 nexent_storage_factory_mock = MagicMock()
 storage_client_mock = MagicMock()
-nexent_storage_factory_mock.create_storage_client_from_config = MagicMock(return_value=storage_client_mock)
+nexent_storage_factory_mock.create_storage_client_from_config = MagicMock(
+    return_value=storage_client_mock)
 nexent_storage_factory_mock.MinIOStorageConfig = MagicMock()
 nexent_storage_mock.storage_client_factory = nexent_storage_factory_mock
 nexent_mock.storage = nexent_storage_mock
@@ -79,7 +59,7 @@
 
 # Patch storage factory before importing
 with patch('nexent.storage.storage_client_factory.create_storage_client_from_config', return_value=storage_client_mock), \
-     patch('nexent.storage.storage_client_factory.MinIOStorageConfig'):
+        patch('nexent.storage.storage_client_factory.MinIOStorageConfig'):
     from backend.database.client import (
         PostgresClient,
         MinioClient,
@@ -94,17 +74,26 @@
 class TestPostgresClient:
     """Test cases for PostgresClient class"""
 
-    @patch('backend.database.client.create_engine')
-    @patch('backend.database.client.sessionmaker')
-    def test_postgres_client_init(self, mock_sessionmaker, mock_create_engine):
+    def test_postgres_client_init(self, mocker):
         """Test PostgresClient initialization"""
         # Reset singleton instance
         PostgresClient._instance = None
 
+        # Patch the constants
+        mocker.patch('backend.database.client.POSTGRES_HOST', 'localhost')
+        mocker.patch('backend.database.client.POSTGRES_USER', 'test_user')
+        mocker.patch(
+            'backend.database.client.NEXENT_POSTGRES_PASSWORD', 'test_password')
+        mocker.patch('backend.database.client.POSTGRES_DB', 'test_db')
+        mocker.patch('backend.database.client.POSTGRES_PORT', 5432)
+
+        # Mock the SQLAlchemy functions
         mock_engine = MagicMock()
-        mock_create_engine.return_value = mock_engine
+        mock_create_engine = mocker.patch(
+            'backend.database.client.create_engine', return_value=mock_engine)
         mock_session = MagicMock()
-        mock_sessionmaker.return_value = mock_session
+        mock_sessionmaker = mocker.patch(
+            'backend.database.client.sessionmaker', return_value=mock_session)
 
         client = PostgresClient()
 
@@ -186,7 +175,7 @@ def test_minio_client_singleton(self):
         MinioClient._instance = None
 
         with patch('backend.database.client.create_storage_client_from_config'), \
-             patch('backend.database.client.MinIOStorageConfig'):
+                patch('backend.database.client.MinIOStorageConfig'):
             client1 = MinioClient()
             client2 = MinioClient()
 
@@ -199,16 +188,19 @@ def test_minio_client_upload_file(self, mock_config_class, mock_create_client):
         MinioClient._instance = None
 
         mock_storage_client = MagicMock()
-        mock_storage_client.upload_file.return_value = (True, '/bucket/file.txt')
+        mock_storage_client.upload_file.return_value = (
+            True, '/bucket/file.txt')
         mock_create_client.return_value = mock_storage_client
         mock_config_class.return_value = MagicMock()
 
         client = MinioClient()
-        success, result = client.upload_file('/path/to/file.txt', 'file.txt', 'bucket')
+        success, result = client.upload_file(
+            '/path/to/file.txt', 'file.txt', 'bucket')
 
         assert success is True
         assert result == '/bucket/file.txt'
-        mock_storage_client.upload_file.assert_called_once_with('/path/to/file.txt', 'file.txt', 'bucket')
+        mock_storage_client.upload_file.assert_called_once_with(
+            '/path/to/file.txt', 'file.txt', 'bucket')
 
     @patch('backend.database.client.create_storage_client_from_config')
     @patch('backend.database.client.MinIOStorageConfig')
@@ -218,7 +210,8 @@ def test_minio_client_upload_fileobj(self, mock_config_class, mock_create_client
 
         from io import BytesIO
         mock_storage_client = MagicMock()
-        mock_storage_client.upload_fileobj.return_value = (True, '/bucket/file.txt')
+        mock_storage_client.upload_fileobj.return_value = (
+            True, '/bucket/file.txt')
         mock_create_client.return_value = mock_storage_client
         mock_config_class.return_value = MagicMock()
 
@@ -228,7 +221,8 @@ def test_minio_client_upload_fileobj(self, mock_config_class, mock_create_client
 
         assert success is True
         assert result == '/bucket/file.txt'
-        mock_storage_client.upload_fileobj.assert_called_once_with(file_obj, 'file.txt', 'bucket')
+        mock_storage_client.upload_fileobj.assert_called_once_with(
+            file_obj, 'file.txt', 'bucket')
 
     @patch('backend.database.client.create_storage_client_from_config')
     @patch('backend.database.client.MinIOStorageConfig')
@@ -237,16 +231,19 @@ def test_minio_client_download_file(self, mock_config_class, mock_create_client)
         MinioClient._instance = None
 
         mock_storage_client = MagicMock()
-        mock_storage_client.download_file.return_value = (True, 'Downloaded successfully')
+        mock_storage_client.download_file.return_value = (
+            True, 'Downloaded successfully')
         mock_create_client.return_value = mock_storage_client
         mock_config_class.return_value = MagicMock()
 
         client = MinioClient()
-        success, result = client.download_file('file.txt', '/path/to/download.txt', 'bucket')
+        success, result = client.download_file(
+            'file.txt', '/path/to/download.txt', 'bucket')
 
         assert success is True
         assert result == 'Downloaded successfully'
-        mock_storage_client.download_file.assert_called_once_with('file.txt', '/path/to/download.txt', 'bucket')
+        mock_storage_client.download_file.assert_called_once_with(
+            'file.txt', '/path/to/download.txt', 'bucket')
 
     @patch('backend.database.client.create_storage_client_from_config')
     @patch('backend.database.client.MinIOStorageConfig')
@@ -255,7 +252,8 @@ def test_minio_client_get_file_url(self, mock_config_class, mock_create_client):
         MinioClient._instance = None
 
         mock_storage_client = MagicMock()
-        mock_storage_client.get_file_url.return_value = (True, 'http://example.com/file.txt')
+        mock_storage_client.get_file_url.return_value = (
+            True, 'http://example.com/file.txt')
         mock_create_client.return_value = mock_storage_client
         mock_config_class.return_value = MagicMock()
 
@@ -264,7 +262,8 @@ def test_minio_client_get_file_url(self, mock_config_class, mock_create_client):
 
         assert success is True
         assert result == 'http://example.com/file.txt'
-        mock_storage_client.get_file_url.assert_called_once_with('file.txt', 'bucket', 7200)
+        mock_storage_client.get_file_url.assert_called_once_with(
+            'file.txt', 'bucket', 7200)
 
     @patch('backend.database.client.create_storage_client_from_config')
     @patch('backend.database.client.MinIOStorageConfig')
@@ -281,7 +280,8 @@ def test_minio_client_get_file_size(self, mock_config_class, mock_create_client)
         size = client.get_file_size('file.txt', 'bucket')
 
         assert size == 1024
-        mock_storage_client.get_file_size.assert_called_once_with('file.txt', 'bucket')
+        mock_storage_client.get_file_size.assert_called_once_with(
+            'file.txt', 'bucket')
 
     @patch('backend.database.client.create_storage_client_from_config')
     @patch('backend.database.client.MinIOStorageConfig')
@@ -302,7 +302,8 @@ def test_minio_client_list_files(self, mock_config_class, mock_create_client):
 
         assert len(files) == 2
         assert files[0]['key'] == 'file1.txt'
-        mock_storage_client.list_files.assert_called_once_with('prefix/', 'bucket')
+        mock_storage_client.list_files.assert_called_once_with(
+            'prefix/', 'bucket')
 
     @patch('backend.database.client.create_storage_client_from_config')
     @patch('backend.database.client.MinIOStorageConfig')
@@ -311,7 +312,8 @@ def test_minio_client_delete_file(self, mock_config_class, mock_create_client):
         MinioClient._instance = None
 
         mock_storage_client = MagicMock()
-        mock_storage_client.delete_file.return_value = (True, 'Deleted successfully')
+        mock_storage_client.delete_file.return_value = (
+            True, 'Deleted successfully')
         mock_create_client.return_value = mock_storage_client
         mock_config_class.return_value = MagicMock()
 
@@ -320,7 +322,8 @@ def test_minio_client_delete_file(self, mock_config_class, mock_create_client):
 
         assert success is True
         assert result == 'Deleted successfully'
-        mock_storage_client.delete_file.assert_called_once_with('file.txt', 'bucket')
+        mock_storage_client.delete_file.assert_called_once_with(
+            'file.txt', 'bucket')
 
     @patch('backend.database.client.create_storage_client_from_config')
     @patch('backend.database.client.MinIOStorageConfig')
@@ -340,7 +343,8 @@ def test_minio_client_get_file_stream(self, mock_config_class, mock_create_clien
 
         assert success is True
         assert result == mock_stream
-        mock_storage_client.get_file_stream.assert_called_once_with('file.txt', 'bucket')
+        mock_storage_client.get_file_stream.assert_called_once_with(
+            'file.txt', 'bucket')
 
 
 class TestGetDbSession:
@@ -410,7 +414,8 @@ def test_filter_property_filters_correctly(self):
         mock_model = MagicMock()
         mock_model.__table__ = MagicMock()
         mock_model.__table__.columns = MagicMock()
-        mock_model.__table__.columns.keys.return_value = ['id', 'name', 'email']
+        mock_model.__table__.columns.keys.return_value = [
+            'id', 'name', 'email']
 
         data = {
             'id': 1,
@@ -454,4 +459,3 @@ def test_filter_property_no_matching_fields(self):
         result = filter_property(data, mock_model)
 
         assert result == {}
-

test/backend/services/test_agent_service.py

@@ -19,11 +19,7 @@
 
 
 # Patch environment variables before any imports that might use them
-os.environ.setdefault('MINIO_ENDPOINT', 'http://localhost:9000')
-os.environ.setdefault('MINIO_ACCESS_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_SECRET_KEY', 'minioadmin')
-os.environ.setdefault('MINIO_REGION', 'us-east-1')
-os.environ.setdefault('MINIO_DEFAULT_BUCKET', 'test-bucket')
+# Environment variables are now configured in conftest.py
 
 # Mock boto3 before importing the module under test
 boto3_mock = MagicMock()