diff --git a/fastapi_startkit/src/fastapi_startkit/application.py b/fastapi_startkit/src/fastapi_startkit/application.py index 3701052a..1c0d8a7a 100644 --- a/fastapi_startkit/src/fastapi_startkit/application.py +++ b/fastapi_startkit/src/fastapi_startkit/application.py @@ -112,6 +112,12 @@ def use_fastapi(self, fastapi: "FastAPI"): def use_base_path(self, path: str): return self.base_path / path + def storage_path(self, path: str = "") -> str: + return str(self.base_path / "storage" / path) + + def public_path(self, path: str = "") -> str: + return str(self.base_path / "public" / path) + def get(self, path: str, **kwargs) -> Callable: return self.fastapi.get(path, **kwargs) diff --git a/fastapi_startkit/src/fastapi_startkit/facades/Storage.py b/fastapi_startkit/src/fastapi_startkit/facades/Storage.py deleted file mode 100644 index 780bea08..00000000 --- a/fastapi_startkit/src/fastapi_startkit/facades/Storage.py +++ /dev/null @@ -1,5 +0,0 @@ -from .Facade import Facade - - -class Storage(metaclass=Facade): - key = "storage" diff --git a/fastapi_startkit/src/fastapi_startkit/facades/Storage.pyi b/fastapi_startkit/src/fastapi_startkit/facades/Storage.pyi deleted file mode 100644 index e0ccc2f1..00000000 --- a/fastapi_startkit/src/fastapi_startkit/facades/Storage.pyi +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Any - -class Storage: - """File storage facade.""" - - def add_driver(name: str, driver: str): ... - def set_configuration(config: dict) -> "Storage": ... - def get_driver(name: str = None) -> Any: ... - def get_config_options(name: str = None) -> dict: ... - def disk(name: str = "default") -> Any: - """Get the file manager instance for the given disk name.""" - ... diff --git a/fastapi_startkit/src/fastapi_startkit/facades/__init__.py b/fastapi_startkit/src/fastapi_startkit/facades/__init__.py index de7ed3a8..da562fb2 100644 --- a/fastapi_startkit/src/fastapi_startkit/facades/__init__.py +++ b/fastapi_startkit/src/fastapi_startkit/facades/__init__.py @@ -11,7 +11,6 @@ from .Config import Config from .Loader import Loader from .Notification import Notification -from .Storage import Storage from .Dump import Dump from .Queue import Queue from .Cache import Cache diff --git a/fastapi_startkit/src/fastapi_startkit/helpers/app.py b/fastapi_startkit/src/fastapi_startkit/helpers/app.py new file mode 100644 index 00000000..f4c6c455 --- /dev/null +++ b/fastapi_startkit/src/fastapi_startkit/helpers/app.py @@ -0,0 +1,9 @@ +def storage_path(path: str = "") -> str: + """Get the path to the storage directory.""" + from fastapi_startkit.application import app + return app().storage_path(path) + +def public_path(path: str = "") -> str: + """Get the path to the public directory.""" + from fastapi_startkit.application import app + return app().public_path(path) diff --git a/fastapi_startkit/src/fastapi_startkit/storage/__init__.py b/fastapi_startkit/src/fastapi_startkit/storage/__init__.py new file mode 100644 index 00000000..2f838374 --- /dev/null +++ b/fastapi_startkit/src/fastapi_startkit/storage/__init__.py @@ -0,0 +1 @@ +from .storage import Storage diff --git a/fastapi_startkit/src/fastapi_startkit/storage/config.py b/fastapi_startkit/src/fastapi_startkit/storage/config.py new file mode 100644 index 00000000..6237e9e5 --- /dev/null +++ b/fastapi_startkit/src/fastapi_startkit/storage/config.py @@ -0,0 +1,41 @@ +import dataclasses +from fastapi_startkit.environment import env +from fastapi_startkit.helpers.app import storage_path + +@dataclasses.dataclass +class StorageConfig: + default: str = dataclasses.field( + default_factory=lambda: env("FILESYSTEM_DISK", "local") + ) + + disks: dict = dataclasses.field( + default_factory=lambda: { + "local": { + "driver": "local", + "root": storage_path("app/private"), + "serve": True, + "throw": False, + "report": False, + }, + "public": { + "driver": "local", + "root": storage_path("app/public"), + "url": env("APP_URL", "http://localhost").rstrip("/") + "/storage", + "visibility": "public", + "throw": False, + "report": False, + }, + "s3": { + "driver": "s3", + "key": env("AWS_ACCESS_KEY_ID"), + "secret": env("AWS_SECRET_ACCESS_KEY"), + "region": env("AWS_DEFAULT_REGION", "us-east-1"), + "bucket": env("AWS_BUCKET"), + "url": env("AWS_URL"), + "endpoint": env("AWS_ENDPOINT"), + "use_path_style_endpoint": env("AWS_USE_PATH_STYLE_ENDPOINT", False), + "throw": False, + "report": False, + }, + } + ) diff --git a/fastapi_startkit/src/fastapi_startkit/storage/drivers/__init__.py b/fastapi_startkit/src/fastapi_startkit/storage/drivers/__init__.py new file mode 100644 index 00000000..570fd59e --- /dev/null +++ b/fastapi_startkit/src/fastapi_startkit/storage/drivers/__init__.py @@ -0,0 +1,2 @@ +from .local import LocalDriver +from .s3 import S3Driver \ No newline at end of file diff --git a/fastapi_startkit/src/fastapi_startkit/storage/drivers/local.py b/fastapi_startkit/src/fastapi_startkit/storage/drivers/local.py new file mode 100644 index 00000000..d900b8d0 --- /dev/null +++ b/fastapi_startkit/src/fastapi_startkit/storage/drivers/local.py @@ -0,0 +1,131 @@ +import os +import uuid +from os.path import isfile, join +from shutil import copyfile, move + +from ..FileStream import FileStream +from ..File import File +from ...utils.filesystem import get_extension + + +class LocalDriver: + def __init__(self, application): + self.application = application + self.options = {} + + def set_options(self, options): + self.options = options + return self + + def get_path(self, path): + root = self.options.get("root") or self.options.get("path") + file_path = os.path.join(root, path) + self.make_file_path_if_not_exists(file_path) + return file_path + + def get_name(self, path, alias): + extension = get_extension(path) + return f"{alias}{extension}" + + def put(self, file_path, content): + if isinstance(content, (bytes, bytearray)): + write_mode = "wb" + else: + write_mode = "w" + with open(self.get_path(os.path.join(file_path)), write_mode) as f: + f.write(content) + return content + + def put_file(self, file_path, content, name=None): + file_name = self.get_name(content.name, name or str(uuid.uuid4())) + + if hasattr(content, "get_content"): + content = content.get_content() + + if isinstance(content, str): + content = bytes(content, "utf-8") + + with open(self.get_path(os.path.join(file_path, file_name)), "wb") as f: + f.write(content) + + return os.path.join(file_path, file_name) + + def get(self, file_path): + try: + with open(self.get_path(file_path), "r") as f: + content = f.read() + + return content + except FileNotFoundError: + return None + + def exists(self, file_path): + return os.path.exists(self.get_path(file_path)) + + def missing(self, file_path): + return not self.exists(file_path) + + def stream(self, file_path): + with open(self.get_path(file_path), "r") as f: + content = f + return FileStream(content) + + def copy(self, from_file_path, to_file_path): + return copyfile(from_file_path, to_file_path) + + def move(self, from_file_path, to_file_path): + return move(self.get_path(from_file_path), self.get_path(to_file_path)) + + def prepend(self, file_path, content): + value = self.get(file_path) + content = content + value + self.put(file_path, content) + return content + + def append(self, file_path, content): + with open(self.get_path(file_path), "a") as f: + f.write(content) + return content + + def delete(self, file_path): + return os.remove(self.get_path(file_path)) + + def make_directory(self, directory): + pass + + def store(self, file, name=None): + if name: + name = f"{name}{file.extension()}" + full_path = self.get_path(name or file.hash_path_name()) + with open(full_path, "wb") as f: + f.write(file.stream()) + + return full_path + + def make_file_path_if_not_exists(self, file_path): + if not os.path.isfile(file_path): + if not os.path.exists(os.path.dirname(file_path)): + # Create the path to the model if it does not exist + os.makedirs(os.path.dirname(file_path)) + + return True + + return False + + def get_files(self, directory=""): + file_path = self.get_path(directory) + files = [] + for f in os.listdir(file_path): + if not isfile(join(file_path, f)): + continue + + files.append(File(self.get(f), f)) + + return files + + def download(self, file_path, name=None, force=False): + from fastapi.responses import FileResponse + return FileResponse( + self.get_path(file_path), + filename=name or os.path.basename(file_path) + ) \ No newline at end of file diff --git a/fastapi_startkit/src/fastapi_startkit/storage/drivers/s3.py b/fastapi_startkit/src/fastapi_startkit/storage/drivers/s3.py new file mode 100644 index 00000000..562dfdd1 --- /dev/null +++ b/fastapi_startkit/src/fastapi_startkit/storage/drivers/s3.py @@ -0,0 +1,194 @@ +import os +import uuid + +from ..FileStream import FileStream +from ..File import File +from ...utils.filesystem import get_extension + + +class S3Driver: + def __init__(self, application): + self.application = application + self.options = {} + self.connection = None + + def set_options(self, options): + self.options = options + return self + + def get_connection(self): + try: + import boto3 + except ImportError: + raise ModuleNotFoundError( + "Could not find the 'boto3' library. Run 'pip install boto3' to fix this." + ) + + if not self.connection: + self.connection = boto3.Session( + aws_access_key_id=self.options.get("key") or self.options.get("client"), + aws_secret_access_key=self.options.get("secret"), + region_name=self.options.get("region"), + ) + + return self.connection + + def get_client(self): + import botocore.config + config = botocore.config.Config( + s3={'addressing_style': 'path' if self.options.get("use_path_style_endpoint") else 'auto'} + ) + return self.get_connection().client( + "s3", + endpoint_url=self.options.get("endpoint"), + config=config + ) + + def get_resource(self): + import botocore.config + config = botocore.config.Config( + s3={'addressing_style': 'path' if self.options.get("use_path_style_endpoint") else 'auto'} + ) + return self.get_connection().resource( + "s3", + endpoint_url=self.options.get("endpoint"), + config=config + ) + + def get_bucket(self): + return self.options.get("bucket") + + def get_name(self, path, alias): + extension = get_extension(path) + return f"{alias}{extension}" + + def put(self, file_path, content): + self.get_resource().Bucket(self.get_bucket()).put_object( + Key=file_path, Body=content + ) + return content + + def put_file(self, file_path, content, name=None): + file_name = self.get_name(content.name, name or str(uuid.uuid4())) + + if hasattr(content, "get_content"): + content = content.get_content() + + self.get_resource().Bucket(self.get_bucket()).put_object( + Key=os.path.join(file_path, file_name), Body=content + ) + return os.path.join(file_path, file_name) + + def get(self, file_path): + try: + return ( + self.get_resource() + .Bucket(self.get_bucket()) + .Object(file_path) + .get() + .get("Body") + .read() + .decode("utf-8") + ) + except self.missing_file_exceptions(): + pass + + def missing_file_exceptions(self): + import botocore + + return (botocore.exceptions.ClientError,) + + def exists(self, file_path): + try: + self.get_resource().Bucket(self.get_bucket()).Object( + file_path + ).load() + return True + except self.missing_file_exceptions(): + return False + + def missing(self, file_path): + return not self.exists(file_path) + + def stream(self, file_path): + return FileStream( + self.get_resource() + .Bucket(self.get_bucket()) + .Object(file_path) + .get() + .get("Body") + .read(), + file_path, + ) + + def copy(self, from_file_path, to_file_path): + copy_source = {"Bucket": self.get_bucket(), "Key": from_file_path} + self.get_resource().meta.client.copy( + copy_source, self.get_bucket(), to_file_path + ) + + def move(self, from_file_path, to_file_path): + self.copy(from_file_path, to_file_path) + self.delete(from_file_path) + + def prepend(self, file_path, content): + value = self.get(file_path) + content = content + value + self.put(file_path, content) + return content + + def append(self, file_path, content): + value = self.get(file_path) or "" + value += content + self.put(file_path, content) + + def delete(self, file_path): + return ( + self.get_resource() + .Object(self.get_bucket(), file_path) + .delete() + ) + + def store(self, file, name=None): + full_path = name or file.hash_path_name() + self.get_resource().Bucket(self.get_bucket()).put_object( + Key=full_path, Body=file.stream() + ) + return full_path + + def make_file_path_if_not_exists(self, file_path): + if not os.path.isfile(file_path): + if not os.path.exists(os.path.dirname(file_path)): + # Create the path to the model if it does not exist + os.makedirs(os.path.dirname(file_path)) + + return True + + return False + + def get_files(self, directory=None): + bucket = self.get_resource().Bucket(self.get_bucket()) + + if directory: + objects = bucket.objects.all().filter(Prefix=directory) + else: + objects = bucket.objects.all() + + files = [] + for my_bucket_object in objects.all(): + if "/" not in my_bucket_object.key: + files.append(File(my_bucket_object, my_bucket_object.key)) + + return files + + def download(self, file_path, name=None, force=False): + url = self.get_client().generate_presigned_url( + "get_object", + Params={"Bucket": self.get_bucket(), "Key": file_path}, + ExpiresIn=3600, + ) + from fastapi.responses import RedirectResponse + return RedirectResponse(url) + + def url(self, file_path): + return f"{self.options.get('url')}/{file_path}" \ No newline at end of file diff --git a/fastapi_startkit/src/fastapi_startkit/storage/providers/provider.py b/fastapi_startkit/src/fastapi_startkit/storage/providers/provider.py new file mode 100644 index 00000000..aa154b62 --- /dev/null +++ b/fastapi_startkit/src/fastapi_startkit/storage/providers/provider.py @@ -0,0 +1,29 @@ +from pathlib import Path +from ...providers import Provider +from ..storage import StorageManager +from ...configuration import config +from ..drivers import LocalDriver, S3Driver +from ..config import StorageConfig + + +class StorageProvider(Provider): + def __init__(self, application): + self.application = application + + def register(self): + config_data = self.resolve_config(StorageConfig) + self.merge_config_from(config_data, "storage") + + storage = StorageManager(self.application, config("storage")) + storage.add_driver("local", LocalDriver(self.application)) + storage.add_driver("s3", S3Driver(self.application)) + self.application.bind("storage", storage) + + def boot(self): + self.publishes( + { + Path(__file__) + .resolve() + .parent.parent.joinpath("config.py"): "config/storage.py" + } + ) \ No newline at end of file diff --git a/fastapi_startkit/src/fastapi_startkit/storage/storage.py b/fastapi_startkit/src/fastapi_startkit/storage/storage.py new file mode 100644 index 00000000..c2c80dec --- /dev/null +++ b/fastapi_startkit/src/fastapi_startkit/storage/storage.py @@ -0,0 +1,177 @@ +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from ..foundation import Application + + +class StorageManager: + """File storage manager handling managing files with different drivers.""" + + def __init__(self, application: "Application", store_config: dict = None): + self.application = application + self.drivers = {} + self.store_config = store_config or {} + self.options = {} + + def add_driver(self, name: str, driver: Any): + self.drivers.update({name: driver}) + + def set_configuration(self, config: dict) -> "StorageManager": + self.store_config = config + return self + + def get_driver(self, name: str = None) -> Any: + if name is None: + name = self.store_config.get("default") + + driver_name = self.get_config_options(name).get("driver") + return self.drivers[driver_name] + + def get_config_options(self, name: str = None) -> dict: + disks = self.store_config.get("disks", {}) + if name is None or name == "default": + name = self.store_config.get("default") + + return disks.get(name, {}) + + def disk(self, name: str = "default") -> Any: + """Get the file manager instance for the given disk name.""" + if name == "default": + name = self.store_config.get("default") + + store_config = self.get_config_options(name) + driver = self.get_driver(name) + return driver.set_options(store_config) + + def fake(self, name: str = "default") -> Any: + """Replace the given disk with a fake local disk for testing.""" + if name == "default": + name = self.store_config.get("default", "local") + + from .drivers import LocalDriver + import tempfile + + # Use a temporary directory for the fake storage + temp_dir = tempfile.mkdtemp(prefix=f"storage_fake_{name}_") + + fake_driver = LocalDriver(self.application) + fake_driver.set_options({"root": temp_dir}) + + # Replace the driver in the manager + self.drivers[name] = fake_driver + return fake_driver + + def put(self, *args, **kwargs): + return self.disk().put(*args, **kwargs) + + def get(self, *args, **kwargs): + return self.disk().get(*args, **kwargs) + + def exists(self, *args, **kwargs): + return self.disk().exists(*args, **kwargs) + + def missing(self, *args, **kwargs): + return self.disk().missing(*args, **kwargs) + + def stream(self, *args, **kwargs): + return self.disk().stream(*args, **kwargs) + + def copy(self, *args, **kwargs): + return self.disk().copy(*args, **kwargs) + + def move(self, *args, **kwargs): + return self.disk().move(*args, **kwargs) + + def prepend(self, *args, **kwargs): + return self.disk().prepend(*args, **kwargs) + + def append(self, *args, **kwargs): + return self.disk().append(*args, **kwargs) + + def delete(self, *args, **kwargs): + return self.disk().delete(*args, **kwargs) + + def store(self, *args, **kwargs): + return self.disk().store(*args, **kwargs) + + def download(self, *args, **kwargs): + return self.disk().download(*args, **kwargs) + + def url(self, *args, **kwargs): + return self.disk().url(*args, **kwargs) + + +class Storage: + instance = None + + def __init__(self): + from fastapi_startkit.application import app + self.app = app() + self.storage: StorageManager = self.app.make("storage") + + @classmethod + def init(cls) -> StorageManager: + if cls.instance: + return cls.instance.storage + cls.instance = Storage() + return cls.instance.storage + + @classmethod + def disk(cls, name="default"): + return cls.init().disk(name) + + @classmethod + def fake(cls, name="default"): + return cls.init().fake(name) + + @classmethod + def put(cls, *args, **kwargs): + return cls.init().put(*args, **kwargs) + + @classmethod + def get(cls, *args, **kwargs): + return cls.init().get(*args, **kwargs) + + @classmethod + def exists(cls, *args, **kwargs): + return cls.init().exists(*args, **kwargs) + + @classmethod + def missing(cls, *args, **kwargs): + return cls.init().missing(*args, **kwargs) + + @classmethod + def stream(cls, *args, **kwargs): + return cls.init().stream(*args, **kwargs) + + @classmethod + def copy(cls, *args, **kwargs): + return cls.init().copy(*args, **kwargs) + + @classmethod + def move(cls, *args, **kwargs): + return cls.init().move(*args, **kwargs) + + @classmethod + def prepend(cls, *args, **kwargs): + return cls.init().prepend(*args, **kwargs) + + @classmethod + def append(cls, *args, **kwargs): + return cls.init().append(*args, **kwargs) + + @classmethod + def delete(cls, *args, **kwargs): + return cls.init().delete(*args, **kwargs) + + @classmethod + def store(cls, *args, **kwargs): + return cls.init().store(*args, **kwargs) + + @classmethod + def download(cls, *args, **kwargs): + return cls.init().download(*args, **kwargs) + + @classmethod + def url(cls, *args, **kwargs): + return cls.init().url(*args, **kwargs) \ No newline at end of file diff --git a/fastapi_startkit/tests/storage/test_storage.py b/fastapi_startkit/tests/storage/test_storage.py new file mode 100644 index 00000000..831ce25a --- /dev/null +++ b/fastapi_startkit/tests/storage/test_storage.py @@ -0,0 +1,109 @@ +import unittest +from unittest.mock import MagicMock, patch +from fastapi_startkit.storage.storage import StorageManager, Storage + +class TestStorage(unittest.TestCase): + def setUp(self): + self.mock_app = MagicMock() + self.config = { + "default": "local", + "disks": { + "local": { + "driver": "local", + "root": "/tmp/storage", + }, + "s3": { + "driver": "s3", + "key": "aws-key", + "secret": "aws-secret", + "region": "us-east-1", + "bucket": "my-bucket", + } + } + } + self.storage_manager = StorageManager(self.mock_app).set_configuration(self.config) + self.mock_driver = MagicMock() + self.storage_manager.add_driver("local", self.mock_driver) + + # Reset Storage singleton + Storage.instance = None + + def test_get_config_options(self): + self.assertEqual(self.storage_manager.get_config_options("local")["root"], "/tmp/storage") + self.assertEqual(self.storage_manager.get_config_options("default")["root"], "/tmp/storage") + + def test_disk_resolution(self): + self.mock_driver.set_options.return_value = self.mock_driver + driver = self.storage_manager.disk("local") + self.assertEqual(driver, self.mock_driver) + self.mock_driver.set_options.assert_called_with(self.config["disks"]["local"]) + + @patch("fastapi_startkit.application.app") + def test_storage_proxy_methods(self, mock_app_getter): + mock_app_getter.return_value = self.mock_app + self.mock_app.make.return_value = self.storage_manager + + # Mock disk() to return the mock driver + self.mock_driver.set_options.return_value = self.mock_driver + + # Test proxying put + Storage.put("test.txt", "content") + self.mock_driver.put.assert_called_with("test.txt", "content") + + # Test disk selection proxy + Storage.disk("local") + self.mock_driver.set_options.assert_called() + + def test_storage_singleton_behavior(self): + with patch("fastapi_startkit.application.app") as mock_app_getter: + mock_app_getter.return_value = self.mock_app + self.mock_app.make.return_value = self.storage_manager + + s1 = Storage.init() + s2 = Storage.init() + self.assertEqual(s1, s2) + self.assertEqual(s1, self.storage_manager) + + def test_local_driver_download(self): + from fastapi_startkit.storage.drivers.local import LocalDriver + from fastapi.responses import FileResponse + + driver = LocalDriver(self.mock_app) + driver.set_options({"root": "/tmp"}) + + with patch("os.makedirs"): + response = driver.download("test.txt") + self.assertIsInstance(response, FileResponse) + self.assertEqual(response.path, "/tmp/test.txt") + + def test_s3_driver_download(self): + from fastapi_startkit.storage.drivers.s3 import S3Driver + from fastapi.responses import RedirectResponse + + driver = S3Driver(self.mock_app) + driver.set_options({"bucket": "test-bucket", "key": "key", "secret": "secret"}) + + with patch.object(driver, "get_client") as mock_client_getter: + mock_client = MagicMock() + mock_client_getter.return_value = mock_client + mock_client.generate_presigned_url.return_value = "https://s3.url" + + response = driver.download("test.txt") + self.assertIsInstance(response, RedirectResponse) + self.assertEqual(str(response.url), "https://s3.url") + + def test_storage_fake(self): + with patch("fastapi_startkit.application.app") as mock_app_getter: + mock_app_getter.return_value = self.mock_app + self.mock_app.make.return_value = self.storage_manager + + # Fake the 's3' disk + Storage.fake('s3') + + # The driver for 's3' should now be a LocalDriver (the fake) + from fastapi_startkit.storage.drivers.local import LocalDriver + driver = self.storage_manager.get_driver('s3') + self.assertIsInstance(driver, LocalDriver) + + # Verify it uses a temporary directory + self.assertTrue(driver.options['root'].startswith('/tmp/storage_fake_s3_')) diff --git a/fastapi_startkit/uv.lock b/fastapi_startkit/uv.lock index 97e7d28b..277e2571 100644 --- a/fastapi_startkit/uv.lock +++ b/fastapi_startkit/uv.lock @@ -443,7 +443,7 @@ wheels = [ [[package]] name = "fastapi-startkit" -version = "0.19.0" +version = "0.20.0" source = { editable = "." } dependencies = [ { name = "cleo" },