diff --git a/amber/src/main/python/pytexera/storage/dataset_file_document.py b/amber/src/main/python/pytexera/storage/dataset_file_document.py index 5a063f60470..9665abbe670 100644 --- a/amber/src/main/python/pytexera/storage/dataset_file_document.py +++ b/amber/src/main/python/pytexera/storage/dataset_file_document.py @@ -22,6 +22,8 @@ from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry +from .resource_type import ResourceType + class DatasetFileDocument: # (connect, read) timeout and retry settings for the file-service GETs below. @@ -56,20 +58,23 @@ def __init__(self, file_path: str): Parses the file path into dataset metadata. :param file_path: - Expected format - "/ownerEmail/datasetName/versionName/fileRelativePath" - Example: "/bob@texera.com/twitterDataset/v1/california/irvine/tw1.csv" + Expected format - + "/datasets/ownerEmail/datasetName/versionName/fileRelativePath" + Example: + "/datasets/bob@texera.com/twitterDataset/v1/california/tw1.csv" """ parts = file_path.strip("/").split("/") - if len(parts) < 4: + + if len(parts) < 5 or parts[0] != ResourceType.DATASETS.value: raise ValueError( - "Invalid file path format. " - "Expected: /ownerEmail/datasetName/versionName/fileRelativePath" + "Invalid file path format. Expected: " + "/datasets/ownerEmail/datasetName/versionName/fileRelativePath" ) - self.owner_email = parts[0] - self.dataset_name = parts[1] - self.version_name = parts[2] - self.file_relative_path = "/".join(parts[3:]) + self.owner_email = parts[1] + self.dataset_name = parts[2] + self.version_name = parts[3] + self.file_relative_path = "/".join(parts[4:]) self.jwt_token = os.getenv("USER_JWT_TOKEN") self.presign_endpoint = os.getenv("FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT") @@ -90,6 +95,7 @@ def get_presigned_url(self) -> str: """ headers = {"Authorization": f"Bearer {self.jwt_token}"} encoded_file_path = urllib.parse.quote( + f"/{ResourceType.DATASETS.value}" f"/{self.owner_email}" f"/{self.dataset_name}" f"/{self.version_name}" diff --git a/amber/src/main/python/pytexera/storage/resource_type.py b/amber/src/main/python/pytexera/storage/resource_type.py new file mode 100644 index 00000000000..596fb5d7453 --- /dev/null +++ b/amber/src/main/python/pytexera/storage/resource_type.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from enum import Enum + + +class ResourceType(str, Enum): + """The leading segment of a logical file path, identifying the resource kind""" + + DATASETS = "datasets" diff --git a/amber/src/test/python/pytexera/storage/test_dataset_file_document.py b/amber/src/test/python/pytexera/storage/test_dataset_file_document.py index 36882fe27ee..fba9b2e1d22 100644 --- a/amber/src/test/python/pytexera/storage/test_dataset_file_document.py +++ b/amber/src/test/python/pytexera/storage/test_dataset_file_document.py @@ -44,51 +44,61 @@ def make_response(status_code: int, body=None, content: bytes = b""): class TestDatasetFileDocumentInit: - def test_parses_minimal_four_part_path(self, auth_env): - doc = DatasetFileDocument("/bob@x.com/ds/v1/file.csv") + def test_parses_prefixed_path(self, auth_env): + doc = DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") assert doc.owner_email == "bob@x.com" assert doc.dataset_name == "ds" assert doc.version_name == "v1" assert doc.file_relative_path == "file.csv" def test_joins_nested_relative_path_back_with_slashes(self, auth_env): - doc = DatasetFileDocument("/bob@x.com/ds/v1/a/b/c/file.csv") + doc = DatasetFileDocument("/datasets/bob@x.com/ds/v1/a/b/c/file.csv") assert doc.file_relative_path == "a/b/c/file.csv" def test_strips_leading_and_trailing_slashes_before_parsing(self, auth_env): - doc = DatasetFileDocument("///bob@x.com/ds/v1/file.csv///") + doc = DatasetFileDocument("///datasets/bob@x.com/ds/v1/file.csv///") assert doc.owner_email == "bob@x.com" assert doc.file_relative_path == "file.csv" - def test_rejects_path_with_fewer_than_four_segments(self, auth_env): + def test_rejects_unprefixed_path(self, auth_env): + # Without the datasets prefix the path is not a dataset path. with pytest.raises(ValueError, match="Invalid file path format"): - DatasetFileDocument("/bob@x.com/ds/v1") + DatasetFileDocument("/bob@x.com/ds/v1/file.csv") + + def test_rejects_non_datasets_prefix(self, auth_env): + # Any other leading segment (e.g. "models") is not the datasets prefix. + with pytest.raises(ValueError, match="Invalid file path format"): + DatasetFileDocument("/models/bob@x.com/ds/v1/file.csv") + + def test_rejects_path_with_too_few_segments(self, auth_env): + with pytest.raises(ValueError, match="Invalid file path format"): + DatasetFileDocument("/datasets/bob@x.com/ds/v1") def test_requires_jwt_token_in_environment(self, monkeypatch): monkeypatch.delenv("USER_JWT_TOKEN", raising=False) monkeypatch.setenv("FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT", CUSTOM_ENDPOINT) with pytest.raises(ValueError, match="JWT token is required"): - DatasetFileDocument("/bob@x.com/ds/v1/file.csv") + DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") def test_treats_empty_jwt_as_missing(self, monkeypatch): # An empty string is falsy and should be rejected just like an unset var. monkeypatch.setenv("USER_JWT_TOKEN", "") with pytest.raises(ValueError, match="JWT token is required"): - DatasetFileDocument("/bob@x.com/ds/v1/file.csv") + DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") def test_falls_back_to_default_endpoint_when_env_missing(self, monkeypatch): monkeypatch.setenv("USER_JWT_TOKEN", "tok") monkeypatch.delenv("FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT", raising=False) - doc = DatasetFileDocument("/bob@x.com/ds/v1/file.csv") + doc = DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") assert doc.presign_endpoint == DEFAULT_ENDPOINT def test_uses_explicit_endpoint_from_environment(self, auth_env): - doc = DatasetFileDocument("/bob@x.com/ds/v1/file.csv") + doc = DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") assert doc.presign_endpoint == CUSTOM_ENDPOINT class TestGetPresignedUrl: - def _make_doc(self, monkeypatch, path="/bob@x.com/ds/v1/file.csv"): + def _make_doc(self, monkeypatch, path="/datasets/bob@x.com/ds/v1/file.csv"): monkeypatch.setenv("USER_JWT_TOKEN", "test-jwt-token") monkeypatch.setenv("FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT", CUSTOM_ENDPOINT) return DatasetFileDocument(path) @@ -116,7 +126,9 @@ def test_sends_bearer_authorization_header_with_jwt(self, monkeypatch): def test_url_encodes_filepath_query_parameter(self, monkeypatch): # urllib.parse.quote keeps "/" as safe by default, but encodes "@" # and " " — pin both pieces so the contract is explicit. - doc = self._make_doc(monkeypatch, path="/bob@x.com/ds/v1/data file.csv") + doc = self._make_doc( + monkeypatch, path="/datasets/bob@x.com/ds/v1/data file.csv" + ) with patch( "pytexera.storage.dataset_file_document.requests.Session.get" ) as mock_get: @@ -128,6 +140,17 @@ def test_url_encodes_filepath_query_parameter(self, monkeypatch): assert "bob%40x.com" in file_path assert file_path.startswith("/") + def test_sends_datasets_prefixed_filepath(self, monkeypatch): + # The reconstructed filePath sent to the file-service carries the "datasets" prefix. + doc = self._make_doc(monkeypatch, path="/datasets/bob@x.com/ds/v1/file.csv") + with patch( + "pytexera.storage.dataset_file_document.requests.Session.get" + ) as mock_get: + mock_get.return_value = make_response(200, body={"presignedUrl": "u"}) + doc.get_presigned_url() + _, kwargs = mock_get.call_args + assert kwargs["params"]["filePath"].startswith("/datasets/") + def test_calls_configured_endpoint(self, monkeypatch): doc = self._make_doc(monkeypatch) with patch( @@ -192,7 +215,7 @@ class TestReadFile: def _make_doc(self, monkeypatch): monkeypatch.setenv("USER_JWT_TOKEN", "test-jwt-token") monkeypatch.setenv("FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT", CUSTOM_ENDPOINT) - return DatasetFileDocument("/bob@x.com/ds/v1/file.csv") + return DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") def test_returns_bytesio_with_downloaded_content(self, monkeypatch): doc = self._make_doc(monkeypatch) @@ -246,7 +269,7 @@ class TestTimeoutsAndRetries: def _make_doc(self, monkeypatch): monkeypatch.setenv("USER_JWT_TOKEN", "test-jwt-token") monkeypatch.setenv("FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT", CUSTOM_ENDPOINT) - return DatasetFileDocument("/bob@x.com/ds/v1/file.csv") + return DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") def test_presigned_url_request_passes_request_timeout(self, monkeypatch): doc = self._make_doc(monkeypatch) diff --git a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResourceSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResourceSpec.scala index c01489aa865..e9c92d4ac5e 100644 --- a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResourceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResourceSpec.scala @@ -666,7 +666,7 @@ class WorkflowExecutionsResourceSpec val content = """{ | "operators": [ - | {"operatorID": "scanA", "operatorProperties": {"fileName": "/owner@example.com/LockedDS/v1/data.csv"}}, + | {"operatorID": "scanA", "operatorProperties": {"fileName": "/datasets/owner@example.com/LockedDS/v1/data.csv"}}, | {"operatorID": "downstreamB", "operatorProperties": {}} | ], | "links": [ @@ -725,7 +725,7 @@ class WorkflowExecutionsResourceSpec val content = """{ | "operators": [ - | {"operatorID": "scan", "operatorProperties": {"fileName": "/test@example.com/MyDS/v1/data.csv"}} + | {"operatorID": "scan", "operatorProperties": {"fileName": "/datasets/test@example.com/MyDS/v1/data.csv"}} | ], | "links": [] |}""".stripMargin diff --git a/bin/single-node/examples/workflows/[Example] Data Exploration on Movies Dataset.json b/bin/single-node/examples/workflows/[Example] Data Exploration on Movies Dataset.json index 3e9f99c613f..cdc9a0e8708 100644 --- a/bin/single-node/examples/workflows/[Example] Data Exploration on Movies Dataset.json +++ b/bin/single-node/examples/workflows/[Example] Data Exploration on Movies Dataset.json @@ -8,7 +8,7 @@ "fileEncoding": "UTF_8", "customDelimiter": ",", "hasHeader": true, - "fileName": "/texera/popular-movies-of-imdb/v1/TMDb_updated.csv", + "fileName": "/datasets/texera/popular-movies-of-imdb/v1/TMDb_updated.csv", "offset": 0, "limit": 1000 }, diff --git a/bin/single-node/examples/workflows/[Example] Machine Learning on Iris Dataset.json b/bin/single-node/examples/workflows/[Example] Machine Learning on Iris Dataset.json index d36d35dae56..7e9a5d11fd4 100644 --- a/bin/single-node/examples/workflows/[Example] Machine Learning on Iris Dataset.json +++ b/bin/single-node/examples/workflows/[Example] Machine Learning on Iris Dataset.json @@ -8,7 +8,7 @@ "fileEncoding": "UTF_8", "customDelimiter": ",", "hasHeader": true, - "fileName": "/texera/iris-species/v1/Iris.csv" + "fileName": "/datasets/texera/iris-species/v1/Iris.csv" }, "inputPorts": [], "outputPorts": [ diff --git a/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala b/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala index a335ddeff6c..a79fc6cef20 100644 --- a/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala +++ b/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala @@ -36,6 +36,8 @@ object EnvironmentalVariable { * FileService related endpoint */ val ENV_FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT = "FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT" + val ENV_FILE_SERVICE_GET_MODEL_PRESIGNED_URL_ENDPOINT = + "FILE_SERVICE_GET_MODEL_PRESIGNED_URL_ENDPOINT" val ENV_FILE_SERVICE_UPLOAD_ONE_FILE_TO_DATASET_ENDPOINT = "FILE_SERVICE_UPLOAD_ONE_FILE_TO_DATASET_ENDPOINT" diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala index a26340e79cc..a020dfdbb48 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala @@ -20,7 +20,10 @@ package org.apache.texera.amber.core.storage import org.apache.texera.common.config.StorageConfig -import org.apache.texera.amber.core.storage.FileResolver.DATASET_FILE_URI_SCHEME +import org.apache.texera.amber.core.storage.FileResolver.{ + DATASET_FILE_URI_SCHEME, + MODEL_FILE_URI_SCHEME +} import org.apache.texera.amber.core.storage.VFSResourceType._ import org.apache.texera.amber.core.storage.VFSURIFactory.{VFS_FILE_URI_SCHEME, decodeURI} import org.apache.texera.amber.core.storage.model._ @@ -58,6 +61,7 @@ object DocumentFactory { def openReadonlyDocument(fileUri: URI): ReadonlyVirtualDocument[_] = { fileUri.getScheme match { case DATASET_FILE_URI_SCHEME => new DatasetFileDocument(fileUri) + case MODEL_FILE_URI_SCHEME => new ModelFileDocument(fileUri) case "file" => new ReadonlyLocalFileDocument(fileUri) case unsupportedScheme => throw new UnsupportedOperationException( @@ -164,6 +168,7 @@ object DocumentFactory { def openDocument(uri: URI): (VirtualDocument[_], Option[Schema]) = { uri.getScheme match { case DATASET_FILE_URI_SCHEME => (new DatasetFileDocument(uri), None) + case MODEL_FILE_URI_SCHEME => (new ModelFileDocument(uri), None) case VFS_FILE_URI_SCHEME => val (_, _, _, resourceType) = decodeURI(uri) val storageKey = sanitizeURIPath(uri) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala index c8a407df993..9265ec9f723 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala @@ -24,8 +24,15 @@ import org.apache.texera.dao.SqlServer import org.apache.texera.dao.SqlServer.withTransaction import org.apache.texera.dao.jooq.generated.tables.Dataset.DATASET import org.apache.texera.dao.jooq.generated.tables.DatasetVersion.DATASET_VERSION +import org.apache.texera.dao.jooq.generated.tables.Model.MODEL +import org.apache.texera.dao.jooq.generated.tables.ModelVersion.MODEL_VERSION import org.apache.texera.dao.jooq.generated.tables.User.USER -import org.apache.texera.dao.jooq.generated.tables.pojos.{Dataset, DatasetVersion} +import org.apache.texera.dao.jooq.generated.tables.pojos.{ + Dataset, + DatasetVersion, + Model, + ModelVersion +} import java.net.{URI, URLEncoder} import java.nio.charset.StandardCharsets @@ -39,6 +46,7 @@ import scala.util.{Success, Try} object FileResolver { val DATASET_FILE_URI_SCHEME = "dataset" + val MODEL_FILE_URI_SCHEME = "model" /** * Resolves a given fileName to either a file on the local file system or a dataset file. @@ -51,7 +59,7 @@ object FileResolver { if (isFileResolved(fileName)) { return new URI(fileName) } - val resolvers: Seq[String => URI] = Seq(localResolveFunc, datasetResolveFunc) + val resolvers: Seq[String => URI] = Seq(localResolveFunc, datasetResolveFunc, modelResolveFunc) // Try each resolver function in sequence resolvers @@ -76,53 +84,102 @@ object FileResolver { } /** - * Parses a dataset file path and extracts its components. - * Expected format: /ownerEmail/datasetName/versionName/fileRelativePath + * Parses a versioned-resource file path of the form + * //ownerEmail/resourceName/versionName/fileRelativePath and extracts its components. * + * @param resourceType the resource-type segment the path must start with * @param fileName The file path to parse - * @return Some((ownerEmail, datasetName, versionName, fileRelativePath)) if valid, None otherwise + * @return Some((ownerEmail, resourceName, versionName, fileRelativePath)) if valid, None otherwise */ - private def parseDatasetFilePath( + private def parsePrefixedPath( + resourceType: ResourceType.Value, fileName: String ): Option[(String, String, String, Array[String])] = { val filePath = Paths.get(fileName) val pathSegments = (0 until filePath.getNameCount).map(filePath.getName(_).toString).toArray - if (pathSegments.length < 4) { + if (pathSegments.length < 5 || pathSegments(0) != resourceType.toString) { return None } - val ownerEmail = pathSegments(0) - val datasetName = pathSegments(1) - val versionName = pathSegments(2) - val fileRelativePathSegments = pathSegments.drop(3) + val ownerEmail = pathSegments(1) + val resourceName = pathSegments(2) + val versionName = pathSegments(3) + val fileRelativePathSegments = pathSegments.drop(4) - Some((ownerEmail, datasetName, versionName, fileRelativePathSegments)) + Some((ownerEmail, resourceName, versionName, fileRelativePathSegments)) } + private def parseDatasetFilePath( + fileName: String + ): Option[(String, String, String, Array[String])] = + parsePrefixedPath(ResourceType.Datasets, fileName) + /** - * Attempts to resolve a given fileName to a URI. + * Resolves a versioned-resource logical path to its physical `scheme:///` URI. * - * The fileName format should be: /ownerEmail/datasetName/versionName/fileRelativePath - * e.g. /bob@texera.com/twitterDataset/v1/california/irvine/tw1.csv - * The output dataset URI format is: {DATASET_FILE_URI_SCHEME}:///{repositoryName}/{versionHash}/fileRelativePath - * e.g. {DATASET_FILE_URI_SCHEME}:///dataset-15/adeq233td/some/dir/file.txt - * - * @param fileName the name of the file to attempt resolving as a DatasetFileDocument - * @return Either[String, DatasetFileDocument] - Right(document) if creation succeeds - * @throws java.io.FileNotFoundException if the dataset file does not exist or cannot be created + * @throws java.io.FileNotFoundException if the path is not a valid `resourceType` path, the + * resource/version does not exist, or the URI is malformed */ - private def datasetResolveFunc(fileName: String): URI = { - val (ownerEmail, datasetName, versionName, fileRelativePathSegments) = - parseDatasetFilePath(fileName).getOrElse( - throw new FileNotFoundException(s"Dataset file $fileName not found.") + private def resolveVersionedFile( + scheme: String, + resourceType: ResourceType.Value, + fileName: String, + notFoundMessage: String + )(lookup: (String, String, String) => (String, String)): URI = { + val (ownerEmail, resourceName, versionName, fileRelativePathSegments) = + parsePrefixedPath(resourceType, fileName).getOrElse( + throw new FileNotFoundException(notFoundMessage) ) + val (repositoryName, versionHash) = lookup(ownerEmail, resourceName, versionName) + val fileRelativePath = Paths.get(fileRelativePathSegments.head, fileRelativePathSegments.tail: _*) - // fetch the dataset and version from DB to get dataset ID and version hash - val (dataset, datasetVersion) = + // Convert each segment of fileRelativePath to an encoded String + val encodedFileRelativePath = fileRelativePath + .iterator() + .asScala + .map { segment => + URLEncoder.encode(segment.toString, StandardCharsets.UTF_8) + } + .toArray + + // Prepend repositoryName and versionHash to the encoded path segments + val allPathSegments = Array(repositoryName, versionHash) ++ encodedFileRelativePath + + // Build the format /{repositoryName}/{versionHash}/{fileRelativePath}, both Linux and Windows use forward slash as the splitter + val uriSplitter = "/" + val encodedPath = uriSplitter + allPathSegments.mkString(uriSplitter) + + try { + new URI(scheme, "", encodedPath, null) + } catch { + case _: Exception => + throw new FileNotFoundException(notFoundMessage) + } + } + + /** + * Attempts to resolve a dataset file path to a URI. + * + * The fileName format should be: /datasets/ownerEmail/datasetName/versionName/fileRelativePath + * e.g. /datasets/bob@texera.com/twitterDataset/v1/california/irvine/tw1.csv + * The output dataset URI format is: {DATASET_FILE_URI_SCHEME}:///{repositoryName}/{versionHash}/fileRelativePath + * e.g. {DATASET_FILE_URI_SCHEME}:///dataset-15/adeq233td/some/dir/file.txt + * + * @param fileName the name of the file to attempt resolving as a DatasetFileDocument + * @throws java.io.FileNotFoundException if the dataset file does not exist or cannot be created + */ + private def datasetResolveFunc(fileName: String): URI = + resolveVersionedFile( + DATASET_FILE_URI_SCHEME, + ResourceType.Datasets, + fileName, + s"Dataset file $fileName not found." + ) { (ownerEmail, datasetName, versionName) => + // fetch the dataset and version from DB to get the repository name and version hash withTransaction( SqlServer .getInstance() @@ -138,6 +195,11 @@ object FileResolver { .and(DATASET.NAME.eq(datasetName)) .fetchOneInto(classOf[Dataset]) + // fail early if the dataset does not exist (before dereferencing it below) + if (dataset == null) { + throw new FileNotFoundException(s"Dataset file $fileName not found.") + } + // fetch the dataset version from DB val datasetVersion = ctx .selectFrom(DATASET_VERSION) @@ -145,38 +207,65 @@ object FileResolver { .and(DATASET_VERSION.NAME.eq(versionName)) .fetchOneInto(classOf[DatasetVersion]) - if (dataset == null || datasetVersion == null) { + if (datasetVersion == null) { throw new FileNotFoundException(s"Dataset file $fileName not found.") } - (dataset, datasetVersion) + (dataset.getRepositoryName, datasetVersion.getVersionHash) } + } - // Convert each segment of fileRelativePath to an encoded String - val encodedFileRelativePath = fileRelativePath - .iterator() - .asScala - .map { segment => - URLEncoder.encode(segment.toString, StandardCharsets.UTF_8) - } - .toArray + /** + * Attempts to resolve a model file path to a URI. + * + * The fileName format should be: /models/ownerEmail/modelName/versionName/fileRelativePath + * e.g. /models/bob@texera.com/resnet/v1/weights/model.pt + * The output model URI format is: {MODEL_FILE_URI_SCHEME}:///{repositoryName}/{versionHash}/fileRelativePath + * e.g. {MODEL_FILE_URI_SCHEME}:///model-15/adeq233td/weights/model.pt + * + * @param fileName the name of the file to attempt resolving as a ModelFileDocument + * @throws java.io.FileNotFoundException if the model file does not exist or cannot be created + */ + private def modelResolveFunc(fileName: String): URI = + resolveVersionedFile( + MODEL_FILE_URI_SCHEME, + ResourceType.Models, + fileName, + s"Model file $fileName not found." + ) { (ownerEmail, modelName, versionName) => + // fetch the model and version from DB to get the repository name and version hash + withTransaction( + SqlServer + .getInstance() + .createDSLContext() + ) { ctx => + // fetch the model from DB + val model = ctx + .select(MODEL.fields: _*) + .from(MODEL) + .leftJoin(USER) + .on(USER.UID.eq(MODEL.OWNER_UID)) + .where(USER.EMAIL.eq(ownerEmail)) + .and(MODEL.NAME.eq(modelName)) + .fetchOneInto(classOf[Model]) - // Prepend dataset name and versionHash to the encoded path segments - val allPathSegments = Array( - dataset.getRepositoryName, - datasetVersion.getVersionHash - ) ++ encodedFileRelativePath + // fail early if the model does not exist (before dereferencing it below) + if (model == null) { + throw new FileNotFoundException(s"Model file $fileName not found.") + } - // Build the format /{repositoryName}/{versionHash}/{fileRelativePath}, both Linux and Windows use forward slash as the splitter - val uriSplitter = "/" - val encodedPath = uriSplitter + allPathSegments.mkString(uriSplitter) + // fetch the model version from DB + val modelVersion = ctx + .selectFrom(MODEL_VERSION) + .where(MODEL_VERSION.MID.eq(model.getMid)) + .and(MODEL_VERSION.NAME.eq(versionName)) + .fetchOneInto(classOf[ModelVersion]) - try { - new URI(DATASET_FILE_URI_SCHEME, "", encodedPath, null) - } catch { - case e: Exception => - throw new FileNotFoundException(s"Dataset file $fileName not found.") + if (modelVersion == null) { + throw new FileNotFoundException(s"Model file $fileName not found.") + } + (model.getRepositoryName, modelVersion.getVersionHash) + } } - } /** * Checks if a given file path has a valid scheme. @@ -194,11 +283,8 @@ object FileResolver { } /** - * Parses a dataset file path to extract owner email and dataset name. - * Expected format: /ownerEmail/datasetName/versionName/fileRelativePath - * - * @param path The file path from operator properties - * @return Some((ownerEmail, datasetName)) if path is valid, None otherwise + * Extracts the owner email and dataset name from a dataset logical path, + * or None if it is not a well-formed dataset path. */ def parseDatasetOwnerAndName(path: String): Option[(String, String)] = { if (path == null) { diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/ResourceType.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/ResourceType.scala new file mode 100644 index 00000000000..03c1ce4c01f --- /dev/null +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/ResourceType.scala @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.core.storage + +/** + * The leading segment of a logical file path, identifying which resource kind (and thus which + * backing table) a path belongs to. + * + * Path shape: //ownerEmail/resourceName/versionName/fileRelativePath + */ +object ResourceType extends Enumeration { + val Datasets: Value = Value("datasets") + val Models: Value = Value("models") +} diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/DatasetFileDocument.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/DatasetFileDocument.scala index 6d8f917c7f4..044db2013b9 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/DatasetFileDocument.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/DatasetFileDocument.scala @@ -19,28 +19,14 @@ package org.apache.texera.amber.core.storage.model -import com.typesafe.scalalogging.LazyLogging import org.apache.texera.common.config.EnvironmentalVariable -import org.apache.texera.amber.core.storage.model.DatasetFileDocument.{ - fileServiceGetPresignURLEndpoint, - userJwtToken -} -import org.apache.texera.amber.core.storage.util.LakeFSStorageClient +import org.apache.texera.amber.core.storage.model.DatasetFileDocument.fileServiceGetPresignURLEndpoint import org.apache.texera.amber.core.storage.util.dataset.GitVersionControlLocalFileStorage -import java.io.{File, FileOutputStream, InputStream} -import java.net._ -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Path, Paths} -import scala.jdk.CollectionConverters.IteratorHasAsScala +import java.net.URI +import java.nio.file.Path object DatasetFileDocument { - // Since requests need to be sent to the FileService in order to read the file, we store USER_JWT_TOKEN in the environment vars - // This variable should be NON-EMPTY in the dynamic-computing-unit architecture, i.e. each user-created computing unit should store user's jwt token. - // In the local development or other architectures, this token can be empty. - lazy val userJwtToken: String = - sys.env.getOrElse(EnvironmentalVariable.ENV_USER_JWT_TOKEN, "").trim - // The endpoint of getting presigned url from the file service, also stored in the environment vars. lazy val fileServiceGetPresignURLEndpoint: String = sys.env @@ -52,122 +38,12 @@ object DatasetFileDocument { } private[storage] class DatasetFileDocument(uri: URI) - extends VirtualDocument[Nothing] - with OnDataset - with LazyLogging { - // Utility function to parse and decode URI segments into individual components - private def parseUri(uri: URI): (String, String, Path) = { - val segments = Paths.get(uri.getPath).iterator().asScala.map(_.toString).toArray - if (segments.length < 3) - throw new IllegalArgumentException("URI format is incorrect") - - // parse uri to dataset components - val repositoryName = segments(0) - val datasetVersionHash = URLDecoder.decode(segments(1), StandardCharsets.UTF_8) - val decodedRelativeSegments = - segments.drop(2).map(part => URLDecoder.decode(part, StandardCharsets.UTF_8)) - val fileRelativePath = Paths.get(decodedRelativeSegments.head, decodedRelativeSegments.tail: _*) - - (repositoryName, datasetVersionHash, fileRelativePath) - } - - // Extract components from URI using the utility function - private val (repositoryName, datasetVersionHash, fileRelativePath) = parseUri(uri) - - private var tempFile: Option[File] = None - - override def getURI: URI = uri - - override def asInputStream(): InputStream = { - - def fallbackToLakeFS(exception: Throwable): InputStream = { - logger.warn(s"${exception.getMessage}. Falling back to LakeFS direct file fetch.", exception) - val file = LakeFSStorageClient.getFileFromRepo( - getRepositoryName(), - getVersionHash(), - getFileRelativePath() - ) - Files.newInputStream(file.toPath) - } - - if (userJwtToken.isEmpty) { - try { - val presignUrl = LakeFSStorageClient.getFilePresignedUrl( - getRepositoryName(), - getVersionHash(), - getFileRelativePath() - ) - new URL(presignUrl).openStream() - } catch { - case e: Exception => - fallbackToLakeFS(e) - } - } else { - val presignRequestUrl = - s"$fileServiceGetPresignURLEndpoint?repositoryName=${getRepositoryName()}&commitHash=${getVersionHash()}&filePath=${URLEncoder - .encode(getFileRelativePath(), StandardCharsets.UTF_8.name())}" - - val connection = new URL(presignRequestUrl).openConnection().asInstanceOf[HttpURLConnection] - connection.setRequestMethod("GET") - connection.setRequestProperty("Authorization", s"Bearer $userJwtToken") - - try { - if (connection.getResponseCode != HttpURLConnection.HTTP_OK) { - throw new RuntimeException( - s"Failed to retrieve presigned URL: HTTP ${connection.getResponseCode}" - ) - } - - // Read response body as a string - val responseBody = - new String(connection.getInputStream.readAllBytes(), StandardCharsets.UTF_8) - - // Extract presigned URL from JSON response - val presignedUrl = responseBody - .split("\"presignedUrl\"\\s*:\\s*\"")(1) - .split("\"")(0) - - new URL(presignedUrl).openStream() - } catch { - case e: Exception => - fallbackToLakeFS(e) - } finally { - connection.disconnect() - } - } - } - - override def asFile(): File = { - tempFile match { - case Some(file) => file - case None => - val tempFilePath = Files.createTempFile("versionedFile", ".tmp") - val tempFileStream = new FileOutputStream(tempFilePath.toFile) - val inputStream = asInputStream() - - val buffer = new Array[Byte](1024) - - // Create an iterator to repeatedly call inputStream.read, and direct buffered data to file - Iterator - .continually(inputStream.read(buffer)) - .takeWhile(_ != -1) - .foreach(tempFileStream.write(buffer, 0, _)) - - inputStream.close() - tempFileStream.close() - - val file = tempFilePath.toFile - tempFile = Some(file) - file - } - } + extends LakeFSFileDocument(uri, fileServiceGetPresignURLEndpoint) { override def clear(): Unit = { - // first remove the temporary file - tempFile match { - case Some(file) => Files.delete(file.toPath) - case None => // Do nothing - } + // first remove the temporary file (handled by the shared base) + super.clear() + lazy val datasetsRootPath = Path .of(sys.env.getOrElse("TEXERA_HOME", ".")) @@ -179,16 +55,10 @@ private[storage] class DatasetFileDocument(uri: URI) datasetsRootPath.resolve(did.toString) } - // then remove the dataset file + // then remove the dataset file from the local git-backed storage GitVersionControlLocalFileStorage.removeFileFromRepo( getDatasetPath(0), getDatasetPath(0).resolve(fileRelativePath) ) } - - override def getRepositoryName(): String = repositoryName - - override def getVersionHash(): String = datasetVersionHash - - override def getFileRelativePath(): String = fileRelativePath.toString } diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/LakeFSFileDocument.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/LakeFSFileDocument.scala new file mode 100644 index 00000000000..044437889e9 --- /dev/null +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/LakeFSFileDocument.scala @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.core.storage.model + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.common.config.EnvironmentalVariable +import org.apache.texera.amber.core.storage.model.LakeFSFileDocument.userJwtToken +import org.apache.texera.amber.core.storage.util.LakeFSStorageClient + +import java.io.{File, FileOutputStream, InputStream} +import java.net._ +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} +import scala.jdk.CollectionConverters.IteratorHasAsScala + +object LakeFSFileDocument { + // Since requests need to be sent to the FileService in order to read the file, we store USER_JWT_TOKEN in the environment vars + // This variable should be NON-EMPTY in the dynamic-computing-unit architecture, i.e. each user-created computing unit should store user's jwt token. + // In the local development or other architectures, this token can be empty. + lazy val userJwtToken: String = + sys.env.getOrElse(EnvironmentalVariable.ENV_USER_JWT_TOKEN, "").trim +} + +/** + * A read-only document over a single file stored in a LakeFS repository, addressed by the URI + * {scheme}:///{repositoryName}/{versionHash}/{fileRelativePath}. This is the shared behavior + * for every versioned-file resource (datasets, models, …): the file bytes are fetched via a + * presigned URL, falling back to a direct LakeFS fetch. + * + * @param uri the resolved {scheme}:///{repositoryName}/{versionHash}/{file} URI + * @param presignEndpoint the file-service presign-download endpoint for this resource kind + */ +private[storage] abstract class LakeFSFileDocument(uri: URI, presignEndpoint: String) + extends VirtualDocument[Nothing] + with OnVersionedFileResource + with LazyLogging { + // Utility function to parse and decode URI segments into individual components + private def parseUri(uri: URI): (String, String, Path) = { + val segments = Paths.get(uri.getPath).iterator().asScala.map(_.toString).toArray + if (segments.length < 3) + throw new IllegalArgumentException("URI format is incorrect") + + // parse uri to (repositoryName, versionHash, fileRelativePath) + val repositoryName = segments(0) + val versionHash = URLDecoder.decode(segments(1), StandardCharsets.UTF_8) + val decodedRelativeSegments = + segments.drop(2).map(part => URLDecoder.decode(part, StandardCharsets.UTF_8)) + val fileRelativePath = Paths.get(decodedRelativeSegments.head, decodedRelativeSegments.tail: _*) + + (repositoryName, versionHash, fileRelativePath) + } + + // Extract components from URI using the utility function + protected val (repositoryName, versionHash, fileRelativePath) = parseUri(uri) + + protected var tempFile: Option[File] = None + + override def getURI: URI = uri + + override def asInputStream(): InputStream = { + + def fallbackToLakeFS(exception: Throwable): InputStream = { + logger.warn(s"${exception.getMessage}. Falling back to LakeFS direct file fetch.", exception) + val file = LakeFSStorageClient.getFileFromRepo( + getRepositoryName(), + getVersionHash(), + getFileRelativePath() + ) + Files.newInputStream(file.toPath) + } + + if (userJwtToken.isEmpty) { + try { + val presignUrl = LakeFSStorageClient.getFilePresignedUrl( + getRepositoryName(), + getVersionHash(), + getFileRelativePath() + ) + new URL(presignUrl).openStream() + } catch { + case e: Exception => + fallbackToLakeFS(e) + } + } else { + val presignRequestUrl = + s"$presignEndpoint?repositoryName=${getRepositoryName()}&commitHash=${getVersionHash()}&filePath=${URLEncoder + .encode(getFileRelativePath(), StandardCharsets.UTF_8.name())}" + + val connection = new URL(presignRequestUrl).openConnection().asInstanceOf[HttpURLConnection] + connection.setRequestMethod("GET") + connection.setRequestProperty("Authorization", s"Bearer $userJwtToken") + + try { + if (connection.getResponseCode != HttpURLConnection.HTTP_OK) { + throw new RuntimeException( + s"Failed to retrieve presigned URL: HTTP ${connection.getResponseCode}" + ) + } + + // Read response body as a string + val responseBody = + new String(connection.getInputStream.readAllBytes(), StandardCharsets.UTF_8) + + // Extract presigned URL from JSON response + val presignedUrl = responseBody + .split("\"presignedUrl\"\\s*:\\s*\"")(1) + .split("\"")(0) + + new URL(presignedUrl).openStream() + } catch { + case e: Exception => + fallbackToLakeFS(e) + } finally { + connection.disconnect() + } + } + } + + override def asFile(): File = { + tempFile match { + case Some(file) => file + case None => + val tempFilePath = Files.createTempFile("versionedFile", ".tmp") + val tempFileStream = new FileOutputStream(tempFilePath.toFile) + val inputStream = asInputStream() + + val buffer = new Array[Byte](1024) + + // Create an iterator to repeatedly call inputStream.read, and direct buffered data to file + Iterator + .continually(inputStream.read(buffer)) + .takeWhile(_ != -1) + .foreach(tempFileStream.write(buffer, 0, _)) + + inputStream.close() + tempFileStream.close() + + val file = tempFilePath.toFile + tempFile = Some(file) + file + } + } + + override def clear(): Unit = { + // remove the temporary file, if one was materialized + tempFile match { + case Some(file) => Files.delete(file.toPath) + case None => // Do nothing + } + tempFile = None + } + + override def getRepositoryName(): String = repositoryName + + override def getVersionHash(): String = versionHash + + override def getFileRelativePath(): String = fileRelativePath.toString +} diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/ModelFileDocument.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/ModelFileDocument.scala new file mode 100644 index 00000000000..e67ec92992b --- /dev/null +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/ModelFileDocument.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.core.storage.model + +import org.apache.texera.common.config.EnvironmentalVariable +import org.apache.texera.amber.core.storage.model.ModelFileDocument.fileServiceGetModelPresignURLEndpoint + +import java.net.URI + +object ModelFileDocument { + // The endpoint of getting a presigned url for a model file from the file service. + lazy val fileServiceGetModelPresignURLEndpoint: String = + sys.env + .getOrElse( + EnvironmentalVariable.ENV_FILE_SERVICE_GET_MODEL_PRESIGNED_URL_ENDPOINT, + "http://localhost:9092/api/model/presign-download" + ) + .trim +} + +/** + * A read-only document over a single file in a model's LakeFS repository (`model-{mid}`), + * addressed by a `model:///{repositoryName}/{versionHash}/{fileRelativePath}` URI. + */ +private[storage] class ModelFileDocument(uri: URI) + extends LakeFSFileDocument(uri, fileServiceGetModelPresignURLEndpoint) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/OnDataset.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/OnVersionedFileResource.scala similarity index 90% rename from common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/OnDataset.scala rename to common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/OnVersionedFileResource.scala index 6c19ee1002b..cbb76ee4f59 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/OnDataset.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/OnVersionedFileResource.scala @@ -19,7 +19,10 @@ package org.apache.texera.amber.core.storage.model -trait OnDataset { +/** + * A document backed by a versioned file in a LakeFS repository. + */ +trait OnVersionedFileResource { def getRepositoryName(): String def getVersionHash(): String diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/storage/DocumentFactorySpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/storage/DocumentFactorySpec.scala index ad8b5580c53..2578d497380 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/storage/DocumentFactorySpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/storage/DocumentFactorySpec.scala @@ -21,6 +21,8 @@ package org.apache.texera.amber.core.storage import org.apache.texera.amber.core.storage.model.{ DatasetFileDocument, + ModelFileDocument, + OnVersionedFileResource, ReadonlyLocalFileDocument, VirtualDocument } @@ -38,7 +40,7 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import java.net.URI -import java.nio.file.Files +import java.nio.file.{Files, Paths} import java.util.UUID /** @@ -135,6 +137,18 @@ class DocumentFactorySpec extends AnyFlatSpec with Matchers with BeforeAndAfterA doc.getURI shouldBe datasetUri } + it should "return a ModelFileDocument for the model scheme and parse its URI components" in { + val modelUri = new URI(s"model:///model-1/$versionHash/weights/model.pt") + val doc = DocumentFactory.openReadonlyDocument(modelUri) + doc shouldBe a[ModelFileDocument] + doc.getURI shouldBe modelUri + + val resource = doc.asInstanceOf[OnVersionedFileResource] + resource.getRepositoryName() shouldBe "model-1" + resource.getVersionHash() shouldBe versionHash + resource.getFileRelativePath() shouldBe Paths.get("weights", "model.pt").toString + } + it should "return a ReadonlyLocalFileDocument for the file scheme" in { val tempFile = Files.createTempFile("doc-factory-spec", ".txt") try { @@ -165,6 +179,13 @@ class DocumentFactorySpec extends AnyFlatSpec with Matchers with BeforeAndAfterA schemaOpt shouldBe None } + it should "return a ModelFileDocument and no schema for the model scheme" in { + val modelUri = new URI(s"model:///model-1/$versionHash/weights/model.pt") + val (doc, schemaOpt) = DocumentFactory.openDocument(modelUri) + doc shouldBe a[ModelFileDocument] + schemaOpt shouldBe None + } + it should "reject an unsupported scheme" in { val thrown = intercept[UnsupportedOperationException] { DocumentFactory.openDocument(new URI("ftp://host/path")) diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/storage/model/DatasetFileDocumentSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/storage/model/DatasetFileDocumentSpec.scala index c16d417a73c..72285984971 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/storage/model/DatasetFileDocumentSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/storage/model/DatasetFileDocumentSpec.scala @@ -142,11 +142,11 @@ class DatasetFileDocumentSpec extends AnyFlatSpec with Matchers { } } - // The companion object resolves the file-service endpoint and the user JWT - // token from environment variables, falling back to a trimmed default. These - // lazy vals are read whenever asInputStream needs to fetch a file; assert their - // fallback behavior without requiring a live FileService or LakeFS. The checks - // are guarded so they hold regardless of whether the env overrides are present. + // The companion object resolves the file-service endpoint from environment + // variables, falling back to a trimmed default. This lazy val is read whenever + // asInputStream needs to fetch a file; assert its fallback behavior without + // requiring a live FileService or LakeFS. The check is guarded so it holds + // regardless of whether the env override is present. "DatasetFileDocument companion" should "expose the default presigned-URL endpoint when the env override is absent" in { val expected = @@ -159,9 +159,11 @@ class DatasetFileDocumentSpec extends AnyFlatSpec with Matchers { DatasetFileDocument.fileServiceGetPresignURLEndpoint shouldBe expected } - it should "expose a trimmed user JWT token defaulting to empty" in { + // The user JWT token is shared by every LakeFS-backed document, so it now lives on + // the LakeFSFileDocument base object rather than on DatasetFileDocument. + "LakeFSFileDocument companion" should "expose a trimmed user JWT token defaulting to empty" in { val expected = sys.env.getOrElse(EnvironmentalVariable.ENV_USER_JWT_TOKEN, "").trim - DatasetFileDocument.userJwtToken shouldBe expected + LakeFSFileDocument.userJwtToken shouldBe expected } } diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/storage/FileResolverSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/storage/FileResolverSpec.scala index 62daa811d35..a5f441265a8 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/storage/FileResolverSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/storage/FileResolverSpec.scala @@ -23,8 +23,20 @@ import org.apache.texera.amber.core.storage.FileResolver import org.apache.commons.vfs2.FileNotFoundException import org.apache.texera.dao.MockTexeraDB import org.apache.texera.dao.jooq.generated.enums.UserRoleEnum -import org.apache.texera.dao.jooq.generated.tables.daos.{DatasetDao, DatasetVersionDao, UserDao} -import org.apache.texera.dao.jooq.generated.tables.pojos.{Dataset, DatasetVersion, User} +import org.apache.texera.dao.jooq.generated.tables.daos.{ + DatasetDao, + DatasetVersionDao, + ModelDao, + ModelVersionDao, + UserDao +} +import org.apache.texera.dao.jooq.generated.tables.pojos.{ + Dataset, + DatasetVersion, + Model, + ModelVersion, + User +} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} @@ -77,11 +89,59 @@ class FileResolverSpec datasetVersion } + private val testModel: Model = { + val model = new Model + model.setMid(Integer.valueOf(1)) + model.setName("test_model") + model.setRepositoryName("model-1") + model.setDescription("model for test") + model.setIsPublic(true) + model.setIsDownloadable(true) + model.setOwnerUid(Integer.valueOf(1)) + model + } + + private val testModelVersion1: ModelVersion = { + val modelVersion = new ModelVersion + modelVersion.setMid(Integer.valueOf(1)) + modelVersion.setName("v1") + modelVersion.setMvid(Integer.valueOf(1)) + modelVersion.setCreatorUid(Integer.valueOf(1)) + modelVersion.setVersionHash("a1b2c3d4e5f60718293a4b5c6d7e8f9001122334") + modelVersion + } + + private val testModelVersion2: ModelVersion = { + val modelVersion = new ModelVersion + modelVersion.setMid(Integer.valueOf(1)) + modelVersion.setName("v2") + modelVersion.setMvid(Integer.valueOf(2)) + modelVersion.setCreatorUid(Integer.valueOf(1)) + modelVersion.setVersionHash("998877665544332211ffeeddccbbaa0099887766") + modelVersion + } + private val localCsvFilePath = "common/workflow-core/src/test/resources/country_sales_small.csv" - private val datasetACsvFilePath = "/test_user@test.com/test_dataset/v2/directory/a.csv" + private val datasetACsvFilePath = "/datasets/test_user@test.com/test_dataset/v2/directory/a.csv" + + private val dataset1TxtFilePath = "/datasets/test_user@test.com/test_dataset/v1/1.txt" + + private val modelWeightsFilePath = "/models/test_user@test.com/test_model/v2/weights/model.pt" + + private val modelReadmeFilePath = "/models/test_user@test.com/test_model/v1/README.md" - private val dataset1TxtFilePath = "/test_user@test.com/test_dataset/v1/1.txt" + // Unprefixed form (no resource-type segment); no longer resolvable as a dataset. + private val unprefixedDataset1TxtFilePath = "/test_user@test.com/test_dataset/v1/1.txt" + + // An unknown resource-type prefix belongs to neither datasets nor models. + private val unknownPrefixFilePath = "/notaprefix/test_user@test.com/test_dataset/v1/1.txt" + + // A model name presented under the datasets prefix must not resolve as a dataset. + private val modelNameUnderDatasetPrefix = "/datasets/test_user@test.com/test_model/v1/README.md" + + // A dataset name presented under the models prefix must not resolve as a model. + private val datasetNameUnderModelPrefix = "/models/test_user@test.com/test_dataset/v1/1.txt" override protected def beforeAll(): Unit = { initializeDBAndReplaceDSLContext() @@ -98,6 +158,15 @@ class FileResolverSpec val datasetVersionDao = new DatasetVersionDao(getDSLContext.configuration()) datasetVersionDao.insert(testDatasetVersion1) datasetVersionDao.insert(testDatasetVersion2) + + // add test model + val modelDao = new ModelDao(getDSLContext.configuration()) + modelDao.insert(testModel) + + // add test model versions + val modelVersionDao = new ModelVersionDao(getDSLContext.configuration()) + modelVersionDao.insert(testModelVersion1) + modelVersionDao.insert(testModelVersion2) } "FileResolver" should "resolve local file correctly" in { @@ -118,6 +187,48 @@ class FileResolverSpec ) } + "FileResolver" should "resolve model file correctly" in { + val modelWeightsUri = FileResolver.resolve(modelWeightsFilePath) + val modelReadmeUri = FileResolver.resolve(modelReadmeFilePath) + + assert( + modelWeightsUri.toString == f"${FileResolver.MODEL_FILE_URI_SCHEME}:///${testModel.getRepositoryName}/${testModelVersion2.getVersionHash}/weights/model.pt" + ) + assert( + modelReadmeUri.toString == f"${FileResolver.MODEL_FILE_URI_SCHEME}:///${testModel.getRepositoryName}/${testModelVersion1.getVersionHash}/README.md" + ) + } + + "FileResolver" should "not resolve an unprefixed dataset path (the datasets prefix is required)" in { + // Without the datasets prefix the path is not a dataset path; it falls through to the + // local-file resolver and fails. + assertThrows[FileNotFoundException] { + FileResolver.resolve(unprefixedDataset1TxtFilePath) + } + } + + "FileResolver" should "not resolve a path whose prefix is neither datasets nor models" in { + assertThrows[FileNotFoundException] { + FileResolver.resolve(unknownPrefixFilePath) + } + } + + "FileResolver" should "keep datasets and models isolated by prefix" in { + // a real dataset name under the models prefix is not a model, and vice-versa + assertThrows[FileNotFoundException] { + FileResolver.resolve(datasetNameUnderModelPrefix) + } + assertThrows[FileNotFoundException] { + FileResolver.resolve(modelNameUnderDatasetPrefix) + } + } + + "FileResolver" should "throw not found exception when a prefixed path has too few segments" in { + assertThrows[FileNotFoundException] { + FileResolver.resolve("/datasets/test_user@test.com/test_dataset") + } + } + "FileResolver" should "throw not found exception" in { assertThrows[FileNotFoundException] { FileResolver.resolve("some/random/path") @@ -143,18 +254,22 @@ class FileResolverSpec "parseDatasetOwnerAndName" should "extract owner email and dataset name from a valid path" in { assert( - FileResolver.parseDatasetOwnerAndName("/test_user@test.com/test_dataset/v1/1.txt") + FileResolver.parseDatasetOwnerAndName("/datasets/test_user@test.com/test_dataset/v1/1.txt") == Some(("test_user@test.com", "test_dataset")) ) // extra segments beyond the file-relative path are ignored assert( - FileResolver.parseDatasetOwnerAndName("/owner@x.com/ds/v2/directory/nested/a.csv") + FileResolver.parseDatasetOwnerAndName("/datasets/owner@x.com/ds/v2/directory/nested/a.csv") == Some(("owner@x.com", "ds")) ) } - it should "return None when the path has fewer than four segments" in { - assert(FileResolver.parseDatasetOwnerAndName("/owner@x.com/ds/v1").isEmpty) + it should "return None for an unprefixed path (the datasets prefix is required)" in { + assert(FileResolver.parseDatasetOwnerAndName(unprefixedDataset1TxtFilePath).isEmpty) + } + + it should "return None when the prefixed path has too few segments" in { + assert(FileResolver.parseDatasetOwnerAndName("/datasets/owner@x.com/ds").isEmpty) assert(FileResolver.parseDatasetOwnerAndName("owner/dataset").isEmpty) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExec.scala index c58da8ca847..2e4bad27324 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExec.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.source.dataset import org.apache.texera.amber.core.executor.SourceOperatorExecutor +import org.apache.texera.amber.core.storage.ResourceType import org.apache.texera.amber.core.storage.util.LakeFSStorageClient import org.apache.texera.amber.core.tuple.TupleLike import org.apache.texera.amber.util.JSONUtils.objectMapper @@ -28,13 +29,34 @@ import org.apache.texera.dao.jooq.generated.tables.Dataset.DATASET import org.apache.texera.dao.jooq.generated.tables.DatasetVersion.DATASET_VERSION import org.apache.texera.dao.jooq.generated.tables.User.USER +object FileListerSourceOpExec { + + /** + * Parses a dataset version path (/datasets/ownerEmail/datasetName/versionName) into its + * (ownerEmail, datasetName, versionName) components. + * + * @throws IllegalArgumentException if the path is not a well-formed dataset version path + */ + private[dataset] def parseDatasetVersionPath( + datasetVersionPath: String + ): (String, String, String) = { + val segments = datasetVersionPath.split("/").filter(_.nonEmpty) + require( + segments.length >= 4 && segments.head == ResourceType.Datasets.toString, + s"Invalid dataset version path '$datasetVersionPath'; " + + "expected /datasets/ownerEmail/datasetName/versionName" + ) + (segments(1), segments(2), segments(3)) + } +} + class FileListerSourceOpExec private[dataset] (descString: String) extends SourceOperatorExecutor { private val desc: FileListerSourceOpDesc = objectMapper.readValue(descString, classOf[FileListerSourceOpDesc]) override def produceTuple(): Iterator[TupleLike] = { - val Seq(_, ownerEmail, datasetName, versionName, _*) = - desc.datasetVersionPath.split("/").toSeq + val (ownerEmail, datasetName, versionName) = + FileListerSourceOpExec.parseDatasetVersionPath(desc.datasetVersionPath) val (repositoryName, versionHash) = SqlServer diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExecSpec.scala new file mode 100644 index 00000000000..b5b5611c991 --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExecSpec.scala @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.source.dataset + +import org.scalatest.flatspec.AnyFlatSpec + +class FileListerSourceOpExecSpec extends AnyFlatSpec { + + "parseDatasetVersionPath" should "extract components from a datasets-prefixed path" in { + val (owner, name, version) = + FileListerSourceOpExec.parseDatasetVersionPath("/datasets/bob@texera.com/twitterDataset/v1") + assert(owner == "bob@texera.com") + assert(name == "twitterDataset") + assert(version == "v1") + } + + it should "work when the owner segment is a username without an '@'" in { + val (owner, name, version) = + FileListerSourceOpExec.parseDatasetVersionPath("/datasets/texera/test-ds/v1") + assert(owner == "texera") + assert(name == "test-ds") + assert(version == "v1") + } + + it should "ignore trailing slashes and extra segments" in { + val (owner, name, version) = + FileListerSourceOpExec.parseDatasetVersionPath("/datasets/alice/ds/v2/extra/") + assert(owner == "alice") + assert(name == "ds") + assert(version == "v2") + } + + it should "reject an unprefixed path (the datasets prefix is required)" in { + assertThrows[IllegalArgumentException] { + FileListerSourceOpExec.parseDatasetVersionPath("/alice/ds/v1") + } + } + + it should "reject a path whose prefix is not the datasets prefix" in { + assertThrows[IllegalArgumentException] { + FileListerSourceOpExec.parseDatasetVersionPath("/models/alice/ds/v1") + } + } + + it should "reject a path with too few segments" in { + assertThrows[IllegalArgumentException] { + FileListerSourceOpExec.parseDatasetVersionPath("/datasets/alice/ds") + } + } +} diff --git a/file-service/src/main/scala/org/apache/texera/service/FileService.scala b/file-service/src/main/scala/org/apache/texera/service/FileService.scala index d8e3ff122d5..4546a9c0ec4 100644 --- a/file-service/src/main/scala/org/apache/texera/service/FileService.scala +++ b/file-service/src/main/scala/org/apache/texera/service/FileService.scala @@ -34,7 +34,9 @@ import org.apache.texera.service.`type`.serde.DatasetFileNodeSerializer import org.apache.texera.service.resource.{ DatasetAccessResource, DatasetResource, - HealthCheckResource + HealthCheckResource, + ModelAccessResource, + ModelResource } import org.apache.texera.service.util.S3StorageClient import org.apache.texera.service.util.LargeBinaryManager @@ -89,6 +91,8 @@ class FileService extends Application[FileServiceConfiguration] with LazyLogging environment.jersey.register(classOf[DatasetResource]) environment.jersey.register(classOf[DatasetAccessResource]) + environment.jersey.register(classOf[ModelResource]) + environment.jersey.register(classOf[ModelAccessResource]) RoleAnnotationEnforcer.enforce(environment.jersey.getResourceConfig, "FileService") diff --git a/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala b/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala index c376e4ce041..e973dc09fca 100644 --- a/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala +++ b/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala @@ -25,9 +25,9 @@ import jakarta.annotation.security.{PermitAll, RolesAllowed} import jakarta.ws.rs._ import jakarta.ws.rs.core._ import org.apache.texera.common.config.StorageConfig -import org.apache.texera.amber.core.storage.model.OnDataset +import org.apache.texera.amber.core.storage.model.OnVersionedFileResource import org.apache.texera.amber.core.storage.util.LakeFSStorageClient -import org.apache.texera.amber.core.storage.{DocumentFactory, FileResolver} +import org.apache.texera.amber.core.storage.{DocumentFactory, FileResolver, ResourceType} import org.apache.texera.auth.SessionUser import org.apache.texera.dao.SiteSettings import org.apache.texera.dao.SqlServer @@ -50,6 +50,7 @@ import org.apache.texera.dao.jooq.generated.tables.pojos.{ import org.apache.texera.service.`type`.DatasetFileNode import org.apache.texera.service.resource.DatasetAccessResource._ import org.apache.texera.service.resource.DatasetResource.{context, _} +import org.apache.texera.service.util.ResourceUploadUtils.{put, validateAndNormalizeFilePathOrThrow} import org.apache.texera.service.util.S3StorageClient import org.apache.texera.service.util.S3StorageClient.{ MAXIMUM_NUM_OF_MULTIPART_S3_PARTS, @@ -61,7 +62,7 @@ import org.jooq.impl.DSL.{inline => inl} import org.jooq.{DSLContext, EnumType, Record2, Result} import java.io.{InputStream, OutputStream} -import java.net.{HttpURLConnection, URI, URL, URLDecoder} +import java.net.{URI, URLDecoder} import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} import java.util @@ -104,27 +105,6 @@ object DatasetResource { dataset } - /** - * Helper function to PUT exactly len bytes from buf to presigned URL, return the ETag - */ - private def put(buf: Array[Byte], len: Int, url: String, partNum: Int): String = { - val conn = new URL(url).openConnection().asInstanceOf[HttpURLConnection] - conn.setDoOutput(true) - conn.setRequestMethod("PUT") - conn.setFixedLengthStreamingMode(len) - val out = conn.getOutputStream - out.write(buf, 0, len) - out.close() - - val code = conn.getResponseCode - if (code != HttpURLConnection.HTTP_OK && code != HttpURLConnection.HTTP_CREATED) - throw new RuntimeException(s"Part $partNum upload failed (HTTP $code)") - - val etag = conn.getHeaderField("ETag").replace("\"", "") - conn.disconnect() - etag - } - /** * Helper function to get the dataset version from DB using dvid */ @@ -156,25 +136,6 @@ object DatasetResource { .toScala } - /** - * Validates a file path using Apache Commons IO. - */ - def validateAndNormalizeFilePathOrThrow(path: String): String = { - if (path == null || path.trim.isEmpty) { - throw new BadRequestException("Path cannot be empty") - } - - val normalized = FilenameUtils.normalize(path, true) - if (normalized == null) { - throw new BadRequestException("Invalid path") - } - - if (FilenameUtils.getPrefixLength(normalized) > 0) { - throw new BadRequestException("Absolute paths not allowed") - } - normalized - } - case class DashboardDataset( dataset: Dataset, ownerEmail: String, @@ -1264,15 +1225,25 @@ class DatasetResource extends LazyLogging { throw new NotFoundException(ERR_DATASET_VERSION_NOT_FOUND_MESSAGE) ) - val ownerNode = DatasetFileNode + val datasetsNode = DatasetFileNode .fromLakeFSRepositoryCommittedObjects( Map( - (user.getEmail, dataset.getName, latestVersion.getName) -> LakeFSStorageClient + ( + getOwner(ctx, did).getEmail, + dataset.getName, + latestVersion.getName + ) -> LakeFSStorageClient .retrieveObjectsOfVersion(dataset.getRepositoryName, latestVersion.getVersionHash) ) ) .head + val ownerNode = datasetsNode.getChildren.headOption.getOrElse( + throw new IllegalStateException( + s"Dataset file tree for ${dataset.getName} is missing its owner node" + ) + ) + DashboardDatasetVersion( latestVersion, ownerNode.children.get @@ -1499,7 +1470,7 @@ class DatasetResource extends LazyLogging { val datasetName = dataset.dataset.getName val repositoryName = dataset.dataset.getRepositoryName - val ownerFileNode = DatasetFileNode + val datasetsNode = DatasetFileNode .fromLakeFSRepositoryCommittedObjects( Map( (dataset.ownerEmail, datasetName, datasetVersion.getName) -> LakeFSStorageClient @@ -1508,6 +1479,12 @@ class DatasetResource extends LazyLogging { ) .head + val ownerFileNode = datasetsNode.getChildren.headOption.getOrElse( + throw new IllegalStateException( + s"Dataset file tree for $datasetName is missing its owner node" + ) + ) + DatasetVersionRootFileNodesResponse( ownerFileNode.children.get .find(_.getName == datasetName) @@ -1518,7 +1495,7 @@ class DatasetResource extends LazyLogging { .head .children .get, - DatasetFileNode.calculateTotalSize(List(ownerFileNode)) + DatasetFileNode.calculateTotalSize(List(datasetsNode)) ) } @@ -1588,7 +1565,8 @@ class DatasetResource extends LazyLogging { // Case 3: Neither repositoryName nor commitHash are provided, resolve normally val response = withTransaction(context) { ctx => val fileUri = FileResolver.resolve(decodedPathStr) - val document = DocumentFactory.openReadonlyDocument(fileUri).asInstanceOf[OnDataset] + val document = + DocumentFactory.openReadonlyDocument(fileUri).asInstanceOf[OnVersionedFileResource] val datasetDao = new DatasetDao(ctx.configuration()) val datasets = datasetDao.fetchByRepositoryName(document.getRepositoryName()).asScala.toList @@ -2216,7 +2194,7 @@ class DatasetResource extends LazyLogging { throw new BadRequestException("Cover image path is required") } - val normalized = DatasetResource.validateAndNormalizeFilePathOrThrow(request.coverImage) + val normalized = validateAndNormalizeFilePathOrThrow(request.coverImage) val extension = FilenameUtils.getExtension(normalized) if (extension == null || !ALLOWED_IMAGE_EXTENSIONS.contains(s".$extension".toLowerCase)) { @@ -2226,9 +2204,11 @@ class DatasetResource extends LazyLogging { val owner = getOwner(ctx, did) val document = DocumentFactory .openReadonlyDocument( - FileResolver.resolve(s"${owner.getEmail}/${dataset.getName}/$normalized") + FileResolver.resolve( + s"${ResourceType.Datasets}/${owner.getEmail}/${dataset.getName}/$normalized" + ) ) - .asInstanceOf[OnDataset] + .asInstanceOf[OnVersionedFileResource] val fileSize = withLakeFSErrorHandling(s"reading the size of cover image '$normalized'") { LakeFSStorageClient.getFileSize( @@ -2280,11 +2260,12 @@ class DatasetResource extends LazyLogging { ) val owner = getOwner(ctx, did) - val fullPath = s"${owner.getEmail}/${dataset.getName}/$coverImage" + val fullPath = + s"${ResourceType.Datasets}/${owner.getEmail}/${dataset.getName}/$coverImage" val document = DocumentFactory .openReadonlyDocument(FileResolver.resolve(fullPath)) - .asInstanceOf[OnDataset] + .asInstanceOf[OnVersionedFileResource] val presignedUrl = withLakeFSErrorHandling( s"generating a presigned URL for cover image '$coverImage'" @@ -2329,11 +2310,12 @@ class DatasetResource extends LazyLogging { Response.ok(Map("url" -> null)).build() case Some(coverImage) => val owner = getOwner(ctx, did) - val fullPath = s"${owner.getEmail}/${dataset.getName}/$coverImage" + val fullPath = + s"${ResourceType.Datasets}/${owner.getEmail}/${dataset.getName}/$coverImage" val document = DocumentFactory .openReadonlyDocument(FileResolver.resolve(fullPath)) - .asInstanceOf[OnDataset] + .asInstanceOf[OnVersionedFileResource] val presignedUrl = withLakeFSErrorHandling( s"generating a presigned URL for cover image '$coverImage'" diff --git a/file-service/src/main/scala/org/apache/texera/service/resource/ModelAccessResource.scala b/file-service/src/main/scala/org/apache/texera/service/resource/ModelAccessResource.scala new file mode 100644 index 00000000000..49effe90f08 --- /dev/null +++ b/file-service/src/main/scala/org/apache/texera/service/resource/ModelAccessResource.scala @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.service.resource + +import io.dropwizard.auth.Auth +import jakarta.annotation.security.RolesAllowed +import jakarta.ws.rs.core.{MediaType, Response} +import jakarta.ws.rs._ +import org.apache.texera.auth.SessionUser +import org.apache.texera.dao.SqlServer +import org.apache.texera.dao.SqlServer.withTransaction +import org.apache.texera.dao.jooq.generated.Tables.USER +import org.apache.texera.dao.jooq.generated.enums.PrivilegeEnum +import org.apache.texera.dao.jooq.generated.tables.ModelUserAccess.MODEL_USER_ACCESS +import org.apache.texera.dao.jooq.generated.tables.daos.{ModelDao, ModelUserAccessDao, UserDao} +import org.apache.texera.dao.jooq.generated.tables.pojos.{ModelUserAccess, User} +import org.apache.texera.service.resource.ModelAccessResource.{ + AccessEntry, + context, + getOwner, + userHasWriteAccess +} +import org.jooq.{DSLContext, EnumType} + +object ModelAccessResource { + private def context: DSLContext = + SqlServer + .getInstance() + .createDSLContext() + + def isModelPublic(ctx: DSLContext, mid: Integer): Boolean = { + val modelDao = new ModelDao(ctx.configuration()) + Option(modelDao.fetchOneByMid(mid)) + .flatMap(model => Option(model.getIsPublic)) + .contains(true) + } + + def userHasReadAccess(ctx: DSLContext, mid: Integer, uid: Integer): Boolean = { + isModelPublic(ctx, mid) || + userHasWriteAccess(ctx, mid, uid) || + getModelUserAccessPrivilege(ctx, mid, uid) == PrivilegeEnum.READ + } + + def userOwnModel(ctx: DSLContext, mid: Integer, uid: Integer): Boolean = { + val modelDao = new ModelDao(ctx.configuration()) + + Option(modelDao.fetchOneByMid(mid)) + .exists(_.getOwnerUid == uid) + } + + def userHasWriteAccess(ctx: DSLContext, mid: Integer, uid: Integer): Boolean = { + userOwnModel(ctx, mid, uid) || + getModelUserAccessPrivilege(ctx, mid, uid) == PrivilegeEnum.WRITE + } + + def getModelUserAccessPrivilege( + ctx: DSLContext, + mid: Integer, + uid: Integer + ): PrivilegeEnum = { + Option( + ctx + .select(MODEL_USER_ACCESS.PRIVILEGE) + .from(MODEL_USER_ACCESS) + .where( + MODEL_USER_ACCESS.MID + .eq(mid) + .and(MODEL_USER_ACCESS.UID.eq(uid)) + ) + .fetchOneInto(classOf[PrivilegeEnum]) + ).getOrElse(PrivilegeEnum.NONE) + } + + def getOwner(ctx: DSLContext, mid: Integer): User = { + val modelDao = new ModelDao(ctx.configuration()) + val userDao = new UserDao(ctx.configuration()) + + Option(modelDao.fetchOneByMid(mid)) + .flatMap(model => Option(model.getOwnerUid)) + .map(ownerUid => userDao.fetchOneByUid(ownerUid)) + .orNull + } + + case class AccessEntry(email: String, name: String, privilege: EnumType) {} + +} + +@Produces(Array(MediaType.APPLICATION_JSON)) +@RolesAllowed(Array("REGULAR", "ADMIN")) +@Path("/access/model") +class ModelAccessResource { + + /** + * This method returns the owner of a model + * + * @param mid , model id + * @return ownerEmail, the owner's email + */ + @GET + @Path("/owner/{mid}") + def getOwnerEmailOfModel(@PathParam("mid") mid: Integer): String = { + var email = "" + withTransaction(context) { ctx => + val owner = getOwner(ctx, mid) + if (owner != null) { + email = owner.getEmail + } + } + email + } + + /** + * Returns information about all current shared access of the given model + * + * @param mid model id + * @return a List of email/name/permission + */ + @GET + @Path("/list/{mid}") + def getAccessList( + @PathParam("mid") mid: Integer + ): java.util.List[AccessEntry] = { + withTransaction(context) { ctx => + val modelDao = new ModelDao(ctx.configuration()) + ctx + .select( + USER.EMAIL, + USER.NAME, + MODEL_USER_ACCESS.PRIVILEGE + ) + .from(MODEL_USER_ACCESS) + .join(USER) + .on(USER.UID.eq(MODEL_USER_ACCESS.UID)) + .where( + MODEL_USER_ACCESS.MID + .eq(mid) + .and(MODEL_USER_ACCESS.UID.notEqual(modelDao.fetchOneByMid(mid).getOwnerUid)) + ) + .fetchInto(classOf[AccessEntry]) + } + } + + /** + * This method shares a model to a user with a specific access type + * + * @param mid the given model + * @param email the email which the access is given to + * @param privilege the type of Access given to the target user + * @return rejection if user not permitted to share the model or Success Message + */ + @PUT + @Path("/grant/{mid}/{email}/{privilege}") + def grantAccess( + @PathParam("mid") mid: Integer, + @PathParam("email") email: String, + @PathParam("privilege") privilege: String, + @Auth user: SessionUser + ): Response = { + withTransaction(context) { ctx => + if (!userHasWriteAccess(ctx, mid, user.getUid)) { + throw new ForbiddenException(s"You do not have permission to modify model $mid") + } + val modelUserAccessDao = new ModelUserAccessDao(ctx.configuration()) + val userDao = new UserDao(ctx.configuration()) + modelUserAccessDao.merge( + new ModelUserAccess( + mid, + userDao.fetchOneByEmail(email).getUid, + PrivilegeEnum.valueOf(privilege) + ) + ) + Response.ok().build() + } + } + + /** + * This method revoke the user's access of the given model + * + * @param mid the given model + * @param email the email of the use whose access is about to be removed + * @return message indicating a success message + */ + @DELETE + @Path("/revoke/{mid}/{email}") + def revokeAccess( + @PathParam("mid") mid: Integer, + @PathParam("email") email: String, + @Auth user: SessionUser + ): Response = { + withTransaction(context) { ctx => + if (!userHasWriteAccess(ctx, mid, user.getUid)) { + throw new ForbiddenException(s"You do not have permission to modify model $mid") + } + + val userDao = new UserDao(ctx.configuration()) + + ctx + .delete(MODEL_USER_ACCESS) + .where( + MODEL_USER_ACCESS.UID + .eq(userDao.fetchOneByEmail(email).getUid) + .and(MODEL_USER_ACCESS.MID.eq(mid)) + ) + .execute() + + Response.ok().build() + } + } +} diff --git a/file-service/src/main/scala/org/apache/texera/service/resource/ModelResource.scala b/file-service/src/main/scala/org/apache/texera/service/resource/ModelResource.scala new file mode 100644 index 00000000000..4761b8f3fff --- /dev/null +++ b/file-service/src/main/scala/org/apache/texera/service/resource/ModelResource.scala @@ -0,0 +1,1750 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.service.resource + +import com.typesafe.scalalogging.LazyLogging +import io.dropwizard.auth.Auth +import jakarta.annotation.security.{PermitAll, RolesAllowed} +import jakarta.ws.rs._ +import jakarta.ws.rs.core._ +import org.apache.texera.amber.core.storage.util.LakeFSStorageClient +import org.apache.texera.auth.SessionUser +import org.apache.texera.common.config.StorageConfig +import org.apache.texera.dao.{SiteSettings, SqlServer} +import org.apache.texera.dao.SqlServer.withTransaction +import org.apache.texera.dao.jooq.generated.enums.PrivilegeEnum +import org.apache.texera.dao.jooq.generated.tables.Model.MODEL +import org.apache.texera.dao.jooq.generated.tables.ModelUploadSession.MODEL_UPLOAD_SESSION +import org.apache.texera.dao.jooq.generated.tables.ModelUploadSessionPart.MODEL_UPLOAD_SESSION_PART +import org.apache.texera.dao.jooq.generated.tables.ModelUserAccess.MODEL_USER_ACCESS +import org.apache.texera.dao.jooq.generated.tables.ModelVersion.MODEL_VERSION +import org.apache.texera.dao.jooq.generated.tables.User.USER +import org.apache.texera.dao.jooq.generated.tables.daos.{ + ModelDao, + ModelUserAccessDao, + ModelVersionDao +} +import org.apache.texera.dao.jooq.generated.tables.pojos.{Model, ModelUserAccess, ModelVersion} +import org.apache.texera.dao.jooq.generated.tables.records.ModelUploadSessionRecord +import org.apache.texera.service.`type`.DatasetFileNode +import org.apache.texera.service.resource.ModelAccessResource._ +import org.apache.texera.service.resource.ModelResource.{context, _} +import org.apache.texera.service.util.ResourceUploadUtils.{put, validateAndNormalizeFilePathOrThrow} +import org.apache.texera.service.util.S3StorageClient +import org.apache.texera.service.util.S3StorageClient.{ + MAXIMUM_NUM_OF_MULTIPART_S3_PARTS, + MINIMUM_NUM_OF_MULTIPART_S3_PART, + PHYSICAL_ADDRESS_EXPIRATION_TIME_HRS +} +import org.apache.texera.service.util.LakeFSExceptionHandler.withLakeFSErrorHandling +import org.jooq.exception.DataAccessException +import org.jooq.impl.DSL +import org.jooq.impl.DSL.{inline => inl} +import org.jooq.{DSLContext, EnumType, Record2, Result} +import software.amazon.awssdk.services.s3.model.UploadPartResponse + +import java.io.InputStream +import java.net.URLDecoder +import java.nio.charset.StandardCharsets +import java.sql.SQLException +import java.time.OffsetDateTime +import java.util.Optional +import scala.collection.mutable.ListBuffer +import scala.jdk.CollectionConverters._ +import scala.jdk.OptionConverters._ +import scala.util.Try + +object ModelResource { + + // MVP supports a single framework; stored on the model so later frameworks can be added. + private val DEFAULT_FRAMEWORK = "pytorch" + + private def context = + SqlServer + .getInstance() + .createDSLContext() + + private def singleFileUploadMaxBytes(defaultMiB: Long = 20L): Long = + SiteSettings.getLong("single_file_upload_max_size_mib", defaultMiB) * 1024L * 1024L + + /** + * Helper function to get the model from DB using mid + */ + private def getModelByID(ctx: DSLContext, mid: Integer): Model = { + val modelDao = new ModelDao(ctx.configuration()) + val model = modelDao.fetchOneByMid(mid) + if (model == null) { + throw new NotFoundException(f"Model $mid not found") + } + model + } + + /** + * Helper function to get the model version from DB using mvid + */ + private def getModelVersionByID(ctx: DSLContext, mvid: Integer): ModelVersion = { + val modelVersionDao = new ModelVersionDao(ctx.configuration()) + val version = modelVersionDao.fetchOneByMvid(mvid) + if (version == null) { + throw new NotFoundException("Model Version not found") + } + version + } + + /** + * Helper function to get the latest model version from the DB + */ + private def getLatestModelVersion(ctx: DSLContext, mid: Integer): Option[ModelVersion] = { + ctx + .selectFrom(MODEL_VERSION) + .where(MODEL_VERSION.MID.eq(mid)) + .orderBy(MODEL_VERSION.CREATION_TIME.desc()) + .limit(1) + .fetchOptionalInto(classOf[ModelVersion]) + .toScala + } + + case class DashboardModel( + model: Model, + ownerEmail: String, + accessPrivilege: EnumType, + isOwner: Boolean, + size: Long + ) + + case class CreateModelRequest( + modelName: String, + modelDescription: String, + isModelPublic: Boolean, + isModelDownloadable: Boolean, + framework: String, + format: String + ) + + case class ModelDescriptionModification(mid: Integer, description: String) + + case class ModelNameModification(mid: Integer, name: String) + + case class DashboardModelVersion( + modelVersion: ModelVersion, + fileNodes: List[DatasetFileNode] + ) + + case class ModelVersionRootFileNodesResponse( + fileNodes: List[DatasetFileNode], + size: Long + ) +} + +@Produces(Array(MediaType.APPLICATION_JSON)) +@Path("/model") +class ModelResource extends LazyLogging { + private val ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE = "User has no access to this model" + private val ERR_MODEL_VERSION_NOT_FOUND_MESSAGE = "The version of the model not found" + + private val MODEL_NAME_MAX_LENGTH = 128 + private val MODEL_NAME_PATTERN = "^[A-Za-z0-9_-]+$".r + + /** + * Validates the model name. + * + * Rules: + * - Must be 1 to 128 characters long. + * - Only letters, numbers, underscores, and hyphens are allowed. + * + * @param name The model name to validate. + * @throws jakarta.ws.rs.BadRequestException if the name is invalid. + */ + private def validateModelName(name: String): Unit = { + if (name == null || !MODEL_NAME_PATTERN.matches(name)) { + throw new BadRequestException( + "Invalid model name: only letters, numbers, underscores, and hyphens are allowed." + ) + } + if (name.length > MODEL_NAME_MAX_LENGTH) { + throw new BadRequestException( + s"Invalid model name: name must be at most $MODEL_NAME_MAX_LENGTH characters long." + ) + } + } + + /** + * Runs a model write and translates a (owner_uid, name) unique-constraint + * violation into the same BadRequestException the pre-checks throw, so + * requests losing a concurrent race get a 400 instead of a 500. + */ + private[resource] def failOnDuplicateModelName[T](op: => T): T = { + try op + catch { + case e: DataAccessException => + if (e.sqlState() == "23505") { + throw new BadRequestException("Model with the same name already exists") + } + throw e + } + } + + /** + * Helper function to get the model from DB with additional information including + * user access privilege and owner email + */ + private def getDashboardModel( + ctx: DSLContext, + mid: Integer, + requesterUid: Option[Integer] + ): DashboardModel = { + val targetModel = getModelByID(ctx, mid) + + if (requesterUid.isEmpty && !targetModel.getIsPublic) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } else if (requesterUid.exists(uid => !userHasReadAccess(ctx, mid, uid))) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + + val userAccessPrivilege = requesterUid + .map(uid => getModelUserAccessPrivilege(ctx, mid, uid)) + .getOrElse(PrivilegeEnum.READ) + + val isOwner = requesterUid.contains(targetModel.getOwnerUid) + + DashboardModel( + targetModel, + getOwner(ctx, mid).getEmail, + userAccessPrivilege, + isOwner, + withLakeFSErrorHandling(s"retrieving the size of model '${targetModel.getName}'") { + LakeFSStorageClient.retrieveRepositorySize(targetModel.getRepositoryName) + } + ) + } + + @POST + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/create") + @Consumes(Array(MediaType.APPLICATION_JSON)) + def createModel( + request: CreateModelRequest, + @Auth user: SessionUser + ): DashboardModel = { + + withTransaction(context) { ctx => + val uid = user.getUid + val modelUserAccessDao: ModelUserAccessDao = new ModelUserAccessDao(ctx.configuration()) + + val modelName = request.modelName + val modelDescription = request.modelDescription + val isModelPublic = request.isModelPublic + val isModelDownloadable = request.isModelDownloadable + + validateModelName(modelName) + + // Check if a model with the same name already exists + val duplicateExists = ctx.fetchExists( + ctx + .selectFrom(MODEL) + .where(MODEL.OWNER_UID.eq(uid)) + .and(MODEL.NAME.eq(modelName)) + ) + if (duplicateExists) { + throw new BadRequestException("Model with the same name already exists") + } + + // insert the model into the database + val model = new Model() + model.setName(modelName) + model.setDescription(modelDescription) + model.setIsPublic(isModelPublic) + model.setIsDownloadable(isModelDownloadable) + model.setOwnerUid(uid) + model.setFramework(Option(request.framework).filter(_.nonEmpty).getOrElse(DEFAULT_FRAMEWORK)) + model.setFormat(request.format) + + // insert record and get created model with mid + val createdModel = failOnDuplicateModelName { + ctx + .insertInto(MODEL) + .set(ctx.newRecord(MODEL, model)) + .returning() + .fetchOne() + } + + // Initialize the repository in LakeFS + val repositoryName = s"model-${createdModel.getMid}" + try { + withLakeFSErrorHandling(s"creating the repository of model '${model.getName}'") { + LakeFSStorageClient.initRepo(repositoryName) + } + } catch { + case e: Exception => + // roll back the model record so a failed LakeFS init leaves no orphan row + ctx + .deleteFrom(MODEL) + .where(MODEL.MID.eq(createdModel.getMid)) + .execute() + e match { + case web: WebApplicationException => throw web + case other => + throw new WebApplicationException( + s"Failed to create the model: ${other.getMessage}" + ) + } + } + + // update repository name of the created model + createdModel.setRepositoryName(repositoryName) + createdModel.update() + + // Insert the requester as the WRITE access user for this model + val modelUserAccess = new ModelUserAccess() + modelUserAccess.setMid(createdModel.getMid) + modelUserAccess.setUid(uid) + modelUserAccess.setPrivilege(PrivilegeEnum.WRITE) + modelUserAccessDao.insert(modelUserAccess) + + DashboardModel( + createdModel.into(classOf[Model]), + user.getEmail, + PrivilegeEnum.WRITE, + isOwner = true, + 0 + ) + } + } + + @DELETE + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/{mid}") + def deleteModel(@PathParam("mid") mid: Integer, @Auth user: SessionUser): Response = { + val uid = user.getUid + withTransaction(context) { ctx => + val modelDao = new ModelDao(ctx.configuration()) + val model = getModelByID(ctx, mid) + if (!userOwnModel(ctx, model.getMid, uid)) { + // throw the exception that user has no access to certain model + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + withLakeFSErrorHandling(s"deleting the repository of model '${model.getName}'") { + LakeFSStorageClient.deleteRepo(model.getRepositoryName) + } + // delete the directory on S3 + if ( + S3StorageClient.directoryExists(StorageConfig.lakefsBucketName, model.getRepositoryName) + ) { + S3StorageClient.deleteDirectory(StorageConfig.lakefsBucketName, model.getRepositoryName) + } + + // delete the model from the DB + modelDao.deleteById(model.getMid) + + Response.ok().build() + } + } + + @POST + @Consumes(Array(MediaType.APPLICATION_JSON)) + @Produces(Array(MediaType.APPLICATION_JSON)) + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/update/description") + def updateModelDescription( + modificator: ModelDescriptionModification, + @Auth sessionUser: SessionUser + ): Response = { + withTransaction(context) { ctx => + val uid = sessionUser.getUid + val modelDao = new ModelDao(ctx.configuration()) + val model = getModelByID(ctx, modificator.mid) + if (!userHasWriteAccess(ctx, modificator.mid, uid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + + model.setDescription(modificator.description) + modelDao.update(model) + Response.ok().build() + } + } + + @POST + @Consumes(Array(MediaType.APPLICATION_JSON)) + @Produces(Array(MediaType.APPLICATION_JSON)) + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/update/name") + def updateModelName( + modificator: ModelNameModification, + @Auth sessionUser: SessionUser + ): Response = { + withTransaction(context) { ctx => + val uid = sessionUser.getUid + val modelDao = new ModelDao(ctx.configuration()) + val model = getModelByID(ctx, modificator.mid) + if (!userHasWriteAccess(ctx, modificator.mid, uid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + + validateModelName(modificator.name) + + // Check if the owner already has another model with the same name + val duplicateExists = ctx.fetchExists( + ctx + .selectFrom(MODEL) + .where(MODEL.OWNER_UID.eq(model.getOwnerUid)) + .and(MODEL.NAME.eq(modificator.name)) + .and(MODEL.MID.notEqual(model.getMid)) + ) + if (duplicateExists) { + throw new BadRequestException("Model with the same name already exists") + } + + model.setName(modificator.name) + failOnDuplicateModelName { + modelDao.update(model) + } + Response.ok().build() + } + } + + @POST + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/{mid}/update/publicity") + def toggleModelPublicity( + @PathParam("mid") mid: Integer, + @Auth sessionUser: SessionUser + ): Response = { + withTransaction(context) { ctx => + val modelDao = new ModelDao(ctx.configuration()) + val uid = sessionUser.getUid + + if (!userHasWriteAccess(ctx, mid, uid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + + val existedModel = getModelByID(ctx, mid) + val newPublicStatus = !existedModel.getIsPublic + existedModel.setIsPublic(newPublicStatus) + + modelDao.update(existedModel) + Response.ok().build() + } + } + + @POST + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/{mid}/update/downloadable") + def toggleModelDownloadable( + @PathParam("mid") mid: Integer, + @Auth sessionUser: SessionUser + ): Response = { + withTransaction(context) { ctx => + val modelDao = new ModelDao(ctx.configuration()) + val uid = sessionUser.getUid + + if (!userOwnModel(ctx, mid, uid)) { + throw new ForbiddenException("Only model owners can modify download permissions") + } + + val existedModel = getModelByID(ctx, mid) + val newDownloadableStatus = !existedModel.getIsDownloadable + + existedModel.setIsDownloadable(newDownloadableStatus) + + modelDao.update(existedModel) + Response.ok().build() + } + } + + @GET + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/list") + def listModels( + @Auth user: SessionUser + ): List[DashboardModel] = { + val uid = user.getUid + withTransaction(context)(ctx => { + var accessibleModels: ListBuffer[DashboardModel] = ListBuffer() + // first fetch all models user have explicit access to + accessibleModels = ListBuffer.from( + ctx + .select() + .from( + MODEL + .leftJoin(MODEL_USER_ACCESS) + .on(MODEL_USER_ACCESS.MID.eq(MODEL.MID)) + .leftJoin(USER) + .on(USER.UID.eq(MODEL.OWNER_UID)) + ) + .where(MODEL_USER_ACCESS.UID.eq(uid)) + .fetch() + .map(record => { + val model = record.into(MODEL).into(classOf[Model]) + val modelAccess = record.into(MODEL_USER_ACCESS).into(classOf[ModelUserAccess]) + val ownerEmail = record.into(USER).getEmail + DashboardModel( + isOwner = model.getOwnerUid == uid, + model = model, + accessPrivilege = modelAccess.getPrivilege, + ownerEmail = ownerEmail, + size = 0 + ) + }) + .asScala + ) + + // then we fetch the public models and merge it as a part of the result if not exist + val publicModels = ctx + .select() + .from( + MODEL + .leftJoin(USER) + .on(USER.UID.eq(MODEL.OWNER_UID)) + ) + .where(MODEL.IS_PUBLIC.eq(true)) + .fetch() + .asScala + .flatMap { record => + val model = record.into(MODEL).into(classOf[Model]) + val ownerEmail = record.into(USER).getEmail + try { + Some( + DashboardModel( + isOwner = false, + model = model, + accessPrivilege = PrivilegeEnum.READ, + ownerEmail = ownerEmail, + size = LakeFSStorageClient.retrieveRepositorySize(model.getRepositoryName) + ) + ) + } catch { + case e: io.lakefs.clients.sdk.ApiException => + logger.error( + s"LakeFS ApiException for model repository '${model.getRepositoryName}': ${e.getMessage}", + e + ) + None + } + } + publicModels.foreach { publicModel => + if (!accessibleModels.exists(_.model.getMid == publicModel.model.getMid)) { + accessibleModels = accessibleModels :+ publicModel + } + } + accessibleModels.toList + }) + } + + @GET + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/{mid}") + def getModel( + @PathParam("mid") mid: Integer, + @Auth user: SessionUser + ): DashboardModel = { + val uid = user.getUid + withTransaction(context)(ctx => getDashboardModel(ctx, mid, Some(uid))) + } + + @GET + @PermitAll + @Path("/public/{mid}") + def getPublicModel( + @PathParam("mid") mid: Integer + ): DashboardModel = { + withTransaction(context)(ctx => getDashboardModel(ctx, mid, None)) + } + + // =========================================================================== + // Versioning + // =========================================================================== + + @POST + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/{mid}/version/create") + @Consumes(Array(MediaType.TEXT_PLAIN)) + def createModelVersion( + versionName: String, + @PathParam("mid") mid: Integer, + @Auth user: SessionUser + ): DashboardModelVersion = { + val uid = user.getUid + withTransaction(context) { ctx => + if (!userHasWriteAccess(ctx, mid, uid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + + val model = getModelByID(ctx, mid) + val modelName = model.getName + val repositoryName = model.getRepositoryName + + // Check if there are any changes in LakeFS before creating a new version + val diffs = withLakeFSErrorHandling { + LakeFSStorageClient.retrieveUncommittedObjects(repoName = repositoryName) + } + + if (diffs.isEmpty) { + throw new WebApplicationException( + "No changes detected in model. Version creation aborted.", + Response.Status.BAD_REQUEST + ) + } + + // Generate a new version name + val versionCount = ctx + .selectCount() + .from(MODEL_VERSION) + .where(MODEL_VERSION.MID.eq(mid)) + .fetchOne(0, classOf[Int]) + + val sanitizedVersionName = Option(versionName).filter(_.nonEmpty).getOrElse("") + val newVersionName = if (sanitizedVersionName.isEmpty) { + s"v${versionCount + 1}" + } else { + s"v${versionCount + 1} - $sanitizedVersionName" + } + + // Create a commit in LakeFS + val commit = withLakeFSErrorHandling { + LakeFSStorageClient.createCommit( + repoName = repositoryName, + branch = "main", + commitMessage = s"Created model version: $newVersionName" + ) + } + + if (commit == null || commit.getId == null) { + throw new WebApplicationException( + "Failed to create commit in LakeFS. Version creation aborted.", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + + // Create a new model version entry in the database + val modelVersion = new ModelVersion() + modelVersion.setMid(mid) + modelVersion.setCreatorUid(uid) + modelVersion.setName(newVersionName) + modelVersion.setVersionHash(commit.getId) // Store LakeFS version hash + + val insertedVersion = ctx + .insertInto(MODEL_VERSION) + .set(ctx.newRecord(MODEL_VERSION, modelVersion)) + .returning() + .fetchOne() + .into(classOf[ModelVersion]) + + // Retrieve committed file structure + val fileNodes = withLakeFSErrorHandling { + LakeFSStorageClient.retrieveObjectsOfVersion(repositoryName, commit.getId) + } + + DashboardModelVersion( + insertedVersion, + DatasetFileNode + .fromLakeFSRepositoryCommittedObjects( + Map((user.getEmail, modelName, newVersionName) -> fileNodes) + ) + ) + } + } + + @GET + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/{mid}/version/list") + def getModelVersionList( + @PathParam("mid") mid: Integer, + @Auth user: SessionUser + ): List[ModelVersion] = { + val uid = user.getUid + withTransaction(context)(ctx => { + val model = getModelByID(ctx, mid) + if (!userHasReadAccess(ctx, model.getMid, uid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + fetchModelVersions(ctx, model.getMid) + }) + } + + @GET + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/{mid}/version/latest") + def retrieveLatestModelVersion( + @PathParam("mid") mid: Integer, + @Auth user: SessionUser + ): DashboardModelVersion = { + val uid = user.getUid + withTransaction(context)(ctx => { + if (!userHasReadAccess(ctx, mid, uid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + val model = getModelByID(ctx, mid) + val latestVersion = getLatestModelVersion(ctx, mid).getOrElse( + throw new NotFoundException(ERR_MODEL_VERSION_NOT_FOUND_MESSAGE) + ) + DashboardModelVersion(latestVersion, versionRootFileNodes(ctx, mid, latestVersion, Some(uid))) + }) + } + + @GET + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/{mid}/version/{mvid}/rootFileNodes") + def retrieveModelVersionRootFileNodes( + @PathParam("mid") mid: Integer, + @PathParam("mvid") mvid: Integer, + @Auth user: SessionUser + ): ModelVersionRootFileNodesResponse = { + val uid = user.getUid + withTransaction(context)(ctx => fetchModelVersionRootFileNodes(ctx, mid, mvid, Some(uid))) + } + + // =========================================================================== + // File upload (one-shot + session-based multipart) + // =========================================================================== + + @POST + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/{mid}/upload") + @Consumes(Array(MediaType.APPLICATION_OCTET_STREAM)) + def uploadOneFileToModel( + @PathParam("mid") mid: Integer, + @QueryParam("filePath") encodedFilePath: String, + @QueryParam("message") message: String, + fileStream: InputStream, + @Context headers: HttpHeaders, + @Auth user: SessionUser + ): Response = { + val uid = user.getUid + var repoName: String = null + var filePath: String = null + var uploadId: String = null + var physicalAddress: String = null + + try { + withTransaction(context) { ctx => + if (!userHasWriteAccess(ctx, mid, uid)) + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + + val model = getModelByID(ctx, mid) + repoName = model.getRepositoryName + filePath = URLDecoder.decode(encodedFilePath, StandardCharsets.UTF_8.name) + + // ---------- decide part-size & number-of-parts ---------- + val declaredLen = Option(headers.getHeaderString(HttpHeaders.CONTENT_LENGTH)).map(_.toLong) + var partSize = StorageConfig.s3MultipartUploadPartSize + + declaredLen.foreach { ln => + val needed = ((ln + partSize - 1) / partSize).toInt + if (needed > MAXIMUM_NUM_OF_MULTIPART_S3_PARTS) + partSize = math.max( + MINIMUM_NUM_OF_MULTIPART_S3_PART, + ln / (MAXIMUM_NUM_OF_MULTIPART_S3_PARTS - 1) + ) + } + + val expectedParts = declaredLen + .map(ln => ((ln + partSize - 1) / partSize).toInt + 1) + .getOrElse(MAXIMUM_NUM_OF_MULTIPART_S3_PARTS) + + // ---------- ask LakeFS for presigned URLs ---------- + val presign = LakeFSStorageClient + .initiatePresignedMultipartUploads(repoName, filePath, expectedParts) + uploadId = presign.getUploadId + val presignedUrls = presign.getPresignedUrls.asScala.iterator + physicalAddress = presign.getPhysicalAddress + + // ---------- stream & upload parts ---------- + val buf = new Array[Byte](partSize.toInt) + var buffered = 0 + var partNumber = 1 + val completedParts = ListBuffer[(Int, String)]() + + @inline def flush(): Unit = { + if (buffered == 0) return + if (!presignedUrls.hasNext) + throw new WebApplicationException("Ran out of presigned part URLs – ask for more parts") + + val etag = put(buf, buffered, presignedUrls.next(), partNumber) + completedParts += ((partNumber, etag)) + partNumber += 1 + buffered = 0 + } + + var read = fileStream.read(buf, buffered, buf.length - buffered) + while (read != -1) { + buffered += read + if (buffered == buf.length) flush() + read = fileStream.read(buf, buffered, buf.length - buffered) + } + fileStream.close() + flush() + + // ---------- complete upload ---------- + withLakeFSErrorHandling(s"completing the multipart upload of file '$filePath'") { + LakeFSStorageClient.completePresignedMultipartUploads( + repoName, + filePath, + uploadId, + completedParts.toList, + physicalAddress + ) + } + + Response.ok(Map("message" -> s"Uploaded $filePath in ${completedParts.size} parts")).build() + } + } catch { + case e: Exception => + if (repoName != null && filePath != null && uploadId != null && physicalAddress != null) { + try { + LakeFSStorageClient.abortPresignedMultipartUploads( + repoName, + filePath, + uploadId, + physicalAddress + ) + } catch { case _: Throwable => () } + } + e match { + case web: WebApplicationException => throw web + case other => + throw new WebApplicationException( + s"Failed to upload file to model: ${other.getMessage}", + other + ) + } + } + } + + @DELETE + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/{mid}/file") + @Consumes(Array(MediaType.APPLICATION_JSON)) + def deleteModelFile( + @PathParam("mid") mid: Integer, + @QueryParam("filePath") encodedFilePath: String, + @Auth user: SessionUser + ): Response = { + val uid = user.getUid + withTransaction(context) { ctx => + if (!userHasWriteAccess(ctx, mid, uid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + val repositoryName = getModelByID(ctx, mid).getRepositoryName + + val filePath = URLDecoder.decode(encodedFilePath, StandardCharsets.UTF_8.name()) + withLakeFSErrorHandling(s"deleting file '$filePath' from the model repository") { + LakeFSStorageClient.deleteObject(repositoryName, filePath) + } + + Response.ok().build() + } + } + + @POST + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/multipart-upload") + @Consumes(Array(MediaType.APPLICATION_JSON)) + def multipartUpload( + @QueryParam("type") operationType: String, + @QueryParam("ownerEmail") ownerEmail: String, + @QueryParam("modelName") modelName: String, + @QueryParam("filePath") filePath: String, + @QueryParam("fileSizeBytes") fileSizeBytes: Optional[java.lang.Long], + @QueryParam("partSizeBytes") partSizeBytes: Optional[java.lang.Long], + @QueryParam("restart") restart: Optional[java.lang.Boolean], + @Auth user: SessionUser + ): Response = { + val uid = user.getUid + val model: Model = getModelBy(ownerEmail, modelName) + + operationType.toLowerCase match { + case "list" => listMultipartUploads(model.getMid, uid) + case "init" => + initMultipartUpload(model.getMid, filePath, fileSizeBytes, partSizeBytes, restart, uid) + case "finish" => finishMultipartUpload(model.getMid, filePath, uid) + case "abort" => abortMultipartUpload(model.getMid, filePath, uid) + case _ => + throw new BadRequestException("Invalid type parameter. Use 'init', 'finish', or 'abort'.") + } + } + + @POST + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Consumes(Array(MediaType.APPLICATION_OCTET_STREAM)) + @Path("/multipart-upload/part") + def uploadPart( + @QueryParam("ownerEmail") modelOwnerEmail: String, + @QueryParam("modelName") modelName: String, + @QueryParam("filePath") encodedFilePath: String, + @QueryParam("partNumber") partNumber: Int, + partStream: InputStream, + @Context headers: HttpHeaders, + @Auth user: SessionUser + ): Response = { + + val uid = user.getUid + val model: Model = getModelBy(modelOwnerEmail, modelName) + val mid = model.getMid + + if (encodedFilePath == null || encodedFilePath.isEmpty) + throw new BadRequestException("filePath is required") + if (partNumber < 1) + throw new BadRequestException("partNumber must be >= 1") + + val filePath = validateAndNormalizeFilePathOrThrow( + URLDecoder.decode(encodedFilePath, StandardCharsets.UTF_8.name()) + ) + + val contentLength = + Option(headers.getHeaderString(HttpHeaders.CONTENT_LENGTH)) + .map(_.trim) + .flatMap(s => Try(s.toLong).toOption) + .filter(_ > 0) + .getOrElse { + throw new BadRequestException("Invalid/Missing Content-Length") + } + + withTransaction(context) { ctx => + if (!userHasWriteAccess(ctx, mid, uid)) + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + + val session = ctx + .selectFrom(MODEL_UPLOAD_SESSION) + .where( + MODEL_UPLOAD_SESSION.UID + .eq(uid) + .and(MODEL_UPLOAD_SESSION.MID.eq(mid)) + .and(MODEL_UPLOAD_SESSION.FILE_PATH.eq(filePath)) + ) + .fetchOne() + + if (session == null) + throw new NotFoundException("Upload session not found. Call type=init first.") + + val expectedParts: Int = session.getNumPartsRequested + val fileSizeBytesValue: Long = session.getFileSizeBytes + val partSizeBytesValue: Long = session.getPartSizeBytes + + if (fileSizeBytesValue <= 0L) { + throw new WebApplicationException( + s"Upload session has an invalid file size of $fileSizeBytesValue. Restart the upload.", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + if (partSizeBytesValue <= 0L) { + throw new WebApplicationException( + s"Upload session has an invalid part size of $partSizeBytesValue. Restart the upload.", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + + // lastPartSize = fileSize - partSize*(expectedParts-1) + val nMinus1: Long = expectedParts.toLong - 1L + if (nMinus1 < 0L) { + throw new WebApplicationException( + s"Upload session has an invalid number of requested parts of $expectedParts. Restart the upload.", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + if (nMinus1 > 0L && partSizeBytesValue > Long.MaxValue / nMinus1) { + throw new WebApplicationException( + "Overflow while computing last part size", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + val prefixBytes: Long = partSizeBytesValue * nMinus1 + if (prefixBytes > fileSizeBytesValue) { + throw new WebApplicationException( + s"Upload session is invalid: computed bytes before last part ($prefixBytes) exceed declared file size ($fileSizeBytesValue). Restart the upload.", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + val lastPartSize: Long = fileSizeBytesValue - prefixBytes + if (lastPartSize <= 0L || lastPartSize > partSizeBytesValue) { + throw new WebApplicationException( + s"Upload session is invalid: computed last part size ($lastPartSize bytes) must be within 1..$partSizeBytesValue bytes. Restart the upload.", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + + val allowedSize: Long = + if (partNumber < expectedParts) partSizeBytesValue else lastPartSize + + if (partNumber > expectedParts) { + throw new BadRequestException( + s"$partNumber exceeds the requested parts on init: $expectedParts" + ) + } + + if (partNumber < expectedParts && contentLength < MINIMUM_NUM_OF_MULTIPART_S3_PART) { + throw new BadRequestException( + s"Part $partNumber is too small ($contentLength bytes). " + + s"All non-final parts must be >= $MINIMUM_NUM_OF_MULTIPART_S3_PART bytes." + ) + } + + if (contentLength != allowedSize) { + throw new BadRequestException( + s"Invalid part size for partNumber=$partNumber. " + + s"Expected Content-Length=$allowedSize, got $contentLength." + ) + } + + val physicalAddr = Option(session.getPhysicalAddress).map(_.trim).getOrElse("") + if (physicalAddr.isEmpty) { + throw new WebApplicationException( + "Upload session is missing physicalAddress. Restart the upload.", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + + val uploadId = session.getUploadId + val (bucket, key) = + try LakeFSStorageClient.parsePhysicalAddress(physicalAddr) + catch { + case e: IllegalArgumentException => + throw new WebApplicationException( + s"Upload session has invalid physicalAddress. Restart the upload. (${e.getMessage})", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + + // Per-part lock: if another request is streaming the same part, fail fast. + val partRow = + try { + ctx + .selectFrom(MODEL_UPLOAD_SESSION_PART) + .where( + MODEL_UPLOAD_SESSION_PART.UPLOAD_ID + .eq(uploadId) + .and(MODEL_UPLOAD_SESSION_PART.PART_NUMBER.eq(partNumber)) + ) + .forUpdate() + .noWait() + .fetchOne() + } catch { + case e: DataAccessException + if Option(e.getCause) + .collect { case s: SQLException => s.getSQLState } + .contains("55P03") => + throw new WebApplicationException( + s"Part $partNumber is already being uploaded", + Response.Status.CONFLICT + ) + } + + if (partRow == null) { + throw new WebApplicationException( + s"Part row not initialized for part $partNumber. Restart the upload.", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + + // Idempotency: if ETag already set, accept the retry quickly. + val existing = Option(partRow.getEtag).map(_.trim).getOrElse("") + if (existing.isEmpty) { + val response: UploadPartResponse = + S3StorageClient.uploadPartWithRequest( + bucket = bucket, + key = key, + uploadId = uploadId, + partNumber = partNumber, + inputStream = partStream, + contentLength = Some(contentLength) + ) + + val etagClean = Option(response.eTag()).map(_.replace("\"", "")).map(_.trim).getOrElse("") + if (etagClean.isEmpty) { + throw new WebApplicationException( + s"Missing ETag returned from S3 for part $partNumber", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + + ctx + .update(MODEL_UPLOAD_SESSION_PART) + .set(MODEL_UPLOAD_SESSION_PART.ETAG, etagClean) + .where( + MODEL_UPLOAD_SESSION_PART.UPLOAD_ID + .eq(uploadId) + .and(MODEL_UPLOAD_SESSION_PART.PART_NUMBER.eq(partNumber)) + ) + .execute() + } + Response.ok().build() + } + } + + // =========================================================================== + // Private helpers + // =========================================================================== + + private def fetchModelVersions(ctx: DSLContext, mid: Integer): List[ModelVersion] = { + ctx + .selectFrom(MODEL_VERSION) + .where(MODEL_VERSION.MID.eq(mid)) + .orderBy(MODEL_VERSION.CREATION_TIME.desc()) + .fetchInto(classOf[ModelVersion]) + .asScala + .toList + } + + /** + * Builds the file-tree children of a single model version, drilling into the + * owner/model/version nesting produced by DatasetFileNode. + */ + private def versionRootFileNodes( + ctx: DSLContext, + mid: Integer, + modelVersion: ModelVersion, + uid: Option[Integer] + ): List[DatasetFileNode] = { + val model = getModelByID(ctx, mid) + val ownerEmail = getOwner(ctx, mid).getEmail + + val modelsNode = DatasetFileNode + .fromLakeFSRepositoryCommittedObjects( + Map( + (ownerEmail, model.getName, modelVersion.getName) -> LakeFSStorageClient + .retrieveObjectsOfVersion(model.getRepositoryName, modelVersion.getVersionHash) + ) + ) + .head + + val ownerNode = modelsNode.getChildren.headOption.getOrElse( + throw new IllegalStateException( + s"Model file tree for ${model.getName} is missing its owner node" + ) + ) + + ownerNode.children.get + .find(_.getName == model.getName) + .head + .children + .get + .find(_.getName == modelVersion.getName) + .head + .children + .get + } + + private def fetchModelVersionRootFileNodes( + ctx: DSLContext, + mid: Integer, + mvid: Integer, + uid: Option[Integer] + ): ModelVersionRootFileNodesResponse = { + val model = getDashboardModel(ctx, mid, uid) + val modelVersion = getModelVersionByID(ctx, mvid) + val modelName = model.model.getName + val repositoryName = model.model.getRepositoryName + + val modelsNode = DatasetFileNode + .fromLakeFSRepositoryCommittedObjects( + Map( + (model.ownerEmail, modelName, modelVersion.getName) -> LakeFSStorageClient + .retrieveObjectsOfVersion(repositoryName, modelVersion.getVersionHash) + ) + ) + .head + + val ownerFileNode = modelsNode.getChildren.headOption.getOrElse( + throw new IllegalStateException( + s"Model file tree for $modelName is missing its owner node" + ) + ) + + ModelVersionRootFileNodesResponse( + ownerFileNode.children.get + .find(_.getName == modelName) + .head + .children + .get + .find(_.getName == modelVersion.getName) + .head + .children + .get, + DatasetFileNode.calculateTotalSize(List(modelsNode)) + ) + } + + private def getModelBy(ownerEmail: String, modelName: String): Model = { + val model = context + .select(MODEL.fields: _*) + .from(MODEL) + .leftJoin(USER) + .on(USER.UID.eq(MODEL.OWNER_UID)) + .where(USER.EMAIL.eq(ownerEmail)) + .and(MODEL.NAME.eq(modelName)) + .fetchOneInto(classOf[Model]) + if (model == null) { + throw new BadRequestException("Model not found") + } + model + } + + private def listMultipartUploads(mid: Integer, requesterUid: Int): Response = { + withTransaction(context) { ctx => + if (!userHasWriteAccess(ctx, mid, requesterUid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + + val filePaths = + ctx + .selectDistinct(MODEL_UPLOAD_SESSION.FILE_PATH) + .from(MODEL_UPLOAD_SESSION) + .where(MODEL_UPLOAD_SESSION.MID.eq(mid)) + .and( + DSL.condition( + "created_at > current_timestamp - (? * interval '1 hour')", + PHYSICAL_ADDRESS_EXPIRATION_TIME_HRS + ) + ) + .orderBy(MODEL_UPLOAD_SESSION.FILE_PATH.asc()) + .fetch(MODEL_UPLOAD_SESSION.FILE_PATH) + .asScala + .toList + + Response.ok(Map("filePaths" -> filePaths.asJava)).build() + } + } + + private def initMultipartUpload( + mid: Integer, + encodedFilePath: String, + fileSizeBytes: Optional[java.lang.Long], + partSizeBytes: Optional[java.lang.Long], + restart: Optional[java.lang.Boolean], + uid: Integer + ): Response = { + + withTransaction(context) { ctx => + if (!userHasWriteAccess(ctx, mid, uid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + + val model = getModelByID(ctx, mid) + val repositoryName = model.getRepositoryName + + val filePath = + validateAndNormalizeFilePathOrThrow( + URLDecoder.decode(encodedFilePath, StandardCharsets.UTF_8.name()) + ) + + if (fileSizeBytes == null || !fileSizeBytes.isPresent) + throw new BadRequestException("fileSizeBytes is required for initialization") + if (partSizeBytes == null || !partSizeBytes.isPresent) + throw new BadRequestException("partSizeBytes is required for initialization") + + val fileSizeBytesValue: Long = fileSizeBytes.get.longValue() + val partSizeBytesValue: Long = partSizeBytes.get.longValue() + + if (fileSizeBytesValue <= 0L) throw new BadRequestException("fileSizeBytes must be > 0") + if (partSizeBytesValue <= 0L) throw new BadRequestException("partSizeBytes must be > 0") + + val totalMaxBytes: Long = singleFileUploadMaxBytes() + if (totalMaxBytes <= 0L) { + throw new WebApplicationException( + "singleFileUploadMaxBytes must be > 0", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + if (fileSizeBytesValue > totalMaxBytes) { + throw new BadRequestException( + s"fileSizeBytes=$fileSizeBytesValue exceeds singleFileUploadMaxBytes=$totalMaxBytes" + ) + } + + val addend: Long = partSizeBytesValue - 1L + if (addend < 0L || fileSizeBytesValue > Long.MaxValue - addend) { + throw new WebApplicationException( + "Overflow while computing numParts", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + + val numPartsLong: Long = (fileSizeBytesValue + addend) / partSizeBytesValue + if (numPartsLong < 1L || numPartsLong > MAXIMUM_NUM_OF_MULTIPART_S3_PARTS.toLong) { + throw new BadRequestException( + s"Computed numParts=$numPartsLong is out of range 1..$MAXIMUM_NUM_OF_MULTIPART_S3_PARTS" + ) + } + val computedNumParts: Int = numPartsLong.toInt + + if (computedNumParts > 1 && partSizeBytesValue < MINIMUM_NUM_OF_MULTIPART_S3_PART) { + throw new BadRequestException( + s"partSizeBytes=$partSizeBytesValue is too small. " + + s"All non-final parts must be >= $MINIMUM_NUM_OF_MULTIPART_S3_PART bytes." + ) + } + var session: ModelUploadSessionRecord = null + var rows: Result[Record2[Integer, String]] = null + try { + session = ctx + .selectFrom(MODEL_UPLOAD_SESSION) + .where( + MODEL_UPLOAD_SESSION.UID + .eq(uid) + .and(MODEL_UPLOAD_SESSION.MID.eq(mid)) + .and(MODEL_UPLOAD_SESSION.FILE_PATH.eq(filePath)) + ) + .forUpdate() + .noWait() + .fetchOne() + if (session != null) { + rows = ctx + .select(MODEL_UPLOAD_SESSION_PART.PART_NUMBER, MODEL_UPLOAD_SESSION_PART.ETAG) + .from(MODEL_UPLOAD_SESSION_PART) + .where(MODEL_UPLOAD_SESSION_PART.UPLOAD_ID.eq(session.getUploadId)) + .forUpdate() + .noWait() + .fetch() + val dbFileSize = session.getFileSizeBytes + val dbPartSize = session.getPartSizeBytes + val dbNumParts = session.getNumPartsRequested + val createdAt: OffsetDateTime = session.getCreatedAt + + val isExpired = + createdAt + .plusHours(PHYSICAL_ADDRESS_EXPIRATION_TIME_HRS.toLong) + .isBefore(OffsetDateTime.now(createdAt.getOffset)) + + val conflictConfig = + dbFileSize != fileSizeBytesValue || + dbPartSize != partSizeBytesValue || + dbNumParts != computedNumParts || + isExpired || + Option(restart).exists(_.orElse(false)) + + if (conflictConfig) { + // Parts will be deleted automatically (ON DELETE CASCADE) + ctx + .deleteFrom(MODEL_UPLOAD_SESSION) + .where(MODEL_UPLOAD_SESSION.UPLOAD_ID.eq(session.getUploadId)) + .execute() + + try { + LakeFSStorageClient.abortPresignedMultipartUploads( + repositoryName, + filePath, + session.getUploadId, + session.getPhysicalAddress + ) + } catch { case _: Throwable => () } + session = null + rows = null + } + } + } catch { + case e: DataAccessException + if Option(e.getCause) + .collect { case s: SQLException => s.getSQLState } + .contains("55P03") => + throw new WebApplicationException( + "Another client is uploading this file", + Response.Status.CONFLICT + ) + } + + if (session == null) { + val presign = withLakeFSErrorHandling { + LakeFSStorageClient.initiatePresignedMultipartUploads( + repositoryName, + filePath, + computedNumParts + ) + } + + val uploadIdStr = presign.getUploadId + val physicalAddr = presign.getPhysicalAddress + + try { + val rowsInserted = ctx + .insertInto(MODEL_UPLOAD_SESSION) + .set(MODEL_UPLOAD_SESSION.FILE_PATH, filePath) + .set(MODEL_UPLOAD_SESSION.MID, mid) + .set(MODEL_UPLOAD_SESSION.UID, uid) + .set(MODEL_UPLOAD_SESSION.UPLOAD_ID, uploadIdStr) + .set(MODEL_UPLOAD_SESSION.PHYSICAL_ADDRESS, physicalAddr) + .set(MODEL_UPLOAD_SESSION.NUM_PARTS_REQUESTED, Integer.valueOf(computedNumParts)) + .set(MODEL_UPLOAD_SESSION.FILE_SIZE_BYTES, java.lang.Long.valueOf(fileSizeBytesValue)) + .set(MODEL_UPLOAD_SESSION.PART_SIZE_BYTES, java.lang.Long.valueOf(partSizeBytesValue)) + .onDuplicateKeyIgnore() + .execute() + + if (rowsInserted == 1) { + val partNumberSeries = + DSL.generateSeries(1, computedNumParts).asTable("gs", "partNumberField") + val partNumberField = partNumberSeries.field("partNumberField", classOf[Integer]) + + ctx + .insertInto( + MODEL_UPLOAD_SESSION_PART, + MODEL_UPLOAD_SESSION_PART.UPLOAD_ID, + MODEL_UPLOAD_SESSION_PART.PART_NUMBER, + MODEL_UPLOAD_SESSION_PART.ETAG + ) + .select( + ctx + .select( + inl(uploadIdStr), + partNumberField, + inl("") + ) + .from(partNumberSeries) + ) + .execute() + + session = ctx + .selectFrom(MODEL_UPLOAD_SESSION) + .where( + MODEL_UPLOAD_SESSION.UID + .eq(uid) + .and(MODEL_UPLOAD_SESSION.MID.eq(mid)) + .and(MODEL_UPLOAD_SESSION.FILE_PATH.eq(filePath)) + ) + .fetchOne() + } else { + try { + LakeFSStorageClient.abortPresignedMultipartUploads( + repositoryName, + filePath, + uploadIdStr, + physicalAddr + ) + } catch { case _: Throwable => () } + + session = ctx + .selectFrom(MODEL_UPLOAD_SESSION) + .where( + MODEL_UPLOAD_SESSION.UID + .eq(uid) + .and(MODEL_UPLOAD_SESSION.MID.eq(mid)) + .and(MODEL_UPLOAD_SESSION.FILE_PATH.eq(filePath)) + ) + .fetchOne() + } + } catch { + case e: Exception => + try { + LakeFSStorageClient.abortPresignedMultipartUploads( + repositoryName, + filePath, + uploadIdStr, + physicalAddr + ) + } catch { case _: Throwable => () } + throw e + } + } + + if (session == null) { + throw new WebApplicationException( + "Failed to create or locate upload session", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + + val uploadId = session.getUploadId + val nParts = session.getNumPartsRequested + + if (rows == null) { + rows = + try { + ctx + .select(MODEL_UPLOAD_SESSION_PART.PART_NUMBER, MODEL_UPLOAD_SESSION_PART.ETAG) + .from(MODEL_UPLOAD_SESSION_PART) + .where(MODEL_UPLOAD_SESSION_PART.UPLOAD_ID.eq(uploadId)) + .forUpdate() + .noWait() + .fetch() + } catch { + case e: DataAccessException + if Option(e.getCause) + .collect { case s: SQLException => s.getSQLState } + .contains("55P03") => + throw new WebApplicationException( + "Another client is uploading parts for this file", + Response.Status.CONFLICT + ) + } + } + + val missingParts = rows.asScala + .filter(r => + Option(r.get(MODEL_UPLOAD_SESSION_PART.ETAG)).map(_.trim).getOrElse("").isEmpty + ) + .map(r => r.get(MODEL_UPLOAD_SESSION_PART.PART_NUMBER).intValue()) + .toList + + val completedPartsCount = nParts - missingParts.size + + Response + .ok( + Map( + "missingParts" -> missingParts.asJava, + "completedPartsCount" -> Integer.valueOf(completedPartsCount) + ) + ) + .build() + } + } + + private def finishMultipartUpload( + mid: Integer, + encodedFilePath: String, + uid: Int + ): Response = { + + val filePath = validateAndNormalizeFilePathOrThrow( + URLDecoder.decode(encodedFilePath, StandardCharsets.UTF_8.name()) + ) + + withTransaction(context) { ctx => + if (!userHasWriteAccess(ctx, mid, uid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + + val model = getModelByID(ctx, mid) + + val session = + try { + ctx + .selectFrom(MODEL_UPLOAD_SESSION) + .where( + MODEL_UPLOAD_SESSION.UID + .eq(uid) + .and(MODEL_UPLOAD_SESSION.MID.eq(mid)) + .and(MODEL_UPLOAD_SESSION.FILE_PATH.eq(filePath)) + ) + .forUpdate() + .noWait() + .fetchOne() + } catch { + case e: DataAccessException + if Option(e.getCause) + .collect { case s: SQLException => s.getSQLState } + .contains("55P03") => + throw new WebApplicationException( + "Upload is already being finalized/aborted", + Response.Status.CONFLICT + ) + } + + if (session == null) { + throw new NotFoundException("Upload session not found or already finalized") + } + + val uploadId = session.getUploadId + val expectedParts = session.getNumPartsRequested + + val physicalAddr = Option(session.getPhysicalAddress).map(_.trim).getOrElse("") + if (physicalAddr.isEmpty) { + throw new WebApplicationException( + "Upload session is missing physicalAddress. Restart the upload.", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + + val total = DSL.count() + val done = + DSL + .count() + .filterWhere(MODEL_UPLOAD_SESSION_PART.ETAG.ne("")) + .as("done") + + val agg = ctx + .select(total.as("total"), done) + .from(MODEL_UPLOAD_SESSION_PART) + .where(MODEL_UPLOAD_SESSION_PART.UPLOAD_ID.eq(uploadId)) + .fetchOne() + + val totalCnt = agg.get("total", classOf[java.lang.Integer]).intValue() + val doneCnt = agg.get("done", classOf[java.lang.Integer]).intValue() + + if (totalCnt != expectedParts) { + throw new WebApplicationException( + s"Part table mismatch: expected $expectedParts rows but found $totalCnt. Restart the upload.", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + + if (doneCnt != expectedParts) { + val missing = ctx + .select(MODEL_UPLOAD_SESSION_PART.PART_NUMBER) + .from(MODEL_UPLOAD_SESSION_PART) + .where( + MODEL_UPLOAD_SESSION_PART.UPLOAD_ID + .eq(uploadId) + .and(MODEL_UPLOAD_SESSION_PART.ETAG.eq("")) + ) + .orderBy(MODEL_UPLOAD_SESSION_PART.PART_NUMBER.asc()) + .limit(50) + .fetch(MODEL_UPLOAD_SESSION_PART.PART_NUMBER) + .asScala + .toList + + throw new WebApplicationException( + s"Upload incomplete. Some missing ETags for parts are: ${missing.mkString(",")}", + Response.Status.CONFLICT + ) + } + + // Build partsList in order + val partsList: List[(Int, String)] = + ctx + .select(MODEL_UPLOAD_SESSION_PART.PART_NUMBER, MODEL_UPLOAD_SESSION_PART.ETAG) + .from(MODEL_UPLOAD_SESSION_PART) + .where(MODEL_UPLOAD_SESSION_PART.UPLOAD_ID.eq(uploadId)) + .orderBy(MODEL_UPLOAD_SESSION_PART.PART_NUMBER.asc()) + .fetch() + .asScala + .map(r => + ( + r.get(MODEL_UPLOAD_SESSION_PART.PART_NUMBER).intValue(), + r.get(MODEL_UPLOAD_SESSION_PART.ETAG) + ) + ) + .toList + + val objectStats = withLakeFSErrorHandling { + LakeFSStorageClient.completePresignedMultipartUploads( + model.getRepositoryName, + filePath, + uploadId, + partsList, + physicalAddr + ) + } + + // FINAL SERVER-SIDE SIZE CHECK (do not rely on init) + val actualSizeBytes = + Option(objectStats.getSizeBytes).map(_.longValue()).getOrElse(-1L) + + if (actualSizeBytes <= 0L) { + throw new WebApplicationException( + "lakeFS did not return sizeBytes for completed multipart upload", + Response.Status.INTERNAL_SERVER_ERROR + ) + } + + val maxBytes = singleFileUploadMaxBytes() + val tooLarge = actualSizeBytes > maxBytes + + if (tooLarge) { + try { + LakeFSStorageClient.resetObjectUploadOrDeletion(model.getRepositoryName, filePath) + } catch { + case _: Throwable => () + } + } + + // always cleanup session + ctx + .deleteFrom(MODEL_UPLOAD_SESSION) + .where( + MODEL_UPLOAD_SESSION.UID + .eq(uid) + .and(MODEL_UPLOAD_SESSION.MID.eq(mid)) + .and(MODEL_UPLOAD_SESSION.FILE_PATH.eq(filePath)) + ) + .execute() + + if (tooLarge) { + throw new WebApplicationException( + s"Upload exceeded max size: actualSizeBytes=$actualSizeBytes maxBytes=$maxBytes", + Response.Status.REQUEST_ENTITY_TOO_LARGE + ) + } + + Response + .ok( + Map( + "message" -> "Multipart upload completed successfully", + "filePath" -> objectStats.getPath + ) + ) + .build() + } + } + + private def abortMultipartUpload( + mid: Integer, + encodedFilePath: String, + uid: Int + ): Response = { + + val filePath = validateAndNormalizeFilePathOrThrow( + URLDecoder.decode(encodedFilePath, StandardCharsets.UTF_8.name()) + ) + + val (repoName, uploadId, physicalAddr) = withTransaction(context) { ctx => + if (!userHasWriteAccess(ctx, mid, uid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_MODEL_MESSAGE) + } + + val model = getModelByID(ctx, mid) + + val session = + try { + ctx + .selectFrom(MODEL_UPLOAD_SESSION) + .where( + MODEL_UPLOAD_SESSION.UID + .eq(uid) + .and(MODEL_UPLOAD_SESSION.MID.eq(mid)) + .and(MODEL_UPLOAD_SESSION.FILE_PATH.eq(filePath)) + ) + .forUpdate() + .noWait() + .fetchOne() + } catch { + case e: DataAccessException + if Option(e.getCause) + .collect { case s: SQLException => s.getSQLState } + .contains("55P03") => + throw new WebApplicationException( + "Upload is already being finalized/aborted", + Response.Status.CONFLICT + ) + } + + if (session == null) { + throw new NotFoundException("Upload session not found or already finalized") + } + + val physicalAddr = Option(session.getPhysicalAddress).map(_.trim).getOrElse("") + + // Delete session; parts removed via ON DELETE CASCADE + ctx + .deleteFrom(MODEL_UPLOAD_SESSION) + .where( + MODEL_UPLOAD_SESSION.UID + .eq(uid) + .and(MODEL_UPLOAD_SESSION.MID.eq(mid)) + .and(MODEL_UPLOAD_SESSION.FILE_PATH.eq(filePath)) + ) + .execute() + + (model.getRepositoryName, session.getUploadId, physicalAddr) + } + + withLakeFSErrorHandling { + LakeFSStorageClient.abortPresignedMultipartUploads(repoName, filePath, uploadId, physicalAddr) + } + + Response.ok(Map("message" -> "Multipart upload aborted successfully")).build() + } +} diff --git a/file-service/src/main/scala/org/apache/texera/service/type/dataset/DatasetFileNode.scala b/file-service/src/main/scala/org/apache/texera/service/type/dataset/DatasetFileNode.scala index 7c91d30b94a..6cb76f13bf6 100644 --- a/file-service/src/main/scala/org/apache/texera/service/type/dataset/DatasetFileNode.scala +++ b/file-service/src/main/scala/org/apache/texera/service/type/dataset/DatasetFileNode.scala @@ -20,15 +20,15 @@ package org.apache.texera.service.`type` import io.lakefs.clients.sdk.model.ObjectStats +import org.apache.texera.amber.core.storage.ResourceType import org.apache.texera.amber.core.storage.util.dataset.PhysicalFileNode import java.util import scala.collection.mutable // DatasetFileNode represents a unique file in dataset, its full path is in the format of: -// /ownerEmail/datasetName/versionName/fileRelativePath -// e.g. /bob@texera.com/twitterDataset/v1/california/irvine/tw1.csv -// ownerName is bob@texera.com; datasetName is twitterDataset, versionName is v1, fileRelativePath is california/irvine/tw1.csv +// /datasets/ownerEmail/datasetName/versionName/fileRelativePath +// e.g. /datasets/bob@texera.com/twitterDataset/v1/california/irvine/tw1.csv class DatasetFileNode( val name: String, // direct name of this node val nodeType: String, // "file" or "directory" @@ -81,6 +81,11 @@ object DatasetFileNode { ): List[DatasetFileNode] = { val rootNode = new DatasetFileNode("/", "directory", null, "") + // Root the tree at the datasets prefix node (a directory node named "datasets"). + val datasetsNode = + new DatasetFileNode(ResourceType.Datasets.toString, "directory", rootNode, "") + rootNode.children = Some(List(datasetsNode)) + // Owner level nodes map val ownerNodes = mutable.Map[String, DatasetFileNode]() @@ -88,8 +93,8 @@ object DatasetFileNode { case ((ownerEmail, datasetName, versionName), objects) => val ownerNode = ownerNodes.getOrElseUpdate( ownerEmail, { - val newNode = new DatasetFileNode(ownerEmail, "directory", rootNode, ownerEmail) - rootNode.children = Some(rootNode.getChildren :+ newNode) + val newNode = new DatasetFileNode(ownerEmail, "directory", datasetsNode, ownerEmail) + datasetsNode.children = Some(datasetsNode.getChildren :+ newNode) newNode } ) @@ -153,14 +158,19 @@ object DatasetFileNode { map: Map[(String, String, String), List[PhysicalFileNode]] ): List[DatasetFileNode] = { val rootNode = new DatasetFileNode("/", "directory", null, "") + + val datasetsNode = + new DatasetFileNode(ResourceType.Datasets.toString, "directory", rootNode, "") + rootNode.children = Some(List(datasetsNode)) + val ownerNodes = mutable.Map[String, DatasetFileNode]() map.foreach { case ((ownerEmail, datasetName, versionName), physicalNodes) => val ownerNode = ownerNodes.getOrElseUpdate( ownerEmail, { - val newNode = new DatasetFileNode(ownerEmail, "directory", rootNode, ownerEmail) - rootNode.children = Some(rootNode.getChildren :+ newNode) + val newNode = new DatasetFileNode(ownerEmail, "directory", datasetsNode, ownerEmail) + datasetsNode.children = Some(datasetsNode.getChildren :+ newNode) newNode } ) diff --git a/file-service/src/main/scala/org/apache/texera/service/util/ResourceUploadUtils.scala b/file-service/src/main/scala/org/apache/texera/service/util/ResourceUploadUtils.scala new file mode 100644 index 00000000000..a3103b263ff --- /dev/null +++ b/file-service/src/main/scala/org/apache/texera/service/util/ResourceUploadUtils.scala @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.service.util + +import jakarta.ws.rs.BadRequestException +import org.apache.commons.io.FilenameUtils + +import java.net.{HttpURLConnection, URL} + +/** + * Resource-agnostic helpers shared by the dataset and model upload flows. + */ +object ResourceUploadUtils { + + /** + * PUT exactly `len` bytes from `buf` to a presigned URL and return the ETag. + */ + def put(buf: Array[Byte], len: Int, url: String, partNum: Int): String = { + val conn = new URL(url).openConnection().asInstanceOf[HttpURLConnection] + conn.setDoOutput(true) + conn.setRequestMethod("PUT") + conn.setFixedLengthStreamingMode(len) + val out = conn.getOutputStream + out.write(buf, 0, len) + out.close() + + val code = conn.getResponseCode + if (code != HttpURLConnection.HTTP_OK && code != HttpURLConnection.HTTP_CREATED) + throw new RuntimeException(s"Part $partNum upload failed (HTTP $code)") + + val etag = conn.getHeaderField("ETag").replace("\"", "") + conn.disconnect() + etag + } + + /** + * Validates a file path using Apache Commons IO. Rejects empty paths, + * paths that traverse above the root, and absolute paths. + */ + def validateAndNormalizeFilePathOrThrow(path: String): String = { + if (path == null || path.trim.isEmpty) { + throw new BadRequestException("Path cannot be empty") + } + + val normalized = FilenameUtils.normalize(path, true) + if (normalized == null) { + throw new BadRequestException("Invalid path") + } + + if (FilenameUtils.getPrefixLength(normalized) > 0) { + throw new BadRequestException("Absolute paths not allowed") + } + normalized + } +} diff --git a/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourcePathSpec.scala b/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourcePathSpec.scala index 63bea046f3d..27a69247b2e 100644 --- a/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourcePathSpec.scala +++ b/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourcePathSpec.scala @@ -20,32 +20,33 @@ package org.apache.texera.service.resource import jakarta.ws.rs.BadRequestException +import org.apache.texera.service.util.ResourceUploadUtils import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -// Focused unit tests for the DatasetResource companion helper +// Focused unit tests for the shared ResourceUploadUtils helper // validateAndNormalizeFilePathOrThrow, which guards every upload/lookup path. -// These call the pure companion method directly and avoid the heavy +// These call the pure helper directly and avoid the heavy // DatasetResourceSpec integration harness (DB + LakeFS). class DatasetResourcePathSpec extends AnyFlatSpec with Matchers { "validateAndNormalizeFilePathOrThrow" should "reject a null path" in { val ex = intercept[BadRequestException] { - DatasetResource.validateAndNormalizeFilePathOrThrow(null) + ResourceUploadUtils.validateAndNormalizeFilePathOrThrow(null) } ex.getMessage shouldBe "Path cannot be empty" } it should "reject an empty path" in { val ex = intercept[BadRequestException] { - DatasetResource.validateAndNormalizeFilePathOrThrow("") + ResourceUploadUtils.validateAndNormalizeFilePathOrThrow("") } ex.getMessage shouldBe "Path cannot be empty" } it should "reject a whitespace-only path" in { val ex = intercept[BadRequestException] { - DatasetResource.validateAndNormalizeFilePathOrThrow(" ") + ResourceUploadUtils.validateAndNormalizeFilePathOrThrow(" ") } ex.getMessage shouldBe "Path cannot be empty" } @@ -54,26 +55,26 @@ class DatasetResourcePathSpec extends AnyFlatSpec with Matchers { // FilenameUtils.normalize returns null when the path traverses above the // root, e.g. a leading "../". val ex = intercept[BadRequestException] { - DatasetResource.validateAndNormalizeFilePathOrThrow("../secret.txt") + ResourceUploadUtils.validateAndNormalizeFilePathOrThrow("../secret.txt") } ex.getMessage shouldBe "Invalid path" } it should "reject an absolute path" in { val ex = intercept[BadRequestException] { - DatasetResource.validateAndNormalizeFilePathOrThrow("/etc/passwd") + ResourceUploadUtils.validateAndNormalizeFilePathOrThrow("/etc/passwd") } ex.getMessage shouldBe "Absolute paths not allowed" } it should "return a normalized relative path unchanged when already clean" in { - DatasetResource.validateAndNormalizeFilePathOrThrow( + ResourceUploadUtils.validateAndNormalizeFilePathOrThrow( "california/irvine/tw1.csv" ) shouldBe "california/irvine/tw1.csv" } it should "collapse interior '.' and '..' segments in a relative path" in { - DatasetResource.validateAndNormalizeFilePathOrThrow( + ResourceUploadUtils.validateAndNormalizeFilePathOrThrow( "a/./b/../c.csv" ) shouldBe "a/c.csv" } diff --git a/file-service/src/test/scala/org/apache/texera/service/resource/ModelAccessResourceSpec.scala b/file-service/src/test/scala/org/apache/texera/service/resource/ModelAccessResourceSpec.scala new file mode 100644 index 00000000000..5a50399a176 --- /dev/null +++ b/file-service/src/test/scala/org/apache/texera/service/resource/ModelAccessResourceSpec.scala @@ -0,0 +1,432 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.service.resource + +import jakarta.ws.rs.ForbiddenException +import org.apache.texera.auth.SessionUser +import org.apache.texera.dao.MockTexeraDB +import org.apache.texera.dao.jooq.generated.enums.{PrivilegeEnum, UserRoleEnum} +import org.apache.texera.dao.jooq.generated.tables.ModelUserAccess.MODEL_USER_ACCESS +import org.apache.texera.dao.jooq.generated.tables.daos.{ModelDao, ModelUserAccessDao, UserDao} +import org.apache.texera.dao.jooq.generated.tables.pojos.{Model, ModelUserAccess, User} +import org.apache.texera.service.resource.ModelAccessResource.{ + getModelUserAccessPrivilege, + getOwner, + isModelPublic, + userHasReadAccess, + userHasWriteAccess, + userOwnModel +} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} + +import scala.jdk.CollectionConverters._ + +class ModelAccessResourceSpec + extends AnyFlatSpec + with Matchers + with MockTexeraDB + with BeforeAndAfterAll + with BeforeAndAfterEach { + + private val ownerUser: User = { + val user = new User + user.setName("model_owner") + user.setPassword("123") + user.setEmail("model_owner@test.com") + user.setRole(UserRoleEnum.REGULAR) + user + } + + private val readGranteeUser: User = { + val user = new User + user.setName("read_grantee") + user.setPassword("123") + user.setEmail("read_grantee@test.com") + user.setRole(UserRoleEnum.REGULAR) + user + } + + private val writeGranteeUser: User = { + val user = new User + user.setName("write_grantee") + user.setPassword("123") + user.setEmail("write_grantee@test.com") + user.setRole(UserRoleEnum.REGULAR) + user + } + + private val strangerUser: User = { + val user = new User + user.setName("stranger") + user.setPassword("123") + user.setEmail("stranger@test.com") + user.setRole(UserRoleEnum.REGULAR) + user + } + + private val privateModel: Model = { + val model = new Model + model.setName("private-model") + model.setRepositoryName("private-model") + model.setIsPublic(false) + model.setIsDownloadable(true) + model.setDescription("private model for access tests") + model.setFramework("pytorch") + model + } + + private val publicModel: Model = { + val model = new Model + model.setName("public-model") + model.setRepositoryName("public-model") + model.setIsPublic(true) + model.setIsDownloadable(true) + model.setDescription("public model for access tests") + model.setFramework("pytorch") + model + } + + private val nonExistentMid: Integer = 999999 + + lazy val accessResource = new ModelAccessResource() + + lazy val ownerSession = new SessionUser(ownerUser) + lazy val writeGranteeSession = new SessionUser(writeGranteeUser) + lazy val readGranteeSession = new SessionUser(readGranteeUser) + lazy val strangerSession = new SessionUser(strangerUser) + + private def grantDirectly(mid: Integer, uid: Integer, privilege: PrivilegeEnum): Unit = { + new ModelUserAccessDao(getDSLContext.configuration()) + .insert(new ModelUserAccess(mid, uid, privilege)) + } + + private def accessList(mid: Integer): List[ModelAccessResource.AccessEntry] = + accessResource.getAccessList(mid).asScala.toList + + override protected def beforeAll(): Unit = { + super.beforeAll() + initializeDBAndReplaceDSLContext() + + val userDao = new UserDao(getDSLContext.configuration()) + userDao.insert(ownerUser) + userDao.insert(readGranteeUser) + userDao.insert(writeGranteeUser) + userDao.insert(strangerUser) + + privateModel.setOwnerUid(ownerUser.getUid) + publicModel.setOwnerUid(ownerUser.getUid) + val modelDao = new ModelDao(getDSLContext.configuration()) + modelDao.insert(privateModel) + modelDao.insert(publicModel) + } + + override protected def beforeEach(): Unit = { + super.beforeEach() + // every test starts with no explicit grants + getDSLContext.deleteFrom(MODEL_USER_ACCESS).execute() + } + + override protected def afterAll(): Unit = { + try shutdownDB() + finally super.afterAll() + } + + // =========================================================================== + // Privilege helpers + // =========================================================================== + + "isModelPublic" should "be true for a public model and false for a private one" in { + isModelPublic(getDSLContext, publicModel.getMid) shouldBe true + isModelPublic(getDSLContext, privateModel.getMid) shouldBe false + } + + "userOwnModel" should "be true only for the owner" in { + userOwnModel(getDSLContext, privateModel.getMid, ownerUser.getUid) shouldBe true + userOwnModel(getDSLContext, privateModel.getMid, strangerUser.getUid) shouldBe false + } + + "getModelUserAccessPrivilege" should "return NONE for a user without an explicit grant" in { + getModelUserAccessPrivilege( + getDSLContext, + privateModel.getMid, + strangerUser.getUid + ) shouldEqual PrivilegeEnum.NONE + } + + it should "return the granted privilege for a grantee" in { + grantDirectly(privateModel.getMid, readGranteeUser.getUid, PrivilegeEnum.READ) + grantDirectly(privateModel.getMid, writeGranteeUser.getUid, PrivilegeEnum.WRITE) + + getModelUserAccessPrivilege( + getDSLContext, + privateModel.getMid, + readGranteeUser.getUid + ) shouldEqual PrivilegeEnum.READ + getModelUserAccessPrivilege( + getDSLContext, + privateModel.getMid, + writeGranteeUser.getUid + ) shouldEqual PrivilegeEnum.WRITE + } + + "the owner" should "have both read and write access to the model" in { + userHasReadAccess(getDSLContext, privateModel.getMid, ownerUser.getUid) shouldBe true + userHasWriteAccess(getDSLContext, privateModel.getMid, ownerUser.getUid) shouldBe true + } + + "a READ grantee" should "have read but not write access" in { + grantDirectly(privateModel.getMid, readGranteeUser.getUid, PrivilegeEnum.READ) + + userHasReadAccess(getDSLContext, privateModel.getMid, readGranteeUser.getUid) shouldBe true + userHasWriteAccess(getDSLContext, privateModel.getMid, readGranteeUser.getUid) shouldBe false + } + + "a WRITE grantee" should "have both read and write access" in { + grantDirectly(privateModel.getMid, writeGranteeUser.getUid, PrivilegeEnum.WRITE) + + userHasReadAccess(getDSLContext, privateModel.getMid, writeGranteeUser.getUid) shouldBe true + userHasWriteAccess(getDSLContext, privateModel.getMid, writeGranteeUser.getUid) shouldBe true + } + + "a user with no grant" should "have no access to a private model" in { + userHasReadAccess(getDSLContext, privateModel.getMid, strangerUser.getUid) shouldBe false + userHasWriteAccess(getDSLContext, privateModel.getMid, strangerUser.getUid) shouldBe false + } + + it should "have read but not write access to a public model" in { + userHasReadAccess(getDSLContext, publicModel.getMid, strangerUser.getUid) shouldBe true + userHasWriteAccess(getDSLContext, publicModel.getMid, strangerUser.getUid) shouldBe false + } + + it should "have no explicit privilege row on a public model" in { + // public read access comes from is_public, not from a model_user_access row + getModelUserAccessPrivilege( + getDSLContext, + publicModel.getMid, + strangerUser.getUid + ) shouldEqual PrivilegeEnum.NONE + } + + "an explicit WRITE grant on a public model" should "give a non-owner write access" in { + grantDirectly(publicModel.getMid, writeGranteeUser.getUid, PrivilegeEnum.WRITE) + + userHasWriteAccess(getDSLContext, publicModel.getMid, writeGranteeUser.getUid) shouldBe true + } + + "the privilege helpers" should "treat a nonexistent model as private, unowned, and ungranted" in { + isModelPublic(getDSLContext, nonExistentMid) shouldBe false + userOwnModel(getDSLContext, nonExistentMid, ownerUser.getUid) shouldBe false + getModelUserAccessPrivilege( + getDSLContext, + nonExistentMid, + ownerUser.getUid + ) shouldEqual PrivilegeEnum.NONE + userHasReadAccess(getDSLContext, nonExistentMid, ownerUser.getUid) shouldBe false + userHasWriteAccess(getDSLContext, nonExistentMid, ownerUser.getUid) shouldBe false + } + + "getOwner" should "return the owning user" in { + getOwner(getDSLContext, privateModel.getMid).getEmail shouldEqual ownerUser.getEmail + } + + it should "return null for a nonexistent model" in { + getOwner(getDSLContext, nonExistentMid) shouldBe null + } + + // =========================================================================== + // grantAccess / getAccessList + // =========================================================================== + + "grantAccess" should "add a grantee that appears in the access list with the granted privilege" in { + val response = accessResource.grantAccess( + privateModel.getMid, + readGranteeUser.getEmail, + "READ", + ownerSession + ) + response.getStatus shouldEqual 200 + + val entries = accessList(privateModel.getMid) + entries should have size 1 + entries.head.email shouldEqual readGranteeUser.getEmail + entries.head.name shouldEqual readGranteeUser.getName + entries.head.privilege shouldEqual PrivilegeEnum.READ + } + + it should "update the privilege in place when re-granting with a different privilege" in { + accessResource.grantAccess( + privateModel.getMid, + readGranteeUser.getEmail, + "READ", + ownerSession + ) + accessResource.grantAccess( + privateModel.getMid, + readGranteeUser.getEmail, + "WRITE", + ownerSession + ) + + val entries = accessList(privateModel.getMid) + entries should have size 1 + entries.head.email shouldEqual readGranteeUser.getEmail + entries.head.privilege shouldEqual PrivilegeEnum.WRITE + } + + it should "allow a WRITE grantee to share the model" in { + grantDirectly(privateModel.getMid, writeGranteeUser.getUid, PrivilegeEnum.WRITE) + + val response = accessResource.grantAccess( + privateModel.getMid, + strangerUser.getEmail, + "READ", + writeGranteeSession + ) + response.getStatus shouldEqual 200 + + userHasReadAccess(getDSLContext, privateModel.getMid, strangerUser.getUid) shouldBe true + } + + it should "be forbidden for a user without write access" in { + val ex = intercept[ForbiddenException] { + accessResource.grantAccess( + privateModel.getMid, + readGranteeUser.getEmail, + "READ", + strangerSession + ) + } + ex.getResponse.getStatus shouldEqual 403 + ex.getMessage should include( + s"You do not have permission to modify model ${privateModel.getMid}" + ) + } + + it should "be forbidden for a READ grantee" in { + grantDirectly(privateModel.getMid, readGranteeUser.getUid, PrivilegeEnum.READ) + + assertThrows[ForbiddenException] { + accessResource.grantAccess( + privateModel.getMid, + strangerUser.getEmail, + "READ", + readGranteeSession + ) + } + } + + "getAccessList" should "return an empty list when no access has been granted" in { + accessList(privateModel.getMid) shouldBe empty + } + + it should "not include the owner's own access row" in { + // even if the owner somehow has an explicit access row, the list only shows other users + grantDirectly(privateModel.getMid, ownerUser.getUid, PrivilegeEnum.WRITE) + grantDirectly(privateModel.getMid, readGranteeUser.getUid, PrivilegeEnum.READ) + + val entries = accessList(privateModel.getMid) + entries should have size 1 + entries.head.email shouldEqual readGranteeUser.getEmail + } + + it should "list multiple grantees with their respective privileges" in { + grantDirectly(privateModel.getMid, readGranteeUser.getUid, PrivilegeEnum.READ) + grantDirectly(privateModel.getMid, writeGranteeUser.getUid, PrivilegeEnum.WRITE) + + val entries = accessList(privateModel.getMid) + entries should have size 2 + val privilegeByEmail = entries.map(entry => entry.email -> entry.privilege).toMap + privilegeByEmail(readGranteeUser.getEmail) shouldEqual PrivilegeEnum.READ + privilegeByEmail(writeGranteeUser.getEmail) shouldEqual PrivilegeEnum.WRITE + } + + // =========================================================================== + // revokeAccess + // =========================================================================== + + "revokeAccess" should "remove the grantee from the access list and drop their access" in { + grantDirectly(privateModel.getMid, readGranteeUser.getUid, PrivilegeEnum.READ) + + val response = accessResource.revokeAccess( + privateModel.getMid, + readGranteeUser.getEmail, + ownerSession + ) + response.getStatus shouldEqual 200 + + accessList(privateModel.getMid) shouldBe empty + getModelUserAccessPrivilege( + getDSLContext, + privateModel.getMid, + readGranteeUser.getUid + ) shouldEqual PrivilegeEnum.NONE + userHasReadAccess(getDSLContext, privateModel.getMid, readGranteeUser.getUid) shouldBe false + } + + it should "allow a WRITE grantee to revoke another user's access" in { + grantDirectly(privateModel.getMid, writeGranteeUser.getUid, PrivilegeEnum.WRITE) + grantDirectly(privateModel.getMid, readGranteeUser.getUid, PrivilegeEnum.READ) + + val response = accessResource.revokeAccess( + privateModel.getMid, + readGranteeUser.getEmail, + writeGranteeSession + ) + response.getStatus shouldEqual 200 + + userHasReadAccess(getDSLContext, privateModel.getMid, readGranteeUser.getUid) shouldBe false + } + + it should "succeed as a no-op when the target user has no explicit grant" in { + val response = accessResource.revokeAccess( + privateModel.getMid, + strangerUser.getEmail, + ownerSession + ) + response.getStatus shouldEqual 200 + accessList(privateModel.getMid) shouldBe empty + } + + it should "be forbidden for a user without write access" in { + grantDirectly(privateModel.getMid, readGranteeUser.getUid, PrivilegeEnum.READ) + + assertThrows[ForbiddenException] { + accessResource.revokeAccess( + privateModel.getMid, + readGranteeUser.getEmail, + strangerSession + ) + } + } + + // =========================================================================== + // getOwnerEmailOfModel + // =========================================================================== + + "getOwnerEmailOfModel" should "return the owner's email" in { + accessResource.getOwnerEmailOfModel(privateModel.getMid) shouldEqual ownerUser.getEmail + } + + it should "return an empty string for a nonexistent model" in { + accessResource.getOwnerEmailOfModel(nonExistentMid) shouldEqual "" + } +} diff --git a/file-service/src/test/scala/org/apache/texera/service/resource/ModelResourceSpec.scala b/file-service/src/test/scala/org/apache/texera/service/resource/ModelResourceSpec.scala new file mode 100644 index 00000000000..f3c737d2e17 --- /dev/null +++ b/file-service/src/test/scala/org/apache/texera/service/resource/ModelResourceSpec.scala @@ -0,0 +1,516 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.service.resource + +import jakarta.ws.rs._ +import org.apache.texera.auth.SessionUser +import org.apache.texera.dao.MockTexeraDB +import org.apache.texera.dao.jooq.generated.enums.{PrivilegeEnum, UserRoleEnum} +import org.apache.texera.dao.jooq.generated.tables.daos.{ModelDao, UserDao} +import org.apache.texera.dao.jooq.generated.tables.pojos.{Model, User} +import org.apache.texera.service.MockLakeFS +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} + +class ModelResourceSpec + extends AnyFlatSpec + with Matchers + with MockTexeraDB + with MockLakeFS + with BeforeAndAfterAll + with BeforeAndAfterEach { + + private val ownerUser: User = { + val user = new User + user.setName("model_user") + user.setPassword("123") + user.setEmail("model_user@test.com") + user.setRole(UserRoleEnum.ADMIN) + user + } + + private val otherUser: User = { + val user = new User + user.setName("model_user2") + user.setPassword("123") + user.setEmail("model_user2@test.com") + user.setRole(UserRoleEnum.ADMIN) + user + } + + private val baseModel: Model = { + val model = new Model + model.setName("test-model") + model.setRepositoryName("test-model") + model.setIsPublic(true) + model.setIsDownloadable(true) + model.setDescription("model for test") + model.setFramework("pytorch") + model + } + + private lazy val modelDao = new ModelDao(getDSLContext.configuration()) + + lazy val modelResource = new ModelResource() + + lazy val sessionUser = new SessionUser(ownerUser) + lazy val sessionUser2 = new SessionUser(otherUser) + + private def assertStatus(ex: WebApplicationException, status: Int): Unit = + ex.getResponse.getStatus shouldEqual status + + override protected def beforeAll(): Unit = { + super.beforeAll() + + initializeDBAndReplaceDSLContext() + + val userDao = new UserDao(getDSLContext.configuration()) + userDao.insert(ownerUser) + userDao.insert(otherUser) + + baseModel.setOwnerUid(ownerUser.getUid) + modelDao.insert(baseModel) + } + + override protected def afterAll(): Unit = { + try shutdownDB() + finally super.afterAll() + } + + // =========================================================================== + // createModel + // =========================================================================== + "createModel" should "create a model successfully if the user has no model with the same name" in { + val request = ModelResource.CreateModelRequest( + modelName = "new-model", + modelDescription = "description for new model", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = "torchscript" + ) + + val created = modelResource.createModel(request, sessionUser) + created.model.getName shouldEqual "new-model" + created.model.getDescription shouldEqual "description for new model" + created.model.getIsPublic shouldBe false + created.model.getIsDownloadable shouldBe true + created.model.getFramework shouldEqual "pytorch" + created.model.getFormat shouldEqual "torchscript" + // the LakeFS repository is named after the created model's id + created.model.getRepositoryName shouldEqual s"model-${created.model.getMid}" + } + + it should "default the framework to pytorch when none is provided" in { + val request = ModelResource.CreateModelRequest( + modelName = "framework-default-model", + modelDescription = "no framework provided", + isModelPublic = false, + isModelDownloadable = true, + framework = "", + format = null + ) + + val created = modelResource.createModel(request, sessionUser) + created.model.getFramework shouldEqual "pytorch" + } + + it should "refuse to create a model if the user already has one with the same name" in { + val request = ModelResource.CreateModelRequest( + modelName = "test-model", + modelDescription = "duplicate name", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ) + + assertThrows[BadRequestException] { + modelResource.createModel(request, sessionUser) + } + } + + it should "create a model successfully if another user has one with the same name" in { + val request = ModelResource.CreateModelRequest( + modelName = "test-model", + modelDescription = "same name, different owner", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ) + + val created = modelResource.createModel(request, sessionUser2) + created.model.getName shouldEqual "test-model" + } + + it should "reject an invalid model name" in { + val request = ModelResource.CreateModelRequest( + modelName = "bad name!", + modelDescription = "invalid name", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ) + + assertThrows[BadRequestException] { + modelResource.createModel(request, sessionUser) + } + } + + it should "return a DashboardModel with owner email, WRITE privilege, isOwner=true and size 0" in { + val request = ModelResource.CreateModelRequest( + modelName = "dashboard-model", + modelDescription = "dashboard properties", + isModelPublic = true, + isModelDownloadable = false, + framework = "pytorch", + format = null + ) + + val dashboard = modelResource.createModel(request, sessionUser) + dashboard.ownerEmail shouldEqual ownerUser.getEmail + dashboard.accessPrivilege shouldEqual PrivilegeEnum.WRITE + dashboard.isOwner shouldBe true + dashboard.size shouldEqual 0 + } + + // =========================================================================== + // getModel / listModels + // =========================================================================== + "getModel" should "return the dashboard model including its LakeFS repository size" in { + val request = ModelResource.CreateModelRequest( + modelName = "get-model", + modelDescription = "for get", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ) + val created = modelResource.createModel(request, sessionUser) + + val dashboard = modelResource.getModel(created.model.getMid, sessionUser) + dashboard.model.getMid shouldEqual created.model.getMid + dashboard.size should be >= 0L + } + + it should "forbid a stranger from getting a private model" in { + val request = ModelResource.CreateModelRequest( + modelName = "private-get-model", + modelDescription = "private", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ) + val created = modelResource.createModel(request, sessionUser) + + assertThrows[ForbiddenException] { + modelResource.getModel(created.model.getMid, sessionUser2) + } + } + + "listModels" should "include models the user owns" in { + val request = ModelResource.CreateModelRequest( + modelName = "listed-model", + modelDescription = "for list", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ) + val created = modelResource.createModel(request, sessionUser) + + val listed = modelResource.listModels(sessionUser) + listed.map(_.model.getMid) should contain(created.model.getMid) + } + + // =========================================================================== + // deleteModel + // =========================================================================== + "deleteModel" should "delete a model successfully if the user owns it" in { + val request = ModelResource.CreateModelRequest( + modelName = "delete-model", + modelDescription = "for delete", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ) + val created = modelResource.createModel(request, sessionUser) + + val response = modelResource.deleteModel(created.model.getMid, sessionUser) + response.getStatus shouldEqual 200 + modelDao.fetchOneByMid(created.model.getMid) shouldBe null + } + + it should "refuse to delete a model not owned by the user" in { + val request = ModelResource.CreateModelRequest( + modelName = "forbidden-delete-model", + modelDescription = "for forbidden delete", + isModelPublic = true, + isModelDownloadable = true, + framework = "pytorch", + format = null + ) + val created = modelResource.createModel(request, sessionUser) + + assertThrows[ForbiddenException] { + modelResource.deleteModel(created.model.getMid, sessionUser2) + } + modelDao.fetchOneByMid(created.model.getMid) should not be null + } + + it should "surface a LakeFS 404 as NotFoundException when deleting a model whose repo is missing" in { + val model = new Model + model.setName("delete-model-no-repo") + model.setRepositoryName("delete-model-no-repo") + model.setDescription("for lakefs 404 mapping test") + model.setOwnerUid(ownerUser.getUid) + model.setIsPublic(true) + model.setIsDownloadable(true) + model.setFramework("pytorch") + modelDao.insert(model) + // intentionally no repo created in LakeFS + + val ex = intercept[NotFoundException] { + modelResource.deleteModel(model.getMid, sessionUser) + } + assertStatus(ex, 404) + } + + // =========================================================================== + // update name / description / publicity / downloadable + // =========================================================================== + "updateModelName" should "rename a model the user can write to" in { + val created = modelResource.createModel( + ModelResource.CreateModelRequest( + modelName = "rename-me", + modelDescription = "d", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ), + sessionUser + ) + + val response = modelResource.updateModelName( + ModelResource.ModelNameModification(created.model.getMid, "renamed"), + sessionUser + ) + response.getStatus shouldEqual 200 + modelDao.fetchOneByMid(created.model.getMid).getName shouldEqual "renamed" + } + + "updateModelDescription" should "update the description of a model the user can write to" in { + val created = modelResource.createModel( + ModelResource.CreateModelRequest( + modelName = "describe-me", + modelDescription = "old", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ), + sessionUser + ) + + val response = modelResource.updateModelDescription( + ModelResource.ModelDescriptionModification(created.model.getMid, "new description"), + sessionUser + ) + response.getStatus shouldEqual 200 + modelDao.fetchOneByMid(created.model.getMid).getDescription shouldEqual "new description" + } + + "toggleModelPublicity" should "flip the public flag for a writer" in { + val created = modelResource.createModel( + ModelResource.CreateModelRequest( + modelName = "publicity-model", + modelDescription = "d", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ), + sessionUser + ) + + modelResource.toggleModelPublicity(created.model.getMid, sessionUser).getStatus shouldEqual 200 + modelDao.fetchOneByMid(created.model.getMid).getIsPublic shouldBe true + } + + "toggleModelDownloadable" should "flip the downloadable flag for the owner" in { + val created = modelResource.createModel( + ModelResource.CreateModelRequest( + modelName = "downloadable-model", + modelDescription = "d", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ), + sessionUser + ) + + modelResource + .toggleModelDownloadable(created.model.getMid, sessionUser) + .getStatus shouldEqual 200 + modelDao.fetchOneByMid(created.model.getMid).getIsDownloadable shouldBe false + } + + it should "forbid a non-owner from toggling downloadable" in { + val created = modelResource.createModel( + ModelResource.CreateModelRequest( + modelName = "downloadable-forbidden-model", + modelDescription = "d", + isModelPublic = true, + isModelDownloadable = true, + framework = "pytorch", + format = null + ), + sessionUser + ) + + assertThrows[ForbiddenException] { + modelResource.toggleModelDownloadable(created.model.getMid, sessionUser2) + } + } + + it should "refuse to rename a model to a name the owner already uses" in { + modelResource.createModel( + ModelResource.CreateModelRequest( + modelName = "dup-target", + modelDescription = "d", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ), + sessionUser + ) + val second = modelResource.createModel( + ModelResource.CreateModelRequest( + modelName = "dup-source", + modelDescription = "d", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ), + sessionUser + ) + + assertThrows[BadRequestException] { + modelResource.updateModelName( + ModelResource.ModelNameModification(second.model.getMid, "dup-target"), + sessionUser + ) + } + } + + it should "forbid a user without write access from renaming or re-describing a model" in { + val created = modelResource.createModel( + ModelResource.CreateModelRequest( + modelName = "no-write-updates", + modelDescription = "d", + isModelPublic = true, + isModelDownloadable = true, + framework = "pytorch", + format = null + ), + sessionUser + ) + + assertThrows[ForbiddenException] { + modelResource.updateModelName( + ModelResource.ModelNameModification(created.model.getMid, "hijacked"), + sessionUser2 + ) + } + assertThrows[ForbiddenException] { + modelResource.updateModelDescription( + ModelResource.ModelDescriptionModification(created.model.getMid, "hijacked"), + sessionUser2 + ) + } + } + + // =========================================================================== + // getPublicModel / listModels public merge + // =========================================================================== + "getPublicModel" should "return a public model without authentication" in { + val created = modelResource.createModel( + ModelResource.CreateModelRequest( + modelName = "public-get-model", + modelDescription = "d", + isModelPublic = true, + isModelDownloadable = true, + framework = "pytorch", + format = null + ), + sessionUser + ) + + val dashboard = modelResource.getPublicModel(created.model.getMid) + dashboard.model.getMid shouldEqual created.model.getMid + } + + it should "forbid access to a private model" in { + val created = modelResource.createModel( + ModelResource.CreateModelRequest( + modelName = "public-get-private-model", + modelDescription = "d", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ), + sessionUser + ) + + assertThrows[ForbiddenException] { + modelResource.getPublicModel(created.model.getMid) + } + } + + "listModels" should "include public models owned by another user" in { + val othersPublic = modelResource.createModel( + ModelResource.CreateModelRequest( + modelName = "others-public-model", + modelDescription = "d", + isModelPublic = true, + isModelDownloadable = true, + framework = "pytorch", + format = null + ), + sessionUser2 + ) + + val listed = modelResource.listModels(sessionUser) + val entry = listed.find(_.model.getMid == othersPublic.model.getMid) + entry should not be empty + entry.get.isOwner shouldBe false + entry.get.accessPrivilege shouldEqual PrivilegeEnum.READ + } +} diff --git a/file-service/src/test/scala/org/apache/texera/service/resource/ModelUploadResourceSpec.scala b/file-service/src/test/scala/org/apache/texera/service/resource/ModelUploadResourceSpec.scala new file mode 100644 index 00000000000..136e604b4de --- /dev/null +++ b/file-service/src/test/scala/org/apache/texera/service/resource/ModelUploadResourceSpec.scala @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.service.resource + +import jakarta.ws.rs._ +import jakarta.ws.rs.core._ +import org.apache.texera.auth.SessionUser +import org.apache.texera.dao.MockTexeraDB +import org.apache.texera.dao.jooq.generated.enums.UserRoleEnum +import org.apache.texera.dao.jooq.generated.tables.daos.UserDao +import org.apache.texera.dao.jooq.generated.tables.pojos.User +import org.apache.texera.service.MockLakeFS +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} + +import java.io.ByteArrayInputStream +import java.net.URLEncoder +import java.nio.charset.StandardCharsets +import java.util.{Collections, Date, Locale, Optional} +import scala.util.Random + +class ModelUploadResourceSpec + extends AnyFlatSpec + with Matchers + with MockTexeraDB + with MockLakeFS + with BeforeAndAfterAll + with BeforeAndAfterEach { + + private val ownerUser: User = { + val user = new User + user.setName("model_upload_user") + user.setPassword("123") + user.setEmail("model_upload_user@test.com") + user.setRole(UserRoleEnum.ADMIN) + user + } + + lazy val modelResource = new ModelResource() + lazy val sessionUser = new SessionUser(ownerUser) + + override protected def beforeAll(): Unit = { + super.beforeAll() + initializeDBAndReplaceDSLContext() + new UserDao(getDSLContext.configuration()).insert(ownerUser) + } + + override protected def afterAll(): Unit = { + try shutdownDB() + finally super.afterAll() + } + + // ---------- helpers ---------- + private def urlEnc(raw: String): String = + URLEncoder.encode(raw, StandardCharsets.UTF_8.name()) + + private def uniqueName(prefix: String): String = + s"$prefix-${System.nanoTime()}-${Random.alphanumeric.take(6).mkString.toLowerCase}" + + /** Minimal HttpHeaders exposing only Content-Length, which the upload paths read. */ + private def mkHeaders(contentLength: Long): HttpHeaders = + new HttpHeaders { + private val headers = new MultivaluedHashMap[String, String]() + headers.putSingle(HttpHeaders.CONTENT_LENGTH, contentLength.toString) + override def getHeaderString(name: String): String = headers.getFirst(name) + override def getRequestHeaders: MultivaluedMap[String, String] = headers + override def getRequestHeader(name: String): java.util.List[String] = + Option(headers.get(name)).getOrElse(Collections.emptyList[String]()) + override def getAcceptableMediaTypes: java.util.List[MediaType] = Collections.emptyList() + override def getAcceptableLanguages: java.util.List[Locale] = Collections.emptyList() + override def getMediaType: MediaType = null + override def getLanguage: Locale = null + override def getCookies: java.util.Map[String, Cookie] = Collections.emptyMap() + override def getDate: Date = null + override def getLength: Int = contentLength.toInt + } + + /** Creates a fresh model (provisions its LakeFS repo) and returns it. */ + private def newModel(): ModelResource.DashboardModel = + modelResource.createModel( + ModelResource.CreateModelRequest( + modelName = uniqueName("upload-model"), + modelDescription = "for upload tests", + isModelPublic = false, + isModelDownloadable = true, + framework = "pytorch", + format = null + ), + sessionUser + ) + + private def uploadOneShot(mid: Integer, path: String, bytes: Array[Byte]): Response = + modelResource.uploadOneFileToModel( + mid, + urlEnc(path), + "upload", + new ByteArrayInputStream(bytes), + mkHeaders(bytes.length.toLong), + sessionUser + ) + + // =========================================================================== + // One-shot upload + version lifecycle + // =========================================================================== + "uploadOneFileToModel + createModelVersion" should "commit an uploaded .pt file into a version" in { + val model = newModel() + val mid = model.model.getMid + + uploadOneShot(mid, "model.pt", Array.fill[Byte](2048)(0x5a)).getStatus shouldEqual 200 + + val version = modelResource.createModelVersion("initial", mid, sessionUser) + version.modelVersion.getName should startWith("v1") + + modelResource.getModelVersionList(mid, sessionUser) should have size 1 + + val latest = modelResource.retrieveLatestModelVersion(mid, sessionUser) + latest.fileNodes.map(_.getName) should contain("model.pt") + + val roots = + modelResource.retrieveModelVersionRootFileNodes( + mid, + version.modelVersion.getMvid, + sessionUser + ) + roots.fileNodes.map(_.getName) should contain("model.pt") + roots.size should be > 0L + } + + it should "accept a .pth extension as well" in { + val model = newModel() + uploadOneShot( + model.model.getMid, + "weights.pth", + Array.fill[Byte](1024)(0x1) + ).getStatus shouldEqual 200 + } + + "createModelVersion" should "reject a version when there are no staged changes" in { + val model = newModel() + val ex = intercept[WebApplicationException] { + modelResource.createModelVersion("empty", model.model.getMid, sessionUser) + } + ex.getResponse.getStatus shouldEqual 400 + } + + "deleteModelFile" should "remove a staged file" in { + val model = newModel() + val mid = model.model.getMid + uploadOneShot(mid, "scratch.pt", Array.fill[Byte](512)(0x2)).getStatus shouldEqual 200 + modelResource.deleteModelFile(mid, urlEnc("scratch.pt"), sessionUser).getStatus shouldEqual 200 + } + + // =========================================================================== + // No per-file type restriction: a model is a folder of files + // =========================================================================== + "uploadOneFileToModel" should "accept companion files alongside weights and commit them together" in { + val model = newModel() + val mid = model.model.getMid + + // a typical model folder: weights plus config/tokenizer companions + uploadOneShot(mid, "model.pt", Array.fill[Byte](256)(0x5)).getStatus shouldEqual 200 + uploadOneShot( + mid, + "config.json", + "{\"hidden\":8}".getBytes(StandardCharsets.UTF_8) + ).getStatus shouldEqual 200 + uploadOneShot(mid, "tokenizer.txt", Array.fill[Byte](32)(0x4)).getStatus shouldEqual 200 + + val version = modelResource.createModelVersion("folder", mid, sessionUser) + version.fileNodes.nonEmpty shouldBe true + + val names = modelResource.retrieveLatestModelVersion(mid, sessionUser).fileNodes.map(_.getName) + names should contain allOf ("model.pt", "config.json", "tokenizer.txt") + } + + it should "preserve a nested folder structure in the committed version tree" in { + val model = newModel() + val mid = model.model.getMid + + // a HuggingFace-style layout: files inside subdirectories + uploadOneShot(mid, "pytorch_model.bin", Array.fill[Byte](128)(0x6)).getStatus shouldEqual 200 + uploadOneShot( + mid, + "tokenizer/vocab.txt", + Array.fill[Byte](64)(0x7) + ).getStatus shouldEqual 200 + uploadOneShot( + mid, + "shards/part-00001/data.bin", + Array.fill[Byte](64)(0x8) + ).getStatus shouldEqual 200 + + modelResource.createModelVersion("nested", mid, sessionUser) + + val roots = modelResource.retrieveLatestModelVersion(mid, sessionUser).fileNodes + roots.map(_.getName) should contain allOf ("pytorch_model.bin", "tokenizer", "shards") + + // directories are preserved as directory nodes holding their children + val tokenizerDir = roots.find(_.getName == "tokenizer").get + tokenizerDir.getNodeType shouldEqual "directory" + tokenizerDir.getChildren.map(_.getName) should contain("vocab.txt") + + // nesting is recursive, not flattened to one level + val shardsDir = roots.find(_.getName == "shards").get + val partDir = shardsDir.getChildren.find(_.getName == "part-00001").get + partDir.getNodeType shouldEqual "directory" + partDir.getChildren.map(_.getName) should contain("data.bin") + } + + // =========================================================================== + // Version semantics: each version is a full snapshot, not a delta + // =========================================================================== + "a later version" should "carry over untouched files and only replace the re-uploaded one" in { + val model = newModel() + val mid = model.model.getMid + + // v1: four files, with b at a known size + uploadOneShot(mid, "a.pt", Array.fill[Byte](100)(0x1)).getStatus shouldEqual 200 + uploadOneShot(mid, "b.pt", Array.fill[Byte](200)(0x2)).getStatus shouldEqual 200 + uploadOneShot(mid, "c.pt", Array.fill[Byte](300)(0x3)).getStatus shouldEqual 200 + uploadOneShot(mid, "d.pt", Array.fill[Byte](400)(0x4)).getStatus shouldEqual 200 + val v1 = modelResource.createModelVersion("first", mid, sessionUser) + + // v2: re-upload ONLY b, with a different size so the two revisions are distinguishable + uploadOneShot(mid, "b.pt", Array.fill[Byte](999)(0x9)).getStatus shouldEqual 200 + val v2 = modelResource.createModelVersion("second", mid, sessionUser) + + def nodesOf(mvid: Integer) = + modelResource.retrieveModelVersionRootFileNodes(mid, mvid, sessionUser).fileNodes + def sizeOf(mvid: Integer, name: String) = + nodesOf(mvid).find(_.getName == name).flatMap(_.getSize) + + // v2 still contains all four files: a, c, d carried over untouched, b replaced + nodesOf(v2.modelVersion.getMvid) + .map(_.getName) should contain allOf ("a.pt", "b.pt", "c.pt", "d.pt") + sizeOf(v2.modelVersion.getMvid, "a.pt") shouldEqual Some(100L) + sizeOf(v2.modelVersion.getMvid, "c.pt") shouldEqual Some(300L) + sizeOf(v2.modelVersion.getMvid, "d.pt") shouldEqual Some(400L) + sizeOf(v2.modelVersion.getMvid, "b.pt") shouldEqual Some(999L) + + // v1 is immutable: it still sees the ORIGINAL b + sizeOf(v1.modelVersion.getMvid, "b.pt") shouldEqual Some(200L) + + // both versions are listed, newest first + modelResource.getModelVersionList(mid, sessionUser).map(_.getName) should have size 2 + } + + // =========================================================================== + // Session-based multipart upload (single part) + // =========================================================================== + "the multipart flow" should "init, upload a part, finish, and be committable as a version" in { + val model = newModel() + val mid = model.model.getMid + val ownerEmail = ownerUser.getEmail + val modelName = model.model.getName + val filePath = "multipart-model.pt" + val payload = Array.fill[Byte](16)(0x7) + val partSize = 8L * 1024L * 1024L + + // init -> one part expected + val initResp = modelResource.multipartUpload( + "init", + ownerEmail, + modelName, + urlEnc(filePath), + Optional.of(java.lang.Long.valueOf(payload.length.toLong)), + Optional.of(java.lang.Long.valueOf(partSize)), + Optional.empty(), + sessionUser + ) + initResp.getStatus shouldEqual 200 + + // upload the single part + val partResp = modelResource.uploadPart( + ownerEmail, + modelName, + urlEnc(filePath), + 1, + new ByteArrayInputStream(payload), + mkHeaders(payload.length.toLong), + sessionUser + ) + partResp.getStatus shouldEqual 200 + + // finish + val finishResp = modelResource.multipartUpload( + "finish", + ownerEmail, + modelName, + urlEnc(filePath), + Optional.empty(), + Optional.empty(), + Optional.empty(), + sessionUser + ) + finishResp.getStatus shouldEqual 200 + + // the finished file is now staged and can be committed as a version + val version = modelResource.createModelVersion("from-multipart", mid, sessionUser) + version.fileNodes.nonEmpty shouldBe true + modelResource + .retrieveLatestModelVersion(mid, sessionUser) + .fileNodes + .map(_.getName) should contain(filePath) + } + + it should "abort an initiated upload" in { + val model = newModel() + val ownerEmail = ownerUser.getEmail + val modelName = model.model.getName + val filePath = "abort-model.pt" + + modelResource + .multipartUpload( + "init", + ownerEmail, + modelName, + urlEnc(filePath), + Optional.of(java.lang.Long.valueOf(16L)), + Optional.of(java.lang.Long.valueOf(8L * 1024L * 1024L)), + Optional.empty(), + sessionUser + ) + .getStatus shouldEqual 200 + + modelResource + .multipartUpload( + "abort", + ownerEmail, + modelName, + urlEnc(filePath), + Optional.empty(), + Optional.empty(), + Optional.empty(), + sessionUser + ) + .getStatus shouldEqual 200 + } +} diff --git a/file-service/src/test/scala/org/apache/texera/service/type/dataset/DatasetFileNodeSpec.scala b/file-service/src/test/scala/org/apache/texera/service/type/dataset/DatasetFileNodeSpec.scala index 93e08a9afc1..c47675dd308 100644 --- a/file-service/src/test/scala/org/apache/texera/service/type/dataset/DatasetFileNodeSpec.scala +++ b/file-service/src/test/scala/org/apache/texera/service/type/dataset/DatasetFileNodeSpec.scala @@ -105,10 +105,13 @@ class DatasetFileNodeSpec extends AnyFlatSpec with Matchers { Map(("bob@texera.com", "twitter", "v1") -> objects) ) - // One owner root. + // The tree is rooted at a single "datasets" prefix node; owners nest under it. roots should have size 1 - val ownerNode = roots.head - ownerNode.getName shouldBe "bob@texera.com" + val datasetsNode = roots.head + datasetsNode.getName shouldBe "datasets" + datasetsNode.getNodeType shouldBe "directory" + + val ownerNode = datasetsNode.getChildren.find(_.getName == "bob@texera.com").get ownerNode.getNodeType shouldBe "directory" val datasetNode = ownerNode.getChildren.find(_.getName == "twitter").get @@ -126,7 +129,7 @@ class DatasetFileNodeSpec extends AnyFlatSpec with Matchers { val file1 = bDir.getChildren.find(_.getName == "1.csv").get file1.getNodeType shouldBe "file" file1.getSize shouldBe Some(2L) - file1.getFilePath shouldBe "/bob@texera.com/twitter/v1/b/1.csv" + file1.getFilePath shouldBe "/datasets/bob@texera.com/twitter/v1/b/1.csv" // Total size equals the sum of the three files. DatasetFileNode.calculateTotalSize(roots) shouldBe 6L @@ -152,7 +155,10 @@ class DatasetFileNodeSpec extends AnyFlatSpec with Matchers { ) roots should have size 1 - val versionNode = roots.head.getChildren + val versionNode = roots.head.getChildren // "datasets" prefix node + .find(_.getName == "alice@texera.com") + .get + .getChildren .find(_.getName == "ds") .get .getChildren @@ -165,7 +171,7 @@ class DatasetFileNodeSpec extends AnyFlatSpec with Matchers { val leaf = dirNode.getChildren.find(_.getName == "a.csv").get leaf.getNodeType shouldBe "file" leaf.getSize shouldBe Some(5L) - leaf.getFilePath shouldBe "/alice@texera.com/ds/v1/dir1/a.csv" + leaf.getFilePath shouldBe "/datasets/alice@texera.com/ds/v1/dir1/a.csv" DatasetFileNode.calculateTotalSize(roots) shouldBe 5L } finally { diff --git a/frontend/src/app/common/type/dataset-file.spec.ts b/frontend/src/app/common/type/dataset-file.spec.ts deleted file mode 100644 index 76acdfe9144..00000000000 --- a/frontend/src/app/common/type/dataset-file.spec.ts +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { DatasetFile, parseDatasetFileToFilePath, parseFilePathToDatasetFile } from "./dataset-file"; - -describe("parseFilePathToDatasetFile", () => { - it("parses owner, dataset, version, and single-segment relative path", () => { - const result = parseFilePathToDatasetFile("/bob@texera.com/twitterDataset/v1/tw1.csv"); - expect(result).toEqual({ - ownerEmail: "bob@texera.com", - datasetName: "twitterDataset", - versionName: "v1", - fileRelativePath: "tw1.csv", - }); - }); - - it("joins remaining segments into a nested relative path", () => { - const result = parseFilePathToDatasetFile("/bob@texera.com/twitterDataset/v1/california/irvine/tw1.csv"); - expect(result.ownerEmail).toBe("bob@texera.com"); - expect(result.datasetName).toBe("twitterDataset"); - expect(result.versionName).toBe("v1"); - expect(result.fileRelativePath).toBe("california/irvine/tw1.csv"); - }); - - it("ignores empty segments from leading, trailing, and duplicate slashes", () => { - const result = parseFilePathToDatasetFile("//bob@texera.com//twitterDataset/v1/dir//file.csv/"); - expect(result).toEqual({ - ownerEmail: "bob@texera.com", - datasetName: "twitterDataset", - versionName: "v1", - fileRelativePath: "dir/file.csv", - }); - }); - - it("throws when there are fewer than four path segments", () => { - expect(() => parseFilePathToDatasetFile("/bob@texera.com/twitterDataset/v1")).toThrow("Invalid file path format"); - expect(() => parseFilePathToDatasetFile("")).toThrow("Invalid file path format"); - expect(() => parseFilePathToDatasetFile("/just/three/parts")).toThrow("Invalid file path format"); - }); -}); - -describe("parseDatasetFileToFilePath", () => { - it("assembles a slash-delimited path with a leading slash", () => { - const datasetFile: DatasetFile = { - ownerEmail: "bob@texera.com", - datasetName: "twitterDataset", - versionName: "v1", - fileRelativePath: "california/irvine/tw1.csv", - }; - expect(parseDatasetFileToFilePath(datasetFile)).toBe("/bob@texera.com/twitterDataset/v1/california/irvine/tw1.csv"); - }); -}); - -describe("dataset-file round trips", () => { - it("path -> DatasetFile -> path is stable for a canonical path", () => { - const path = "/bob@texera.com/twitterDataset/v1/california/irvine/tw1.csv"; - expect(parseDatasetFileToFilePath(parseFilePathToDatasetFile(path))).toBe(path); - }); - - it("DatasetFile -> path -> DatasetFile is stable for a canonical object", () => { - const datasetFile: DatasetFile = { - ownerEmail: "alice@texera.com", - datasetName: "sensorData", - versionName: "v42", - fileRelativePath: "2026/reading.json", - }; - expect(parseFilePathToDatasetFile(parseDatasetFileToFilePath(datasetFile))).toEqual(datasetFile); - }); -}); diff --git a/frontend/src/app/common/type/dataset-file.ts b/frontend/src/app/common/type/dataset-file.ts deleted file mode 100644 index 5fe561d3720..00000000000 --- a/frontend/src/app/common/type/dataset-file.ts +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -// user given filePath is /ownerEmail/datasetName/versionName/fileRelativePath -// e.g. /bob@texera.com/twitterDataset/v1/california/irvine/tw1.csv -export interface DatasetFile { - ownerEmail: string; - datasetName: string; - versionName: string; - fileRelativePath: string; -} - -/** - * Parses a file path string to a DatasetFile interface. - * @param filePath - The file path string to parse. - * @returns The parsed DatasetFile object. - */ -export function parseFilePathToDatasetFile(filePath: string): DatasetFile { - const parts = filePath.split("/").filter(part => part.length > 0); - - if (parts.length < 4) { - throw new Error("Invalid file path format"); - } - - const [ownerEmail, datasetName, versionName, ...fileRelativePathParts] = parts; - const fileRelativePath = fileRelativePathParts.join("/"); - - return { - ownerEmail, - datasetName, - versionName, - fileRelativePath, - }; -} - -/** - * Converts a DatasetFile object to a file path string. - * @param datasetFile - The DatasetFile object to convert. - * @returns The file path string. - */ -export function parseDatasetFileToFilePath(datasetFile: DatasetFile): string { - const { ownerEmail, datasetName, versionName, fileRelativePath } = datasetFile; - return `/${ownerEmail}/${datasetName}/${versionName}/${fileRelativePath}`; -} diff --git a/frontend/src/app/common/type/datasetVersionFileTree.spec.ts b/frontend/src/app/common/type/datasetVersionFileTree.spec.ts index 76aa8f286d3..64231dd1564 100644 --- a/frontend/src/app/common/type/datasetVersionFileTree.spec.ts +++ b/frontend/src/app/common/type/datasetVersionFileTree.spec.ts @@ -9,12 +9,11 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ import { @@ -38,27 +37,40 @@ describe("getFullPathFromDatasetFileNode", () => { }); describe("getRelativePathFromDatasetFileNode", () => { - it("strips the first three path segments", () => { - const node: DatasetFileNode = { name: "file.csv", type: "file", parentDir: "/owner/dataset/v1" }; - // full path is /owner/dataset/v1/file.csv -> segments [owner, dataset, v1, file.csv] - expect(getRelativePathFromDatasetFileNode(node)).toBe("file.csv"); + it("strips the datasets/owner/dataset/version prefix (4 segments)", () => { + const node: DatasetFileNode = { + name: "tw1.csv", + type: "file", + parentDir: "/datasets/bob@texera.com/twitterDataset/v1/california/irvine", + }; + expect(getRelativePathFromDatasetFileNode(node)).toBe("california/irvine/tw1.csv"); }); - it("preserves nested relative segments beyond the first three", () => { - const node: DatasetFileNode = { name: "f.txt", type: "file", parentDir: "/owner/dataset/v1/sub/dir" }; - expect(getRelativePathFromDatasetFileNode(node)).toBe("sub/dir/f.txt"); + it("returns the bare file name for a file at the version root", () => { + const node: DatasetFileNode = { + name: "readme.txt", + type: "file", + parentDir: "/datasets/bob@texera.com/twitterDataset/v1", + }; + expect(getRelativePathFromDatasetFileNode(node)).toBe("readme.txt"); }); - it("returns an empty string when there are three or fewer segments", () => { - const node: DatasetFileNode = { name: "v1", type: "directory", parentDir: "/owner/dataset" }; - // full path /owner/dataset/v1 -> exactly 3 segments -> no relative path + it("returns an empty string when there is no path below the version", () => { + const node: DatasetFileNode = { + name: "v1", + type: "directory", + parentDir: "/datasets/bob@texera.com/twitterDataset", + }; expect(getRelativePathFromDatasetFileNode(node)).toBe(""); }); it("ignores empty segments from duplicate slashes when counting", () => { - const node: DatasetFileNode = { name: "file.csv", type: "file", parentDir: "/owner//dataset/v1" }; - // empty segment between the duplicate slashes is filtered out, leaving 4 real segments - expect(getRelativePathFromDatasetFileNode(node)).toBe("file.csv"); + const node: DatasetFileNode = { + name: "f.csv", + type: "file", + parentDir: "/datasets/bob@texera.com/twitterDataset//v1/sub", + }; + expect(getRelativePathFromDatasetFileNode(node)).toBe("sub/f.csv"); }); }); diff --git a/frontend/src/app/common/type/datasetVersionFileTree.ts b/frontend/src/app/common/type/datasetVersionFileTree.ts index 8d1686998ca..98f898d4432 100644 --- a/frontend/src/app/common/type/datasetVersionFileTree.ts +++ b/frontend/src/app/common/type/datasetVersionFileTree.ts @@ -31,19 +31,20 @@ export function getFullPathFromDatasetFileNode(node: DatasetFileNode): string { } /** - * Returns the relative path of a DatasetFileNode by stripping the first three segments. + * Returns the relative path of a DatasetFileNode by stripping the first four segments + * (datasets/ownerEmail/datasetName/versionName). * @param node The DatasetFileNode whose relative path is needed. - * @returns The relative path (without the first three segments and without a leading slash). + * @returns The relative path (without the first four segments and without a leading slash). */ export function getRelativePathFromDatasetFileNode(node: DatasetFileNode): string { const fullPath = getFullPathFromDatasetFileNode(node); // Get the full path const pathSegments = fullPath.split("/").filter(segment => segment.length > 0); // Split and remove empty segments - if (pathSegments.length <= 3) { - return ""; // If there are 3 or fewer segments, return an empty string (no relative path exists) + if (pathSegments.length <= 4) { + return ""; // If there are 4 or fewer segments, return an empty string (no relative path exists) } - return pathSegments.slice(3).join("/"); // Join remaining segments as the relative path + return pathSegments.slice(4).join("/"); // Join remaining segments as the relative path } export function getPathsUnderOrEqualDatasetFileNode(node: DatasetFileNode): string[] { diff --git a/frontend/src/app/common/type/resource-type.ts b/frontend/src/app/common/type/resource-type.ts new file mode 100644 index 00000000000..a129c917e27 --- /dev/null +++ b/frontend/src/app/common/type/resource-type.ts @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The leading segment of a logical file path, identifying the resource kind (e.g. /datasets/...). + */ +export enum ResourceType { + Datasets = "datasets", +} diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-version-filetree/user-dataset-version-filetree.component.spec.ts b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-version-filetree/user-dataset-version-filetree.component.spec.ts index 0abb9626218..673c1d39bda 100644 --- a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-version-filetree/user-dataset-version-filetree.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-version-filetree/user-dataset-version-filetree.component.spec.ts @@ -167,9 +167,9 @@ describe("UserDatasetVersionFiletreeComponent", () => { const emitted: string[] = []; component.setCoverImage.subscribe((path: string) => emitted.push(path)); - // parentDir has exactly the three stripped segments (owner/dataset/version), + // parentDir has exactly the four stripped segments (datasets/owner/dataset/version), // so the relative path is just the file name. - component.onSetCover({ name: "photo.png", type: "file", parentDir: "/owner/dataset/v1" }); + component.onSetCover({ name: "photo.png", type: "file", parentDir: "/datasets/owner/dataset/v1" }); expect(emitted).toEqual(["photo.png"]); }); diff --git a/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.spec.ts b/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.spec.ts index bf3bfdb1334..d1483b71ad5 100644 --- a/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.spec.ts +++ b/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.spec.ts @@ -108,7 +108,7 @@ describe("DatasetSelectionModalComponent", () => { it("ngOnInit initializes selectedDataset and selectedVersion from data.selectedPath", () => { modalData.fileMode = true; - modalData.selectedPath = `/${OWNER}/myds/v1`; + modalData.selectedPath = `/datasets/${OWNER}/myds/v1`; build(); @@ -141,7 +141,7 @@ describe("DatasetSelectionModalComponent", () => { expect(datasetService.retrieveDatasetVersionFileTree).toHaveBeenCalledWith(10, 100); expect(component.fileTree).toEqual([fileNode]); - expect(component.selectedPath).toBe(`/${OWNER}/myds/v1`); + expect(component.selectedPath).toBe(`/datasets/${OWNER}/myds/v1`); }); it("onFileSelected sets selectedPath to the node's full path in file mode", () => { diff --git a/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.ts b/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.ts index fb289b02904..5ee5b9115e8 100644 --- a/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.ts +++ b/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.ts @@ -22,6 +22,7 @@ import { NZ_MODAL_DATA, NzModalRef } from "ng-zorro-antd/modal"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; import { DatasetFileNode, getFullPathFromDatasetFileNode } from "../../../common/type/datasetVersionFileTree"; import { DatasetVersion } from "../../../common/type/dataset"; +import { ResourceType } from "../../../common/type/resource-type"; import { DashboardDataset } from "../../../dashboard/type/dashboard-dataset.interface"; import { DatasetService } from "../../../dashboard/service/user/dataset/dataset.service"; import { NzRowDirective, NzColDirective } from "ng-zorro-antd/grid"; @@ -83,7 +84,8 @@ export class DatasetSelectionModalComponent implements OnInit { this.datasets = datasets; const selectedPath = this.data.selectedPath; if (selectedPath) { - const [ownerEmail, datasetName, versionName] = selectedPath.split("/").filter(part => part.length > 0); + // Stored paths always carry the resource-type prefix; skip it so that owner/dataset/version line up. + const [, ownerEmail, datasetName, versionName] = selectedPath.split("/").filter(part => part.length > 0); this.selectedDataset = this.datasets.find( dataset => dataset.ownerEmail === ownerEmail && dataset.dataset.name === datasetName ); @@ -118,7 +120,7 @@ export class DatasetSelectionModalComponent implements OnInit { this.fileTree = data.fileNodes; }); if (!this.data.fileMode) { - this.selectedPath = `/${this.selectedDataset.ownerEmail}/${this.selectedDataset.dataset.name}/${this.selectedVersion.name}`; + this.selectedPath = `/${ResourceType.Datasets}/${this.selectedDataset.ownerEmail}/${this.selectedDataset.dataset.name}/${this.selectedVersion.name}`; } } } diff --git a/sql/changelog.xml b/sql/changelog.xml index 469868f8958..a21dc050e51 100644 --- a/sql/changelog.xml +++ b/sql/changelog.xml @@ -53,6 +53,24 @@ + + + + + + + + + + + + + + + + + +