diff --git a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst index c0301b624ab..5f74280956e 100644 --- a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst +++ b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst @@ -70,11 +70,11 @@ In this section all the attributes that can be used in the DIRAC JDL job descrip +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ | *InputDataPolicy* | Job input data policy | InputDataPolicy = ``"DIRAC.WorkloadManagementSystem.Client.DownloadInputData";`` | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ -| *OutputData* | Job output data files | OutputData = ``{"output1","output2"};`` | +| *OutputData* [1] | Job output data files | OutputData = ``{"output1","output2"};`` | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ -| *OutputPath* | The output data path in the File Catalog | OutputPath = ``{"/myjobs/output"};`` | +| *OutputPath* [2] | The output data path in the File Catalog | OutputPath = ``{"/myjobs/output"};`` | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ -| *OutputSE* | The output data Storage Element | OutputSE = ``{"DIRAC-USER"};`` | +| *OutputSE* [3] | The output data Storage Element | OutputSE = ``{"DIRAC-USER"};`` | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ | | | :subtitle:`Parametric Jobs` | @@ -91,3 +91,26 @@ In this section all the attributes that can be used in the DIRAC JDL job descrip +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ | *ParameterFactor* | Parameter multiplier | ParameterFactor = 1.1; (default 1.) | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ + +1. Elements of OutputData can be specified in several forms: + + - filenames; in this case files with the specified names will be looked for in the job directory and uploaded + to a location specified by the OutputPath (see below); + - filenames with wild cards, e.g. ``"*.log"`` ; same after the filenames expansion; + - output data specified in a form ``"LFN:/vo/full/destination/path/filename"``; in this case the file ``"filename"`` + in the job directory will be uploaded to the specified LFN path without taking into account the OutputPath. + Note that "filename" here can be also specified with wild cards, e.g. ``"LFN:/vo/full/destination/path/*.log"`` . + +2. The OutputPath can be specified in several ways + + - if not given, it will be taken as the user's home directory + the job directory + for example ``"/lhcb/user/a/atsareg/1234/1234567"``, where 1234567 is the job ID; + - if given as a path starting with "/", it will be appended to the user's home + directory, e.g. outputPath = ``"/my/analysis"`` will make output files to go to the + ``"/lhcb/user/a/atsareg/my/analysis"`` directory + - if given as ``"LFN:/output/path"``, it will be taken as an absolute path for + output files in the logical namespace. It is the responsibility of the user to make + sure that this path is accessible for writing for the user's data. + +3. If multiple output SEs are specified, they will be tried one-by-one for each + output file until a successful file upload. diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 53c803b3400..91a9c3a0eeb 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -951,9 +951,19 @@ def __transferOutputDataFiles(self, outputData, outputSE, outputPath): else: nonlfnList.append(out) - # Check whether list of outputData has a globbable pattern + # Check whether the list of LFNs has globbable patterns + globbedLfnList = [] + for lfn in lfnList: + lfnPath = os.path.dirname(lfn) + lfnLocal = os.path.basename(lfn) + globbedLfnList += [os.path.join(lfnPath, gLfn) for gLfn in List.uniqueElements(getGlobbedFiles(lfnLocal))] + if globbedLfnList and globbedLfnList != lfnList: + self.log.info("Found a pattern in the output data LFN list, LFNs to upload are:", ", ".join(globbedLfnList)) + lfnList = globbedLfnList + + # Check whether the list of outputData has a globbable pattern globbedOutputList = List.uniqueElements(getGlobbedFiles(nonlfnList)) - if globbedOutputList != nonlfnList and globbedOutputList: + if globbedOutputList and globbedOutputList != nonlfnList: self.log.info( "Found a pattern in the output data file list, files to upload are:", ", ".join(globbedOutputList) ) @@ -1113,6 +1123,10 @@ def __getLFNfromOutputFile(self, outputFile, outputPath=""): # If output path is given, append it to the user path and put output files in this directory if outputPath.startswith("/"): outputPath = outputPath[1:] + # If output path is given with the LFN: prefix, take it as an absolute path + elif outputPath.startswith("LFN:"): + outputPath = outputPath[4:] + basePath = "" else: # By default the output path is constructed from the job id subdir = str(int(self.jobID / 1000)) diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py index 4fb1521f718..551d3b70d7a 100644 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py @@ -4,6 +4,7 @@ import shutil import pytest from unittest.mock import MagicMock +from pathlib import Path from DIRAC import gLogger @@ -16,6 +17,14 @@ getSystemSectionMock = MagicMock() getSystemSectionMock.return_value = "aValue" +uploadSandboxMock = MagicMock() +uploadSandboxMock.return_value = {"OK": True} + + +def uploadFileMockFunc(**kwargs): + destinationSEList = kwargs["destinationSEList"] + return {"OK": True, "Value": {"uploadedSE": destinationSEList[0]}} + gLogger.setLevel("DEBUG") @@ -48,6 +57,135 @@ def test_InputData(mocker): assert res["OK"] +@pytest.fixture +def jobIDPath(): + """Return the path to the job ID file.""" + # Create a temporary directory named ./123123/ + jobid = "123123" + p = Path(jobid) + if p.exists(): + shutil.rmtree(jobid) + p.mkdir() + + # Output sandbox files + (p / "std.out").touch() + (p / "std.err").touch() + # Output data files + (p / "00232454_00000244.xml").touch() + (p / "result_dir").mkdir() + (p / "result_dir" / "output.xml").touch() + (p / "result_dir" / "output.txt").touch() + (p / "00232454_00000244_1.sim").touch() + (p / "1720442808testFileUpload.txt").touch() + (p / "testFileUploadFullLFN.txt").touch() + + yield int(jobid) + + # Remove the temporary directory + shutil.rmtree(jobid) + + +@pytest.mark.parametrize( + "outputData, outputPath, expectedResult", + [ + ( + "00232454_00000244.xml", + None, + "/dirac/user/u/unknown/123/123123/00232454_00000244.xml", + ), + ( + "00232454_00000244*", + None, + [ + "/dirac/user/u/unknown/123/123123/00232454_00000244.xml, " + "/dirac/user/u/unknown/123/123123/00232454_00000244_1.sim", + "/dirac/user/u/unknown/123/123123/00232454_00000244_1.sim, " + "/dirac/user/u/unknown/123/123123/00232454_00000244.xml", + ], + ), + ( + "*.txt", + None, + [ + "/dirac/user/u/unknown/123/123123/1720442808testFileUpload.txt, " + "/dirac/user/u/unknown/123/123123/testFileUploadFullLFN.txt", + "/dirac/user/u/unknown/123/123123/testFileUploadFullLFN.txt, " + "/dirac/user/u/unknown/123/123123/1720442808testFileUpload.txt", + ], + ), + ( + "00232454_00000244.xml", + "/my_output_dir/00232454", + "/dirac/user/u/unknown/my_output_dir/00232454/00232454_00000244.xml", + ), + ( + "00232454_00000244.xml", + "LFN:/dirac/prod/00232454", + "/dirac/prod/00232454/00232454_00000244.xml", + ), + ( + "LFN:/dirac/prod/00232454/00232454_00000244.xml", + None, + "/dirac/prod/00232454/00232454_00000244.xml", + ), + ( + "LFN:/dirac/prod/00232454/00232454_00000244.xml", + "/my_output_dir/00232454", + "/dirac/prod/00232454/00232454_00000244.xml", + ), + ( + "result_dir", + None, + [ + "/dirac/user/u/unknown/123/123123/output.xml, /dirac/user/u/unknown/123/123123/output.txt", + "/dirac/user/u/unknown/123/123123/output.txt, /dirac/user/u/unknown/123/123123/output.xml", + ], + ), + ( + "result_dir/*.xml", + None, + "/dirac/user/u/unknown/123/123123/output.xml", + ), + ( + "result_dir/*.xml", + "/my_output_dir/00232454", + "/dirac/user/u/unknown/my_output_dir/00232454/output.xml", + ), + ], +) +def test_OutputData(mocker, jobIDPath, outputData, outputPath, expectedResult): + mocker.patch( + "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock + ) + mocker.patch( + "DIRAC.DataManagementSystem.Client.FailoverTransfer.FailoverTransfer.transferAndRegisterFile", + side_effect=uploadFileMockFunc, + ) + mocker.patch( + "DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient.SandboxStoreClient.uploadFilesAsSandboxForJob", + side_effect=uploadSandboxMock, + ) + + jw = JobWrapper(jobIDPath) + os.chdir(str(jw.jobID)) + jw.jobArgs = { + "OutputData": outputData, + "OutputPath": outputPath, + "Owner": "duser", + "OutputSE": "DIRAC-disk", + "OutputSandbox": ["std.out", "std.err"], + } + + jw.failedFlag = False + jw.dm = dm_mock + jw.fc = fc_mock + + result = jw.processJobOutputs() + os.chdir(jw.root) + assert result["OK"] + assert jw.jobReport.jobParameters[0][1] in expectedResult + + def test_performChecks(): wd = Watchdog( pid=os.getpid(),