From 0045bac1768338be4b3c0f4e12af39d92937536a Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Fri, 12 Jun 2026 23:02:05 +0200 Subject: [PATCH 1/9] feat: interpret outputPath with LFN: prefix as absolute one --- src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 53c803b3400..bb2cf8f3811 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -1113,6 +1113,10 @@ def __getLFNfromOutputFile(self, outputFile, outputPath=""): # If output path is given, append it to the user path and put output files in this directory if outputPath.startswith("/"): outputPath = outputPath[1:] + # If output path is given with the LFN: prefix, take it as an absolute path + elif outputPath.startswith("LFN:"): + outputPath = outputPath[5:] + basePath = "" else: # By default the output path is constructed from the job id subdir = str(int(self.jobID / 1000)) From ad22da4b2323a74ebf0b043626ea7693e310927c Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Sat, 13 Jun 2026 00:34:06 +0200 Subject: [PATCH 2/9] feat: make output LFNs globbable --- .../JobWrapper/JobWrapper.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index bb2cf8f3811..37cc2ff1db7 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -951,7 +951,17 @@ def __transferOutputDataFiles(self, outputData, outputSE, outputPath): else: nonlfnList.append(out) - # Check whether list of outputData has a globbable pattern + # Check whether the list of LFNs has globbable patterns + globbedLfnList = [] + for lfn in lfnList: + lfnPath = os.path.dirname(lfn) + lfnLocal = os.path.basename(lfn) + globbedLfnList += [os.path.join(lfnPath, gLfn) for gLfn in List.uniqueElements(getGlobbedFiles(lfnLocal))] + if globbedLfnList != lfnList and globbedLfnList: + self.log.info("Found a pattern in the output data LFN list, LFNs to upload are:", ", ".join(globbedLfnList)) + lfnList = globbedLfnList + + # Check whether the list of outputData has a globbable pattern globbedOutputList = List.uniqueElements(getGlobbedFiles(nonlfnList)) if globbedOutputList != nonlfnList and globbedOutputList: self.log.info( From ab0fb974d88cafde6242ae232d9f761db2d7f0fb Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Sat, 13 Jun 2026 15:17:30 +0200 Subject: [PATCH 3/9] fix: fixed path evaluation --- src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 37cc2ff1db7..35498d2aa09 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -1125,7 +1125,7 @@ def __getLFNfromOutputFile(self, outputFile, outputPath=""): outputPath = outputPath[1:] # If output path is given with the LFN: prefix, take it as an absolute path elif outputPath.startswith("LFN:"): - outputPath = outputPath[5:] + outputPath = outputPath[4:] basePath = "" else: # By default the output path is constructed from the job id From e7fb85f54280455e486aa14f09ccaa2dfff0a46e Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Mon, 15 Jun 2026 16:13:02 +0200 Subject: [PATCH 4/9] feat: added documentation on outputData/Path/SE specification --- .../UserJobs/JDLReference/index.rst | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst index c0301b624ab..1ca6a8e2fcd 100644 --- a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst +++ b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst @@ -70,11 +70,11 @@ In this section all the attributes that can be used in the DIRAC JDL job descrip +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ | *InputDataPolicy* | Job input data policy | InputDataPolicy = ``"DIRAC.WorkloadManagementSystem.Client.DownloadInputData";`` | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ -| *OutputData* | Job output data files | OutputData = ``{"output1","output2"};`` | +| *OutputData* [1] | Job output data files | OutputData = ``{"output1","output2"};`` | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ -| *OutputPath* | The output data path in the File Catalog | OutputPath = ``{"/myjobs/output"};`` | +| *OutputPath* [2] | The output data path in the File Catalog | OutputPath = ``{"/myjobs/output"};`` | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ -| *OutputSE* | The output data Storage Element | OutputSE = ``{"DIRAC-USER"};`` | +| *OutputSE* [3] | The output data Storage Element | OutputSE = ``{"DIRAC-USER"};`` | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ | | | :subtitle:`Parametric Jobs` | @@ -91,3 +91,24 @@ In this section all the attributes that can be used in the DIRAC JDL job descrip +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ | *ParameterFactor* | Parameter multiplier | ParameterFactor = 1.1; (default 1.) | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ + +1. Elements of OutputData can be specified in several forms: + + - file names; in this case files with the specified names will be looked for in the job directory and uploaded + to a location specified by the OutputPath; + - file names with wild cards; same after the file names expansion; + - file names in a form "LFN:/vo/full/destination/path/file.name"; in this case the file will be uploaded + to the specified LFN path without taking into account the OutputPath. Note that file.name here can be also + specified with wild cards. + +2. The OutputPath can be specified in several ways + + - if not given it will be taken as the user's home directory + the job directory + for example "/lhcb/user/a/atsareg/1234/1234567", where 1234567 is the job ID; + - if given as path starting with "/", it will be appended to the user's home + directory; + - if given as "LFN:/output/path", it will be taken as an absolute path for + output files in the logical namespace. + +3. If multiple output SEs are specified, they will be tried one-by-one for each + output file until a successful file upload. From 9e7827f680bc4f1d5a0ce3080ce7358294f1cb33 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Mon, 15 Jun 2026 17:47:41 +0200 Subject: [PATCH 5/9] feat: improve the docs quality --- .../UserJobs/JDLReference/index.rst | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst index 1ca6a8e2fcd..a6e556a9317 100644 --- a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst +++ b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst @@ -94,21 +94,23 @@ In this section all the attributes that can be used in the DIRAC JDL job descrip 1. Elements of OutputData can be specified in several forms: - - file names; in this case files with the specified names will be looked for in the job directory and uploaded - to a location specified by the OutputPath; - - file names with wild cards; same after the file names expansion; - - file names in a form "LFN:/vo/full/destination/path/file.name"; in this case the file will be uploaded - to the specified LFN path without taking into account the OutputPath. Note that file.name here can be also - specified with wild cards. + - filenames; in this case files with the specified names will be looked for in the job directory and uploaded + to a location specified by the OutputPath (see below); + - filenames with wild cards, e.g. "*.log"; same after the filenames expansion; + - output data specified in a form "LFN:/vo/full/destination/path/filename"; in this case the file "filename" in + the job directory will be uploaded to the specified LFN path without taking into account the OutputPath. + Note that "filename" here can be also specified with wild cards, e.g. "LFN:/vo/full/destination/path/*.log". 2. The OutputPath can be specified in several ways - - if not given it will be taken as the user's home directory + the job directory + - if not given, it will be taken as the user's home directory + the job directory for example "/lhcb/user/a/atsareg/1234/1234567", where 1234567 is the job ID; - - if given as path starting with "/", it will be appended to the user's home - directory; + - if given as a path starting with "/", it will be appended to the user's home + directory, e.g. outputPath = "/my/analysis" will make output files to go to the + "/lhcb/user/a/atsareg/my/analysis" directory - if given as "LFN:/output/path", it will be taken as an absolute path for - output files in the logical namespace. + output files in the logical namespace. It is the responsibility of the user to make + sure that this path is accessible for writing for the user's data. 3. If multiple output SEs are specified, they will be tried one-by-one for each output file until a successful file upload. From 6db9106027ffb6d35cbd88092c3d2046db4eafe9 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Tue, 21 Jul 2026 16:01:45 +0200 Subject: [PATCH 6/9] feat: added output data jobWrapper tests --- .../JobWrapper/test/Test_JobWrapper.py | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py index 4fb1521f718..3986ff61326 100644 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py @@ -4,6 +4,7 @@ import shutil import pytest from unittest.mock import MagicMock +from pathlib import Path from DIRAC import gLogger @@ -16,6 +17,14 @@ getSystemSectionMock = MagicMock() getSystemSectionMock.return_value = "aValue" +uploadSandboxMock = MagicMock() +uploadSandboxMock.return_value = {"OK": True} + + +def uploadFileMockFunc(**kwargs): + destinationSEList = kwargs["destinationSEList"] + return {"OK": True, "Value": {"uploadedSE": destinationSEList[0]}} + gLogger.setLevel("DEBUG") @@ -48,6 +57,124 @@ def test_InputData(mocker): assert res["OK"] +@pytest.fixture +def jobIDPath(): + """Return the path to the job ID file.""" + # Create a temporary directory named ./123123/ + jobid = "123123" + p = Path(jobid) + if p.exists(): + shutil.rmtree(jobid) + p.mkdir() + + # Output sandbox files + (p / "std.out").touch() + (p / "std.err").touch() + # Output data files + (p / "00232454_00000244.xml").touch() + (p / "result_dir").mkdir() + (p / "result_dir" / "output.xml").touch() + (p / "result_dir" / "output.txt").touch() + (p / "00232454_00000244_1.sim").touch() + (p / "1720442808testFileUpload.txt").touch() + (p / "testFileUploadFullLFN.txt").touch() + + yield int(jobid) + + # Remove the temporary directory + shutil.rmtree(jobid) + + +@pytest.mark.parametrize( + "outputData, outputPath, expectedResult", + [ + ( + "00232454_00000244.xml", + None, + "/dirac/user/u/unknown/123/123123/00232454_00000244.xml", + ), + ( + "00232454_00000244*", + None, + "/dirac/user/u/unknown/123/123123/00232454_00000244.xml, " + "/dirac/user/u/unknown/123/123123/00232454_00000244_1.sim", + ), + ( + "*.txt", + None, + "/dirac/user/u/unknown/123/123123/1720442808testFileUpload.txt, " + "/dirac/user/u/unknown/123/123123/testFileUploadFullLFN.txt", + ), + ( + "00232454_00000244.xml", + "/my_output_dir/00232454", + "/dirac/user/u/unknown/my_output_dir/00232454/00232454_00000244.xml", + ), + ( + "00232454_00000244.xml", + "LFN:/dirac/prod/00232454", + "/dirac/prod/00232454/00232454_00000244.xml", + ), + ( + "LFN:/dirac/prod/00232454/00232454_00000244.xml", + None, + "/dirac/prod/00232454/00232454_00000244.xml", + ), + ( + "LFN:/dirac/prod/00232454/00232454_00000244.xml", + "/my_output_dir/00232454", + "/dirac/prod/00232454/00232454_00000244.xml", + ), + ( + "result_dir", + None, + "/dirac/user/u/unknown/123/123123/output.xml, /dirac/user/u/unknown/123/123123/output.txt", + ), + ( + "result_dir/*.xml", + None, + "/dirac/user/u/unknown/123/123123/output.xml", + ), + ( + "result_dir/*.xml", + "/my_output_dir/00232454", + "/dirac/user/u/unknown/my_output_dir/00232454/output.xml", + ), + ], +) +def test_OutputData(mocker, jobIDPath, outputData, outputPath, expectedResult): + mocker.patch( + "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock + ) + mocker.patch( + "DIRAC.DataManagementSystem.Client.FailoverTransfer.FailoverTransfer.transferAndRegisterFile", + side_effect=uploadFileMockFunc, + ) + mocker.patch( + "DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient.SandboxStoreClient.uploadFilesAsSandboxForJob", + side_effect=uploadSandboxMock, + ) + + jw = JobWrapper(jobIDPath) + os.chdir(str(jw.jobID)) + jw.jobArgs = { + "OutputData": outputData, + "OutputPath": outputPath, + "Owner": "duser", + "OutputSE": "DIRAC-disk", + "OutputSandbox": ["std.out", "std.err"], + } + + jw.failedFlag = False + jw.dm = dm_mock + jw.fc = fc_mock + + result = jw.processJobOutputs() + os.chdir(jw.root) + assert result["OK"] + assert jw.jobReport.jobParameters[0][1] == expectedResult + + def test_performChecks(): wd = Watchdog( pid=os.getpid(), From 12535c31af94fd2ea8953318e029dc59a6671cb8 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Tue, 21 Jul 2026 16:16:41 +0200 Subject: [PATCH 7/9] fix: make more explicit file list evaluation test --- src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 35498d2aa09..91a9c3a0eeb 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -957,13 +957,13 @@ def __transferOutputDataFiles(self, outputData, outputSE, outputPath): lfnPath = os.path.dirname(lfn) lfnLocal = os.path.basename(lfn) globbedLfnList += [os.path.join(lfnPath, gLfn) for gLfn in List.uniqueElements(getGlobbedFiles(lfnLocal))] - if globbedLfnList != lfnList and globbedLfnList: + if globbedLfnList and globbedLfnList != lfnList: self.log.info("Found a pattern in the output data LFN list, LFNs to upload are:", ", ".join(globbedLfnList)) lfnList = globbedLfnList # Check whether the list of outputData has a globbable pattern globbedOutputList = List.uniqueElements(getGlobbedFiles(nonlfnList)) - if globbedOutputList != nonlfnList and globbedOutputList: + if globbedOutputList and globbedOutputList != nonlfnList: self.log.info( "Found a pattern in the output data file list, files to upload are:", ", ".join(globbedOutputList) ) From dadf667239f5a8362f00fc5333798a89c05437d7 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Tue, 21 Jul 2026 17:19:43 +0200 Subject: [PATCH 8/9] fix: make the tests independent of the order of reported files --- .../JobWrapper/test/Test_JobWrapper.py | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py index 3986ff61326..551d3b70d7a 100644 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py @@ -96,14 +96,22 @@ def jobIDPath(): ( "00232454_00000244*", None, - "/dirac/user/u/unknown/123/123123/00232454_00000244.xml, " - "/dirac/user/u/unknown/123/123123/00232454_00000244_1.sim", + [ + "/dirac/user/u/unknown/123/123123/00232454_00000244.xml, " + "/dirac/user/u/unknown/123/123123/00232454_00000244_1.sim", + "/dirac/user/u/unknown/123/123123/00232454_00000244_1.sim, " + "/dirac/user/u/unknown/123/123123/00232454_00000244.xml", + ], ), ( "*.txt", None, - "/dirac/user/u/unknown/123/123123/1720442808testFileUpload.txt, " - "/dirac/user/u/unknown/123/123123/testFileUploadFullLFN.txt", + [ + "/dirac/user/u/unknown/123/123123/1720442808testFileUpload.txt, " + "/dirac/user/u/unknown/123/123123/testFileUploadFullLFN.txt", + "/dirac/user/u/unknown/123/123123/testFileUploadFullLFN.txt, " + "/dirac/user/u/unknown/123/123123/1720442808testFileUpload.txt", + ], ), ( "00232454_00000244.xml", @@ -128,7 +136,10 @@ def jobIDPath(): ( "result_dir", None, - "/dirac/user/u/unknown/123/123123/output.xml, /dirac/user/u/unknown/123/123123/output.txt", + [ + "/dirac/user/u/unknown/123/123123/output.xml, /dirac/user/u/unknown/123/123123/output.txt", + "/dirac/user/u/unknown/123/123123/output.txt, /dirac/user/u/unknown/123/123123/output.xml", + ], ), ( "result_dir/*.xml", @@ -172,7 +183,7 @@ def test_OutputData(mocker, jobIDPath, outputData, outputPath, expectedResult): result = jw.processJobOutputs() os.chdir(jw.root) assert result["OK"] - assert jw.jobReport.jobParameters[0][1] == expectedResult + assert jw.jobReport.jobParameters[0][1] in expectedResult def test_performChecks(): From 279175fd1e25df990e3c04b18ca820e5c564b938 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Wed, 22 Jul 2026 01:51:36 +0200 Subject: [PATCH 9/9] fix: try to please docutils --- .../UserJobs/JDLReference/index.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst index a6e556a9317..5f74280956e 100644 --- a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst +++ b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst @@ -96,19 +96,19 @@ In this section all the attributes that can be used in the DIRAC JDL job descrip - filenames; in this case files with the specified names will be looked for in the job directory and uploaded to a location specified by the OutputPath (see below); - - filenames with wild cards, e.g. "*.log"; same after the filenames expansion; - - output data specified in a form "LFN:/vo/full/destination/path/filename"; in this case the file "filename" in - the job directory will be uploaded to the specified LFN path without taking into account the OutputPath. - Note that "filename" here can be also specified with wild cards, e.g. "LFN:/vo/full/destination/path/*.log". + - filenames with wild cards, e.g. ``"*.log"`` ; same after the filenames expansion; + - output data specified in a form ``"LFN:/vo/full/destination/path/filename"``; in this case the file ``"filename"`` + in the job directory will be uploaded to the specified LFN path without taking into account the OutputPath. + Note that "filename" here can be also specified with wild cards, e.g. ``"LFN:/vo/full/destination/path/*.log"`` . 2. The OutputPath can be specified in several ways - if not given, it will be taken as the user's home directory + the job directory - for example "/lhcb/user/a/atsareg/1234/1234567", where 1234567 is the job ID; + for example ``"/lhcb/user/a/atsareg/1234/1234567"``, where 1234567 is the job ID; - if given as a path starting with "/", it will be appended to the user's home - directory, e.g. outputPath = "/my/analysis" will make output files to go to the - "/lhcb/user/a/atsareg/my/analysis" directory - - if given as "LFN:/output/path", it will be taken as an absolute path for + directory, e.g. outputPath = ``"/my/analysis"`` will make output files to go to the + ``"/lhcb/user/a/atsareg/my/analysis"`` directory + - if given as ``"LFN:/output/path"``, it will be taken as an absolute path for output files in the logical namespace. It is the responsibility of the user to make sure that this path is accessible for writing for the user's data.