diff --git a/Dataproducts-Summary-table.tex b/Dataproducts-Summary-table.tex new file mode 100644 index 0000000..f4d1fe7 --- /dev/null +++ b/Dataproducts-Summary-table.tex @@ -0,0 +1,33 @@ +% Mireille Louys- 2026-June 24 +%proposal for taking into account VEP for analysis products and response function +%\newpage +\subsubsection{Summary Table} + +The proposed vocabulary entries are listed in Table~\ref{tab:dp_vocabulary} with their labels and parents identified. Two vocabularies are proposed for response functions and analysis products following the semantics procedure for VEPs (see \url{https://github.com/ivoa-std/VEPs}). + +\begin{landscape} +\begin{longtable}{p{0.1\linewidth}p{0.175\linewidth}p{0.475\linewidth}p{0.175\linewidth}} +\sptablerule +\textbf{Term} & \textbf{Label} & \textbf{Description} &\textbf{Parent}\cr +\sptablerule +\multicolumn{4}{|r|}{\bf Added to \emph{product-type} Vocabulary \url{https://www.ivoa.net/rdf/product-type/}} \\ \hline +{\bf hea-event-bundle} & Event Bundle & A compounded dataset containing containing an {\bf hea-event-list} and multiple files or other substructures that are products necessary to analyze the {\bf hea-event-list} & none \cr +{\bf hea-event-list} & Event list & A dataset that records a collection of observed particle-detection events, such as incoming high-energy particles, where an event is typically characterized by a spatial position, a time, and a spectral value ({\em e.g.\/}, an energy, a channel, a pulse height) & none \cr + \hline +\multicolumn{4}{|r|}{\bf Added to \emph{response-type} vocabulary \url{https://www.ivoa.net/rdf/response-type/ } } \\ \hline +{\bf response-function} & Response Function & A dataset that maps a physical quantity to an observable quantitiy. This term is mainly intended for retrieval. To annotate datasets, use a narrower term Narrower terms are preferred to indicate more precisely the type of {\bf response-function} & \cr +{\bf rmf} &\raggedright Redistribution Matrix File & A dataset that records the probability density function mapping from energy space into detector pulse height (or position) space & \#response-function \cr +{\bf aeff} & Effective Area & A dataset that records the ``effective area'' of a telescope and/or instrument. The effective area is the geometric area of the telescope and/or instrument reduced by efficiency factors such as reflectivity and vignetting, among other effects & \#response-function \cr +{\bf arf} &\raggedright Ancillary Response File & A dataset that records the combined telescope/instrument effective area and detector quantum efficiency as a function of energy & \#response-function \cr +{\bf bkgrate} & Background Rate & A dataset that models the rate of residual events that are not from the expected source type ({\em e.g.\/}, for gamma-ray instrument {\bf bkgrate} measures residual non-gamma-ray events coming from charged cosmic rays) & \#response-function \cr +{\bf edisp} & Energy Dispersion & A dataset that records the probability density of detecting an event with an energy estimator (proxy) given the true energy of the event & \#response-function \cr +{\bf psf} &\raggedright Point Spread Function & A dataset that records the probability density function of spatial/angular spreading of incident particles from a point source caused by the instrument (detector and/or mirror and/or analysis) & \#response-function \cr \hline +\multicolumn{4}{|r|}{\bf Added to \emph{analysis-product} vocabulary \url{ https://github.com/ivoa-std/VEPs}} \\ \hline +{\bf draws} & Draws & A dataset that records statistical draws computed from a probability distribution, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable & none \cr +{\bf pdf} &\raggedright Probability Density Function & A dataset that records the probability density function of a quantity, for example the Bayesian marginal probability density function for a random variable & none \cr +{\bf region} & Region & A dataset that encodes (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary & none \cr +%\sptablerule +\caption{IVOA Vocabulary Extension for High energy data products. } +\label{tab:dp_vocabulary} +\end{longtable} +\end{landscape} diff --git a/HighEnergyObsCoreExt.bib b/HighEnergyObsCoreExt.bib index 673d980..8a70679 100644 --- a/HighEnergyObsCoreExt.bib +++ b/HighEnergyObsCoreExt.bib @@ -7,6 +7,15 @@ @MISC{2024ivoa.note.heig url = {https://www.ivoa.net/documents/Notes/VOHE/20241112/index.html}, } +@MISC{2024ivoa.note.datalink, + author = {{Bonnarel}, F. and {Demleitner}, M. and {Dowler}, P. and {Michel}, L. and {Taylor}, M. }, + title = "{IVOA DataLink Implementation note}", + howpublished = {IVOA Note 12 November 2024}, + year = 2024, + month = jan, + url = {https://www.ivoa.net/documents/Notes/DataLinkImp/20240719/index.html}, +} + @INPROCEEDINGS{2006SPIE.6270E..1VF, author = {{Fruscione}, Antonella and {McDowell}, Jonathan C. and {Allen}, Glenn E. and {Brickhouse}, Nancy S. and {Burke}, Douglas J. and {Davis}, John E. and {Durham}, Nick and {Elvis}, Martin and {Galle}, Elizabeth C. and {Harris}, Daniel E. and {Huenemoerder}, David P. and {Houck}, John C. and {Ishibashi}, Bish and {Karovska}, Margarita and {Nicastro}, Fabrizio and {Noble}, Michael S. and {Nowak}, Michael A. and {Primini}, Frank A. and {Siemiginowska}, Aneta and {Smith}, Randall K. and {Wise}, Michael}, title = "{CIAO: Chandra's data analysis system}", @@ -109,4 +118,4 @@ @INCOLLECTION{2019adds.book..283C doi = {10.1007/978-94-024-1631-2_7}, adsurl = {https://ui.adsabs.harvard.edu/abs/2019adds.book..283C}, adsnote = {Provided by the SAO/NASA Astrophysics Data System} -} \ No newline at end of file +} diff --git a/HighEnergyObsCoreExt.tex b/HighEnergyObsCoreExt.tex index d4ab57e..ab08956 100644 --- a/HighEnergyObsCoreExt.tex +++ b/HighEnergyObsCoreExt.tex @@ -439,35 +439,10 @@ \subsubsection{Advanced Data Products} {\bf region}: A dataset that includes an encoding of (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary. \end{quote} +%mireille proposal for vocabularies +\input{Dataproducts-Summary-table} -%\newpage -\subsubsection{Summary Table} - -The proposed vocabulary entries are listed in Table~\ref{tab:dp_vocabulary} with their labels and parents identified. - -\begin{landscape} -\begin{longtable}{p{0.125\linewidth}p{0.175\linewidth}p{0.475\linewidth}p{0.175\linewidth}} -\sptablerule -\textbf{Term} & \textbf{Label} & \textbf{Description} &\textbf{Parent}\cr -\sptablerule -{\bf aeff} & Effective Area & A dataset that records the ``effective area'' of a telescope and/or instrument. The effective area is the geometric area of the telescope and/or instrument reduced by efficiency factors such as reflectivity and vignetting, among other effects & \#response-function \cr -{\bf arf} &\raggedright Ancillary Response File & A dataset that records the combined telescope/instrument effective area and detector quantum efficiency as a function of energy & \#response-function \cr -{\bf bkgrate} & Background Rate & A dataset that models the rate of residual events that are not from the expected source type ({\em e.g.\/}, for gamma-ray instrument {\bf bkgrate} measures residual non-gamma-ray events coming from charged cosmic rays) & \#response-function \cr -{\bf draws} & Draws & A dataset that records statistical draws computed from a probability distribution, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable & \#measurements \cr -{\bf edisp} & Energy Dispersion & A dataset that records the probability density of detecting an event with an energy estimator (proxy) given the true energy of the event & \#response-function, \#pdf \cr -{\bf hea-event-bundle} & Event Bundle & A compounded dataset containing containing an {\bf hea-event-list} and multiple files or other substructures that are products necessary to analyze the {\bf hea-event-list} & \cr -{\bf hea-event-list} & Event list & A dataset that records a collection of observed particle-detection events, such as incoming high-energy particles, where an event is typically characterized by a spatial position, a time, and a spectral value ({\em e.g.\/}, an energy, a channel, a pulse height) & \#temporally-resolved-dataset \cr -{\bf pdf} &\raggedright Probability Density Function & A dataset that records the probability density function of a quantity, for example the Bayesian marginal probability density function for a random variable & \#measurements \cr -{\bf psf} &\raggedright Point Spread Function & A dataset that records the probability density function of spatial/angular spreading of incident particles from a point source caused by the instrument (detector and/or mirror and/or analysis) & \#response-function, \#pdf \cr -{\bf region} & Region & A dataset that encodes (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary & \#measurements \cr -{\bf response-function} & Response Function & A dataset that maps a physical quantity to an observable quantitiy. This term is mainly intended for retrieval. To annotate datasets, use a narrower term Narrower terms are preferred to indicate more precisely the type of {\bf response-function} & \cr -{\bf rmf} &\raggedright Redistribution Matrix File & A dataset that records the probability density function mapping from energy space into detector pulse height (or position) space & \#response-function, \#pdf \cr -\sptablerule -\caption{IVOA Data Product Type Vocabulary Extension} -\label{tab:dp_vocabulary} -\end{longtable} -\end{landscape} - +% \subsubsection{Clarification of ``Flux'' in Data Product Type Vocabulary Definitions} \label{sec:flux} @@ -551,12 +526,12 @@ \subsubsection{Statistical Parameters} % mireille We should use here the terms that have been recently discussed and approved by the semantics group . I would like the semantics discussions that happened to appear in this note. -The shape of any statistical distribution is an essential quantity for interpreting the meaning of any statistical properties. Too often a Gaussian distribution or a distribution that can be characterized by a simple set of moments ({\em e.g.\/}, mean, variance, skewness, kurtosis) are assumed, but in the extreme Poisson regime common in \gls{HEA} these assumptions are often invalid. We propose adding a UCD {\em stat.distribution\/} to identify a quantity that defines the distribution of a statistical variable such as a likelihood profile. +The shape of any statistical distribution is an essential quantity for interpreting the meaning of any statistical properties. Too often a Gaussian distribution or a distribution that can be characterized by a simple set of moments ({\em e.g.\/}, mean, variance, skewness, kurtosis) are assumed, but in the extreme Poisson regime common in \gls{HEA} these assumptions are often invalid. We propose adding a UCD {\em stat.distribution \/} to identify a quantity that defines the distribution of a statistical variable such as a likelihood profile. % mireille This assumes that a column will contain a full set of data values. Is this ? %In fact it would make the UCD work as a term compatible to a data product type , so ambiguous in terms of role . % in the case we want to mention the kind of distribution this parameter describes , then the UCD for that would be : meta.name;stat.distribution and a new UCD ' stat.distribution' is needed to cover the idea of a statistical distribution. %This would be -%. S& {\em stat.distribution\/} & Related to a statistical distribution \cr as inserted below +%. S& {\em stat.distribution \/} & Related to a statistical distribution \cr as inserted below \subsubsection{Evolution of UCD list} @@ -571,7 +546,7 @@ \subsubsection{Evolution of UCD list} S & {\em em.gamma.uhe\/} & Ultra-High-Energy gamma ray ($>100$ TeV) \cr Q & {\em instr.detection\/} & Particle event detection \cr %Q & {\em instr.event.grade\/} & Particle event grade \cr -Q & {\em instr.pulse\/} & Pulse height amplitude measure \cr +Q & {\em instr.pulse\/} & Pulse height amplitude measure \cr %Q & {\em instr.event.type\/} & Particle event type \cr E & {\em phot.count.density\/} & Count flux density (dimensionality: $\rm [L^{-2}\,T^{-1}\,E^{-1}]$) \cr E & {\em phot.count.density.sb\/} & Count flux density surface brightness (dimensionality: $\rm [L^{-2}\,T^{-1}\,E^{-1}\,\hbox{sr}^{-1}]$) \cr @@ -591,11 +566,12 @@ \subsubsection{Evolution of UCD list} S & {\em phys.particle.electron\/} & Related to electron \cr S & {\em phys.particle.photon\/} & Related to photon \cr S & {\em phys.particle.positron\/} & Related to positron \cr -% Mireille we cannot have these terms in the UCD tree ; that would imply importing all possible encoding of any kind -S & {\em phys.particle.pdgid\/} & Particle Data Group Identifier \cr +%S & {\em phys.particle.pdgid\/} & Particle Data Group Identifier \cr +P& meta.ref.pdgid &Particle data group identifier encoding a type of particle \cr +% we stated with semantics to use meta.ref.pdgdid;phys.particle for messenger %S & {\em phys.particle.pdgid$\pm$XX\/} & Related to a particle with PDG ID $\pm$XX \cr %mireille -S& {\em stat.distribution\/} & Related to a statistical distribution \cr +S& {\em stat.distribution \/} & Related to a statistical distribution \cr %mireille update to latest discussed term P & {\em stat.error.minus\/} & Negative statistical error \cr P & {\em stat.error.plus\/} & Positive statistical error \cr @@ -682,6 +658,9 @@ \section{Proposed ivoa.obscore\_hea Table Attributes}\label{sec:ibscoreext} \end{center} \end{landscape} +\input{extendedAccessTonewtypesOfproducts.tex} + + \pagebreak %\printglossaries diff --git a/Makefile b/Makefile index dcae619..a4deef9 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ DOCNAME = HighEnergyObsCoreExt DOCVERSION = 1.0 # Publication date, ISO format; update manually for "releases" -DOCDATE = 2026-05-20 +DOCDATE = 2026-06-24 # What is it you're writing: NOTE, WD, PR, REC, PEN, or EN DOCTYPE = PEN diff --git a/Obscore+datalink.png b/Obscore+datalink.png new file mode 100644 index 0000000..6c434eb Binary files /dev/null and b/Obscore+datalink.png differ diff --git a/extendedAccessTonewtypesOfproducts.tex b/extendedAccessTonewtypesOfproducts.tex new file mode 100644 index 0000000..6f8ad3b --- /dev/null +++ b/extendedAccessTonewtypesOfproducts.tex @@ -0,0 +1,305 @@ + +%\newpage +\newcommand{\blinks}{\{links\}} +% François Version 2: +\section{Extending access to new types of data products in ObsTAP services} +%\section{Access methods for data related to sky datasets discovered with ObsTAP} +%(Various implementation strategies for data access) + +\subsection{Difference between hea-event-list, hea-event-bundle, response function and analysis data products} +As discussed in the previous section, high energy data distribution requires a special focus on event lists and associated response functions as well as analysis data products. +Event lists without corresponding response functions do not allow data interpretation. That’s the reason why hea-event-lists are often gathered in the same package with the response function in an “hea-event-bundle” with a specific format (OGIP, GADF or VODF). In addition, the High Energy astronomical community has developed a lot of probabilistic analysis methods to analyse and extract information from the event lists. They cannot all be described by ObsCore parameters due to the richness and complexity of their parameters. They are listed in section~\ref{sec:voc_product_type}. Access to this category of data can be managed similarly to response functions in some cases, as proposed in section \ref{sec:analysis-dp}. + +\subsection{Direct access to an hea-event-bundle} + The simplest method to access hea-event-list and associated response function is to provide a link to an hea-event-bundle package; this can be done by providing an URL to the package in the {\em access\_url \/} FIELD of the ObsCore (or ObsCore + extension) table. The {\em access\_format \/} FIELD will contain the appropriate MIME type for the bundle package (for example x-fits-ogip, x-fits-gadf, or x-fits-vodf). Table~\ref{tab:bundle} shows an excerpt of an ObsCore result and illustrates this method. This method can be combined with methods described below in~\ref{sec:datalink} for a common access to the bundle and other resources such as previews or some specific analysis data products. + + +%\begin{landscape} +\begin{longtable}{|p{4.1cm}|p{4.4cm}|p{4.4cm}|} +\sptablerule +\textbf{\em Column name\/}&\textbf{\em Row 1 value\/}&\textbf{\em Row 2 value\/}\\ +\sptablerule +\sptablerule + \textbf{dataproduct\_type}& hea-event-list&hea-event-list\\ +\sptablerule + \textbf{dataproduct\_subtype}& GADF DL3&GADF DL3\\ +\sptablerule +\textbf{calib\_level}& 2&2\\ +\sptablerule + \textbf{obs\_collection}& HESS-DL3-DR1&HESS-DL3-DR1\\ +\sptablerule + \textbf{obs\_id}&20136&20317\\ +\sptablerule + \textbf{obs\_title}& & \\ +\sptablerule + \textbf{obs\_publisher\_did}&ivo:\/\/padc.obspm\/hess\#20136&ivo:\/\/padc.obspm\/hess\#20137\\ +\sptablerule + \textbf{obs\_creator\_did}&&\\ +\sptablerule +\textbf{access\_url}& \footnotesize{https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz}&\footnotesize{https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020137.fits.gz}\\ +\sptablerule + \textbf{access\_format}&x-fits-gadf&x-fits-gadf\\ +\sptablerule +\textbf{access\_estsize}&414720&216000\\ +\sptablerule + \textbf{target\_name}&MSH15-52&MSH15-52\\ +\sptablerule + \textbf{target\_class} &&\\ +\sptablerule + \textbf{s\_ra}&228.6125&228.6125\\ +\sptablerule + \textbf{s\_dec}&-58.771667&-59.77771677\\ +\sptablerule +\textbf{s\_fov}&5.0&5.0\\ +%\sptablerule \newpage +\sptablerule + \textbf{s\_region}&&\\ +\sptablerule + \textbf{s\_resolution}&0.10000000149011612&0.10000000149011612\\ +\sptablerule + \textbf{t\_min}&53090.123451203704&53090.144735925925\\ +\sptablerule + \textbf{t\_max}&53090.14291879629&53090.155175740736\\ + \sptablerule + \textbf{t\_exptime}&1521.02685546875&819.2053833007812\\ +\sptablerule +\textbf{t\_resolution}&9.999999974752427E-7&9.999999974752427E-7\\ +\sptablerule +\textbf {em\_min}&1.1889202653543914E-20&1.208503847135941E-20\\ +\sptablerule + \textbf{em\_max}&5.5549210638422004E-18&5.2809212082467665E-18\\ +\sptablerule +\textbf {em\_res\_power}&&\\ +\sptablerule + \textbf{o\_ucd}&time;pos;phys.energy&time;pos;phys.energy\\ +\sptablerule + \textbf{pol\_states}& &\\ +\sptablerule + \textbf{facility\_name}&H.E.S.S.&H.E.S.S\\ +\sptablerule + \textbf{instrument\_name}& 1,2,3,4&1,2,3,4\\ +\sptablerule + \textbf{s\_xel1}&-1&-1\\ +\sptablerule + \textbf{s\_xel2}& -1&-1\\ +\sptablerule + \textbf{t\_xel}&-1&-1\\ +\sptablerule + \textbf{em\_xel}& -1&-1\\ +\sptablerule +\textbf{pol\_xel}& -1&-1\\ +\sptablerule +\caption{ObsCore response for hea-event-bundles\\ \footnotesize{inspired from H.E.S.S ObsTAP prototype at Paris Observatory }} +\label{tab:bundle} +\end{longtable} +%\end{landscape} + +\subsection{Access via DataLink} +\label{sec:datalink} + +DataLink \citep{2023ivoa.spec.1215B} allows to relate a main record provided by a service (ObsTAP, SSA/SIA or a catalogue service) to a various number of links which complement the information contained in the record. An implementation note \citep{2024ivoa.note.datalink} gives some hints on how to use this standard. +There are at least three ways to implement this standard for hea-event-lists depending how many steps are used in the access method to the associated datasets.\\ + \begin{itemize} + \item Subsection~\ref{sec:datalinkinobs} is a full 2-step approach. \\ + \item Subsection~\ref{sec:datalinksd} allows both direct access to hea-event-list and 2-step access to the additional material.\\ + \item Subsection~\ref{sec:datalinktap} is not described in the DataLink standard but is fully consistant with TAP and the DataLink \blinks\ endpoint response specification. It allows direct access to both hea-event-list and additional material, like response files, or analysis data products.\\ + \end{itemize} + + \subsubsection{From an hea-event-list in ObsCore : DataLink to access the event list itself and response functions via {\em access\_url \/} } +\label{sec:datalinkinobs} + The { \em data\_product type \/} exposed in ObsCore is “hea-event-list” and the DataLink \blinks\ endpoint response points to the event list itself and to the various response functions individually. This solution allows retrieval of data with explicit access to the various components of a “bundle” which doesn't need to be packaged in that case. In addition to response functions, analysis data products can also be linked to the event list this way. Figure~\ref{fig:obscoretodl} shows an ObsCore response where the selected record points to a \blinks\ endpoint response illustrating this method. Table~\ref{tab:datalink} shows this DataLink response content. +\begin{figure} + +\includegraphics[width=1.5\textwidth, angle=90]{Obscore+datalink.png} +\caption{Selection of HESS data from Aladin Desktop: In the galactic center area, an ObsTAP query has been performed and HESS datasets are discovered. One of the ObsCore record has been selected and points to a DataLink response, with 8 links provided in the white box. The content of this DataLink table is described in Table~\ref{tab:datalink}. } +\label{fig:obscoretodl} +\end{figure} + +\begin{landscape} +\begin{center} +\begin{longtable}{|p{0.08\linewidth}|p{0.3\linewidth}|p{0.08\linewidth}|p{0.22\linewidth}|p{0.1\linewidth}|p{0.14\linewidth}|p{0.13\linewidth}|} +\hline%\sptablerule +\textbf{ID} &\textbf{\footnotesize access\_url} &\textbf{\footnotesize semantics}&\textbf{\footnotesize description} &\textbf{\footnotesize content\_type} &\textbf{\footnotesize content\_qualifier}\cr +\hline%\sptablerule +{\footnotesize 20136} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz\#EVENTS} & {\footnotesize \#this} & {\footnotesize Event list dataset (HDU=EVENTS) } & {\footnotesize image/fits} & {\footnotesize hea-event-list} \cr +\hline%\sptablerule +{\footnotesize 20136} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136\_image.png} & {\footnotesize \#preview-image} & {\footnotesize Preview image of the observation} & {\footnotesize image/png} & {\footnotesize preview} \cr +\hline%\sptablerule +{\footnotesize 20136} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136\_plot.png} & {\footnotesize \#preview-plot} & {\footnotesize Preview significance distribution of the observation } & {\footnotesize image/png} & {\footnotesize preview} \cr +\hline%\sptablerule +{\footnotesize 20136} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz\#AEFF } & {\footnotesize \#calibration} & {\footnotesize Effective Area (HDU=AEFF)} & {\footnotesize image/fits} & {\footnotesize aeff} \cr +\hline%\sptablerule +{\footnotesize 20316} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz\#EDISP} & {\footnotesize \#calibration} & {\footnotesize Energy Dispersion (HDU=EDISP)} & {\footnotesize image/fits} & {\footnotesize edisp} \cr +\hline%\sptablerule +{\footnotesize 20316} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz\#PSF} & {\footnotesize \#calibration} & {\footnotesize Point Spread Function (HDU=PSF) } & {\footnotesize image/fits} & {\footnotesize psf} \cr +\hline%\sptablerule +{\footnotesize 20316} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz\#BKG} & {\footnotesize \#calibration} & {\footnotesize Background Rate (HDU=BKG) } & {\footnotesize image/fits} & {\footnotesize bkgrate} \cr +\hline%\sptablerule +{\footnotesize 20316} & {\footnotesize https://hess-dr.obspm.fr/retrieve/hess\_dl3\newline \_dr1\_obs\_id\_020136.fits.gz} & {\footnotesize \#package} & {\footnotesize Event bundle (including HDU=EVENTS,GTI,AEFF,\newline EDISP,PSF,BKG) } & {\footnotesize x-fits-gadf } & {\footnotesize hea-event-bundle } \cr +\hline%\sptablerule + +\caption{DataLink response table attached to an hea-event-list record in ObsCore. Inspired from Paris Observatory H.E.S.S ObsTAP prototype} +\label{tab:datalink} +\end{longtable} +\end{center} +\end{landscape} + + \subsubsection{Datalink access using a service descriptor} +\label{sec:datalinksd} + To avoid preventing a direct access to the hea-event-list while keeping the explicit access to the various response functions, it is possible to add a “DataLink” service descriptor in the ObsTAP result as can be seen in the example below. + +\begin{verbatim} + +Datalink service + + + + + + +\end{verbatim} + + In this case the service descriptor provides the root URL to the DataLink \blinks\ endpoint. The full \blinks\ endpoint URL for a specific row (or group of rows) is then built by adding the content of the referred ID FIELD this way: \\ http://provider.org/dataservice/datalink?ID={\it referred\_ID\_FIELD\_content}. + It is important to notice that it is not mandatory that the {\it referred\_ID\_FIELD} be {\em obs\_publisher\_did \/}. It can be {\em obs\_id \/}, or any free identifier FIELD allowing to group several rows together. For example, referring to {\em obs\_id \/} will allow all the datasets belonging to the same observation to be bound to the same \blinks\ endpoint response. + + This method is implemented in many archive services such as ESO, CADC, GAVO, HEASARC and even VizieR. + + \subsubsection{Datalink table in a TAP service / join with ObsCore table} + \label{sec:datalinktap} + +The \blinks\ endpoint table can also be implemented as a TAP table of its own. This table will contain columns describing all the links (hea-event-list, response functions, analysis data products) desirable for any dataset record in the \texttt{ivoa.obscore} table of ObsTAP service. +A direct access to the ObsCore record and the links provided by the \blinks\ endpoint response table is then possible. + An identifier column would be common in the ObsCore table and the DataLink table and used as a keyreference. +Suppose we have {\em dataproduct\_type \/} = ‘hea-event-list’ in the \texttt{ivoa.obscore table}, and we we name this data link table “datalink.response”, we can then query the TAP service this way in order to access directly to the hea-event-list product: + +\begin{verbatim} +SELECT * FROM ivoa.obscore + NATURAL JOIN ivoa.obscore_hea + JOIN datalink.response ON + ivoa.obscore.obs_publisher_did = datalink.response.ID + WHERE (INTERSECTS(s_region, CIRCLE(312.775, 30.683, 1.5)) = 6) + AND (dataproduct_type = ’hea-event-list’ ) + AND ( semantics = '#this') +\end{verbatim} +While to access directly the psf we can select this way : +\begin{verbatim} +SELECT * FROM ivoa.obscore + NATURAL JOIN ivoa.obscore_hea + JOIN datalink.response ON + ivoa.obscore.obs_publisher_did = datalink.response.ID + WHERE (INTERSECTS(s_region, CIRCLE(312.775, 30.683, 1.5)) = 6) + AND (dataproduct_type = ’hea-event-list’ ) + AND ( semantics = '#calibration') + AND (content_qualifier = 'psf') +\end{verbatim} +and to access directly a background image : +\begin{verbatim} +SELECT * FROM ivoa.obscore + NATURAL JOIN ivoa.obscore_hea + JOIN datalink.response ON + ivoa.obscore.obs_publisher_did = datalink.response.ID + WHERE (INTERSECTS(s_region, CIRCLE(312.775, 30.683, 1.5)) = 6) + AND (dataproduct_type = ’hea-event-list’ ) + AND ( semantics = '#auxiliary') + AND (content_qualifier = 'bkgimage') +\end{verbatim} +One caveat : if response functions and other auxiliary or analysis data are common to a full observation (instead of dataset) or even worse to several observations, then the DataLink-like table will repeat the information several times. To avoid that it's possible to create an additional index column (for example {\em dl\_association\_id \/} or whatever appropriate term) and modify the first query this way (for others it would be similar of course) : +\begin{verbatim} +SELECT * FROM ivoa.obscore + NATURAL JOIN ivoa.obscore_hea + JOIN datalink.response ON ivoa.obscore.response_id = datalink.response.ID + WHERE (INTERSECTS(s_region, CIRCLE(312.775, 30.683, 1.5)) = 6) + AND (dataproduct_type = ’hea-event-list’ ) + AND ( semantics = '#this') +\end{verbatim} + +\subsection{Accessing Response datasets via a specific table joined with the \texttt{ivoa.obscore} table} + +For the discovery of response data products, the use cases recorded show that selection criteria are based on 1) the type of response 2) the observed data set properties, and on the observation, like the observing position and field of view, the observing date and time, the energy and time bounds. However, the cardinality of the relationship between a response dataset and an hea-event-list dataset varies according to projects and may be 1 to 1, like in CTA project or 1 to N, N to 1 or N to N . + +Compared to the \texttt{ivoa.obscore} table less attributes are needed to select a response file. +The response details, after discovery will be into the response files, in the metadata encoded according to the file format (OGIP, GADF, VODF, etc). +So response data products can be described by a specific response table, with a short list of attributes as proposed in Tab. \ref{tab:response_table} . + +In order to handle the various cardinality between response and observation datasets, the foreign keys are defined in the response table and will allow JOIN operations between the \texttt{ivoa.obscore} and \texttt{ivoa.response} table. + +\section*{Response table proposal} +%\TODO{ check fields for this new table } +%\begin{table}[htbp] +%\begin{center} +\begin{longtable}{|p{0.25\linewidth}|p{0.12\linewidth}|p{0.1\linewidth}|p{0.39\linewidth}|} +\hline +Column Name & Unit & Type & Description\\\hline +%obs\_collection & unitless & String & Name of the data collection \\\hline +% to bind a response with an observation data set +obs\_id & unitless & String & Related Observation ID (foreign key) \\\hline +obs\_publisher\_did & unitless & String & Related dataset ID (foreign key) \\\hline +% copied from related observation if needed +target\_name & unitless & String & Astronomical object observed, if any\\\hline +facility\_name & unitless & String & Facility Name used for observed data products \\\hline +instrument\_name & unitless & String & Instrument Name used for observed data products \\\hline +% response specific fields +resp\_product\_type & unitless & String & Product type as defined in response-type vocabulary\\\hline +resp\_publisher\_did & unitless & String & Dataset identifier given by the publisher\\\hline +resp\_access\_url & unitless & String & URL used to access (download) response dataset\\\hline +resp\_access\_format & unitless & String & File content format (see in ObsCore MIME Types )\\\hline +% response spatial coverage +resp\_s\_ra & deg & double & ICRS Central right ascension, for the region where the response-product applies \\\hline +resp\_s\_dec & deg & double & ICRS Central declination, for the region where the response-product applies\\\hline +resp\_s\_region & unitless & String & Region of interest for response use (expressed in ICRS frame)\\\hline +resp\_t\_intervals & unitless& String (TMOC) & time coverage for response use \\\hline +%response energy coverage +resp\_energy\_min & m & double & Energy band minimal value for response use \\\hline +resp\_energy\_max & m & double & Energy band maximal value for response use \\\hline + +\caption{Mandatory fields of the Response table proposed in this HEIG ObsCore extension serialisation} +\label{tab:response_table} +\end{longtable} +%\end{center} +%\end{table} + +Implementing this extra response table will allow to write queries like : +\medskip +\noindent Find all datasets satisfying: +\begin{enumerate}[(i)] + \item Position inside 3 arcmin from (83.84358, $-5.43639$) + \item Response product\_type = ``psf'' + \item obs\_id = ``1527'' + \item obs\_collection = ``HESS'' +\end{enumerate} + +When using the response table defined in Table \ref{tab:response_table}, +we can join the \texttt{ivoa.obscore} table with the \texttt{ivoa.response} table on +the two keys {\em obs\_id \/} and {\em obs\_collection \/} for instance, as in : +\begin{verbatim} +SELECT obs_publisher_did, s_ra, s_dec, s_fov, t_min, t_max, +energy_min, energy_max, access_url, access_format, +resp_publisher_did, resp_access_url, resp_access_format, +resp_energy_max , resp_energy_min +FROM ivoa.obscore +NATURAL JOIN ivoa.response +WHERE +(target_name = ’Crab’ OR target_name = ’M1’ OR +CONTAINS(POINT(s_ra, s_dec), CIRCLE, 83.6324, +22.0174, 0.083333) = 1) +AND (resp_product_type = 'psf') +AND (obs_id = '1527') +AND (obs_collection = 'HESS') +\end{verbatim} + +If a time constraint is needed, then we can use {\em resp\_t\_min \/}, and {\em resp\_t\_max \/} and +constrain the interval like : +\begin{verbatim} +AND (resp_t_min > 56000.0 and resp_t_max < 56001.5) +\end{verbatim} + +\section*{Querying for Analysis Data products} +\label{sec:analysis-dp} + +The same strategy can apply for searching analysis data products for one observation or for a hea-event-list data product, using a dedicated \texttt{ivoa.obscore.adp} table . +The benefice is to describe more specific properties of these data products in the optional columns, without overloading the main \texttt{ivoa.obscore} table. + +