Enforce UK consent check on aggregate endpoints, upgrade 403 assertions, fix build-script hang

flushall_build_and_run.sh

@@ -155,15 +155,24 @@ JAVA_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED \
 RUNTIME_LOG=/tmp/obp-api.log
 
 if [ "$RUN_BACKGROUND" = true ]; then
-    # Run in background with output to log file (tee'd to /tmp as well)
-    nohup java $JAVA_OPTS -jar obp-api/target/obp-api.jar > >(tee "$RUNTIME_LOG") 2>&1 &
+    # Run in background with output redirected straight to a log file. Do NOT
+    # use `> >(tee "$RUNTIME_LOG")` here: process substitution's tee inherits
+    # this script's own stdout, so a caller that captures this script's output
+    # via `out=$(./flushall_build_and_run.sh --background ...)` never sees EOF
+    # — the substitution hangs forever, since the server (and thus the tee
+    # process backing the substitution) never exits on its own.
+    nohup java $JAVA_OPTS -jar obp-api/target/obp-api.jar > "$RUNTIME_LOG" 2>&1 &
     SERVER_PID=$!
+    # Report the port the server will actually bind (dev.port in props), so callers
+    # that capture this script's output (e.g. smoke_test.sh) can parse it out.
+    SERVER_PORT=$(grep -E '^dev.port=' obp-api/src/main/resources/props/default.props 2>/dev/null | cut -d= -f2)
     echo "✓ HTTP4S server started in background"
     echo "  PID: $SERVER_PID"
-    echo "  Log: http4s-server.log (also $RUNTIME_LOG)"
+    echo "  PORT: ${SERVER_PORT:-8080}"
+    echo "  Log: $RUNTIME_LOG"
     echo ""
     echo "To stop the server: kill $SERVER_PID"
-    echo "To view logs: tail -f http4s-server.log"
+    echo "To view logs: tail -f $RUNTIME_LOG"
 else
     # Run in foreground (Ctrl+C to stop). Also tee output to /tmp so it can be
     # tailed from another terminal without taking over this one.

flushall_fast_build_and_run.sh

@@ -324,8 +324,12 @@ if [ "$RUN_BACKGROUND" = true ]; then
     # Run in background with output to log file
     nohup java $JAVA_OPTS -jar obp-api/target/obp-api.jar > http4s-server.log 2>&1 &
     SERVER_PID=$!
+    # Report the port the server will actually bind (dev.port in props), so callers
+    # that capture this script's output (e.g. smoke_test.sh) can parse it out.
+    SERVER_PORT=$(grep -E '^dev.port=' obp-api/src/main/resources/props/default.props 2>/dev/null | cut -d= -f2)
     echo "✓ HTTP4S server started in background"
     echo "  PID: $SERVER_PID"
+    echo "  PORT: ${SERVER_PORT:-8080}"
     echo "  Log: http4s-server.log"
     echo ""
     echo "To stop the server: kill $SERVER_PID"

obp-api/src/main/resources/props/sample.props.template

@@ -1275,8 +1275,11 @@ database_messages_scheduler_interval=3600
                                                     ReadBalances,\
                                                     ReadTransactionsBasic,\
                                                     ReadTransactionsDebits,\
+                                                    ReadTransactionsCredits,\
                                                     ReadTransactionsDetail, \
                                                     ReadAccountsBerlinGroup, \
+                                                    ReadBalancesBerlinGroup, \
+                                                    ReadTransactionsBerlinGroup, \
                                                     InitiatePaymentsBerlinGroup
 # -----------------------------------------------------------------------------
 

obp-api/src/main/scala/bootstrap/liftweb/Boot.scala

@@ -324,6 +324,7 @@ class Boot extends MdcLoggable {
             SYSTEM_READ_BALANCES_VIEW_ID,
             SYSTEM_READ_TRANSACTIONS_BASIC_VIEW_ID,
             SYSTEM_READ_TRANSACTIONS_DEBITS_VIEW_ID,
+            SYSTEM_READ_TRANSACTIONS_CREDITS_VIEW_ID,
             SYSTEM_READ_TRANSACTIONS_DETAIL_VIEW_ID,
             SYSTEM_READ_ACCOUNTS_BERLIN_GROUP_VIEW_ID,
             SYSTEM_READ_BALANCES_BERLIN_GROUP_VIEW_ID,

obp-api/src/main/scala/code/api/UKOpenBanking/v3_1_0/Http4sUKOBv310AccountAccess.scala

@@ -5,13 +5,13 @@ import cats.data.{Kleisli, OptionT}
 import cats.effect.IO
 import code.api.Constant
 import code.api.UKOpenBanking.v3_1_0.JSONFactory_UKOpenBanking_310.ConsentPostBodyUKV310
-import code.api.util.APIUtil.{EmptyBody, ResourceDoc, connectorEmptyResponse, mockedDataText, passesPsd2Aisp, unboxFullOrFail, DateWithDayFormat}
+import code.api.util.APIUtil.{EmptyBody, ResourceDoc, connectorEmptyResponse, mockedDataText, passesPsd2Aisp, unboxFullOrFail, parseIso8601OrDayDate}
 import code.api.util.ApiTag
 import code.api.util.CustomJsonFormats
-import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, ConsentNotFound, ConsentViewNotFund, UnknownError}
+import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, ConsentNotFound, ConsentViewNotFund, InvalidJsonFormat, UnknownError}
 import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps}
 import code.api.util.CallContext
-import code.api.util.{ConsentJWT, JwtUtil}
+import code.api.util.{ConsentJWT, JwtUtil, NewStyle}
 import code.consent.Consents
 import code.util.Helper.MdcLoggable
 import com.github.dwickern.macros.NameOf.nameOf
@@ -47,24 +47,40 @@ object Http4sUKOBv310AccountAccess extends MdcLoggable {
       EndpointHelpers.executeFutureCreated(req) {
         implicit val cc: CallContext = req.callContext
         for {
-          u <- cc.user.toOption match {
-            case Some(user) => Future.successful(user)
-            case None       => Future.failed(new RuntimeException(AuthenticatedUserIsRequired))
-          }
+          // Client-credentials lodging: require some authentication (consumer or user) but not a
+          // PSU specifically; reject only a fully anonymous request. The PSU is bound later at
+          // authorise time. Mirrors the Berlin Group native consent flow and the v4.0.1 handler.
+          _ <- if (cc.user.isEmpty && cc.consumer.isEmpty)
+                 Future.failed(new RuntimeException(AuthenticatedUserIsRequired))
+               else Future.successful(())
+          createdByUser = cc.user.toOption
           consentJson <- Future.fromTry(scala.util.Try(
             com.openbankproject.commons.util.JsonAliases.parse(cc.httpBody.getOrElse("{}")).extract[ConsentPostBodyUKV310]
           ))
+          // Separate step (not inlined into the saveUKConsent call below) so a bad date string
+          // fails here -> 400. NewStyle.function.tryons (not Future.fromTry) is required for
+          // that: ErrorResponseConverter only special-cases APIFailureNewStyle to preserve a set
+          // HTTP code -- tryons wraps failures that way, a bare Future.fromTry(Try(...)) doesn't
+          // and falls through to unknownErrorToResponse, i.e. 500.
+          (expirationDateTime, transactionFromDateTime, transactionToDateTime) <- NewStyle.function.tryons(
+            s"$InvalidJsonFormat The Json body should have valid ISO-8601 ExpirationDateTime/TransactionFromDateTime/TransactionToDateTime values ", 400, Some(cc)) {
+            (
+              consentJson.Data.ExpirationDateTime.map(parseIso8601OrDayDate),
+              consentJson.Data.TransactionFromDateTime.map(parseIso8601OrDayDate),
+              consentJson.Data.TransactionToDateTime.map(parseIso8601OrDayDate)
+            )
+          }
           consumerId = cc.consumer.map(_.consumerId.get)
           _ <- passesPsd2Aisp(Some(cc))
           createdConsent <- Future(Consents.consentProvider.vend.saveUKConsent(
-            Some(u),
+            createdByUser,
             bankId = None,
             accountIds = None,
             consumerId = consumerId,
             permissions = consentJson.Data.Permissions,
-            expirationDateTime = DateWithDayFormat.parse(consentJson.Data.ExpirationDateTime),
-            transactionFromDateTime = DateWithDayFormat.parse(consentJson.Data.TransactionFromDateTime),
-            transactionToDateTime = DateWithDayFormat.parse(consentJson.Data.TransactionToDateTime),
+            expirationDateTime = expirationDateTime,
+            transactionFromDateTime = transactionFromDateTime,
+            transactionToDateTime = transactionToDateTime,
             apiStandard = Some("UKOpenBanking"),
             apiVersion = Some("3.1.0")
           )) map { i => connectorEmptyResponse(i, Some(cc)) }
@@ -83,11 +99,11 @@ object Http4sUKOBv310AccountAccess extends MdcLoggable {
               "Status" : "${createdConsent.status}",
               "StatusUpdateDateTime" : "${createdConsent.statusUpdateDateTime}",
               "CreationDateTime" : "${createdConsent.creationDateTime}",
-              "TransactionToDateTime" : "${consentJson.Data.TransactionToDateTime}",
-              "ExpirationDateTime" : "${consentJson.Data.ExpirationDateTime}",
+              "TransactionToDateTime" : "${consentJson.Data.TransactionToDateTime.getOrElse("")}",
+              "ExpirationDateTime" : "${consentJson.Data.ExpirationDateTime.getOrElse("")}",
               "Permissions" : ${consentJson.Data.Permissions.mkString("[\"", "\",\"", "\"]")},
               "ConsentId" : "${createdConsent.consentId}",
-              "TransactionFromDateTime" : "${consentJson.Data.TransactionFromDateTime}"
+              "TransactionFromDateTime" : "${consentJson.Data.TransactionFromDateTime.getOrElse("")}"
             }
           }""")
         }
@@ -196,11 +212,11 @@ object Http4sUKOBv310AccountAccess extends MdcLoggable {
               "Status" : "${consent.status}",
               "StatusUpdateDateTime" : "${consent.statusUpdateDateTime}",
               "CreationDateTime" : "${consent.creationDateTime}",
-              "TransactionToDateTime" : "${consent.transactionToDateTime}",
-              "ExpirationDateTime" : "${consent.expirationDateTime}",
+              "TransactionToDateTime" : "${Option(consent.transactionToDateTime).getOrElse("")}",
+              "ExpirationDateTime" : "${Option(consent.expirationDateTime).getOrElse("")}",
               "Permissions" : ${consentViews.mkString("[\"", "\",\"", "\"]")},
               "ConsentId" : "${consent.consentId}",
-              "TransactionFromDateTime" : "${consent.transactionFromDateTime}"
+              "TransactionFromDateTime" : "${Option(consent.transactionFromDateTime).getOrElse("")}"
             }
           }""")
         }

obp-api/src/main/scala/code/api/UKOpenBanking/v3_1_0/JSONFactory_UKOpenBanking_310.scala

@@ -204,11 +204,13 @@ object JSONFactory_UKOpenBanking_310 extends CustomJsonFormats {
     Links: LinksV310,
     Risk: String
   )
+  // The three datetimes are 0..1 (open-ended if absent) per the UK spec's OBReadConsent1 —
+  // shared by the v3.1.0 and v4.0.1 consent-creation handlers.
   case class ConsentPostBodyDataUKV310(
-    TransactionToDateTime: String,
-    ExpirationDateTime: String,
+    TransactionToDateTime: Option[String],
+    ExpirationDateTime: Option[String],
     Permissions: List[String],
-    TransactionFromDateTime: String
+    TransactionFromDateTime: Option[String]
   )
   case class ConsentPostBodyUKV310(
     Data: ConsentPostBodyDataUKV310,

obp-api/src/main/scala/code/api/UKOpenBanking/v4_0_1/Http4sUKOBv401.scala

@@ -2,11 +2,13 @@ package code.api.UKOpenBanking.v4_0_1
 
 import cats.data.{Kleisli, OptionT}
 import cats.effect._
+import code.api.util.APIUtil
 import code.api.util.APIUtil.ResourceDoc
 import code.api.util.http4s.ResourceDocMiddleware
 import code.util.Helper.MdcLoggable
 import com.openbankproject.commons.util.ApiVersion
 import org.http4s._
+import org.typelevel.ci.CIString
 
 import scala.collection.mutable.ArrayBuffer
 
@@ -46,5 +48,19 @@ object Http4sUKOBv401 extends MdcLoggable {
       .orElse(Http4sUKOBv401Vrp.routes(req))
   }
 
-  val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allRoutes)
+  private val fapiInteractionIdHeader = CIString("x-fapi-interaction-id")
+
+  // FAPI: every response carries x-fapi-interaction-id -- echoed from the request if the TPP sent
+  // one (for end-to-end tracing), otherwise generated as a fresh UUID so tracing works even when
+  // the TPP omits it. Applied as an outer middleware so it covers every UK v4.0.1 response,
+  // including error responses produced inside ResourceDocMiddleware.
+  private def withFapiInteractionId(routes: HttpRoutes[IO]): HttpRoutes[IO] =
+    Kleisli[HttpF, Request[IO], Response[IO]] { req =>
+      val interactionId = req.headers.get(fapiInteractionIdHeader)
+        .map(_.head.value)
+        .getOrElse(APIUtil.generateUUID())
+      routes(req).map(_.putHeaders(Header.Raw(fapiInteractionIdHeader, interactionId)))
+    }
+
+  val wrappedRoutes: HttpRoutes[IO] = withFapiInteractionId(ResourceDocMiddleware.apply(resourceDocs)(allRoutes))
 }

obp-api/src/main/scala/code/api/UKOpenBanking/v4_0_1/Http4sUKOBv401AccountInfo.scala

@@ -5,11 +5,11 @@ import cats.effect.IO
 import code.api.APIFailureNewStyle
 import code.api.Constant
 import code.api.UKOpenBanking.v3_1_0.JSONFactory_UKOpenBanking_310.ConsentPostBodyUKV310
-import code.api.util.APIUtil.{EmptyBody, ResourceDoc, HTTPParam, connectorEmptyResponse, createQueriesByHttpParams, defaultBankId, fullBoxOrException, passesPsd2Aisp, unboxFull, unboxFullOrFail, DateWithDayFormat}
+import code.api.util.APIUtil.{EmptyBody, ResourceDoc, HTTPParam, connectorEmptyResponse, createQueriesByHttpParams, defaultBankId, fullBoxOrException, passesPsd2Aisp, unboxFull, unboxFullOrFail, parseIso8601OrDayDate}
 import code.api.util.ApiTag
 import code.api.util.CallContext
 import code.api.util.CustomJsonFormats
-import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, ConsentNotFound, ConsentViewNotFund, UnknownError}
+import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, ConsentNotFound, ConsentViewNotFund, InvalidJsonFormat, UnknownError}
 import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps}
 import code.api.util.newstyle.ViewNewStyle
 import code.api.util.{APIUtil, ConsentJWT, JwtUtil, NewStyle}
@@ -94,24 +94,43 @@ object Http4sUKOBv401AccountInfo extends MdcLoggable {
       EndpointHelpers.executeFutureCreated(req) {
         implicit val cc: CallContext = req.callContext
         for {
-          u <- cc.user.toOption match {
-            case Some(user) => Future.successful(user)
-            case None       => Future.failed(new RuntimeException(AuthenticatedUserIsRequired))
-          }
+          // Spec Step 2: the TPP lodges the consent via a client-credentials grant -- authenticated
+          // as an app (consumer) but with no PSU yet. Require some authentication (a consumer or a
+          // user) but not a PSU specifically; reject only a fully anonymous request. The PSU is
+          // bound later at authorise time (mUserId stays null until then), mirroring the Berlin
+          // Group native consent flow (Http4sBGv13AIS.createConsent). A request with a real user
+          // (e.g. DirectLogin) still works -- createdByUser just carries it through.
+          _ <- if (cc.user.isEmpty && cc.consumer.isEmpty)
+                 Future.failed(new RuntimeException(AuthenticatedUserIsRequired))
+               else Future.successful(())
+          createdByUser = cc.user.toOption
           consentJson <- Future.fromTry(scala.util.Try(
             JsonAliases.parse(cc.httpBody.getOrElse("{}")).extract[ConsentPostBodyUKV310]
           ))
+          // Separate step (not inlined into the saveUKConsent call below) so a bad date string
+          // fails here -> 400. NewStyle.function.tryons (not Future.fromTry) is required for
+          // that: ErrorResponseConverter only special-cases APIFailureNewStyle to preserve a set
+          // HTTP code -- tryons wraps failures that way, a bare Future.fromTry(Try(...)) doesn't
+          // and falls through to unknownErrorToResponse, i.e. 500.
+          (expirationDateTime, transactionFromDateTime, transactionToDateTime) <- NewStyle.function.tryons(
+            s"$InvalidJsonFormat The Json body should have valid ISO-8601 ExpirationDateTime/TransactionFromDateTime/TransactionToDateTime values ", 400, Some(cc)) {
+            (
+              consentJson.Data.ExpirationDateTime.map(parseIso8601OrDayDate),
+              consentJson.Data.TransactionFromDateTime.map(parseIso8601OrDayDate),
+              consentJson.Data.TransactionToDateTime.map(parseIso8601OrDayDate)
+            )
+          }
           consumerId = cc.consumer.map(_.consumerId.get)
           _ <- passesPsd2Aisp(Some(cc))
           createdConsent <- Future(Consents.consentProvider.vend.saveUKConsent(
-            Some(u),
+            createdByUser,
             bankId = None,
             accountIds = None,
             consumerId = consumerId,
             permissions = consentJson.Data.Permissions,
-            expirationDateTime = DateWithDayFormat.parse(consentJson.Data.ExpirationDateTime),
-            transactionFromDateTime = DateWithDayFormat.parse(consentJson.Data.TransactionFromDateTime),
-            transactionToDateTime = DateWithDayFormat.parse(consentJson.Data.TransactionToDateTime),
+            expirationDateTime = expirationDateTime,
+            transactionFromDateTime = transactionFromDateTime,
+            transactionToDateTime = transactionToDateTime,
             apiStandard = Some("UKOpenBanking"),
             apiVersion = Some("4.0.1")
           )) map { i => connectorEmptyResponse(i, Some(cc)) }
@@ -195,9 +214,9 @@ object Http4sUKOBv401AccountInfo extends MdcLoggable {
             status = consent.status,
             statusUpdateDateTime = consent.statusUpdateDateTime.toString,
             permissions = consentViews,
-            expirationDateTime = consent.expirationDateTime.toString,
-            transactionFromDateTime = consent.transactionFromDateTime.toString,
-            transactionToDateTime = consent.transactionToDateTime.toString,
+            expirationDateTime = Option(consent.expirationDateTime).map(_.toString),
+            transactionFromDateTime = Option(consent.transactionFromDateTime).map(_.toString),
+            transactionToDateTime = Option(consent.transactionToDateTime).map(_.toString),
             selfPath = s"/aisp/account-access-consents/$consentId"
           )
         }
@@ -2236,7 +2255,7 @@ object Http4sUKOBv401AccountInfo extends MdcLoggable {
             transactions.map(_.id),
             view.viewId,
             Some(cc))
-        } yield JSONFactory_UKOpenBanking_401.createTransactionsJsonNew(account.bankId, transactions, moderatedAttributes, view)
+        } yield JSONFactory_UKOpenBanking_401.createTransactionsJsonNew(account.bankId, accountId.value, transactions, moderatedAttributes, view)
       }
   }
   resourceDocs += ResourceDoc(
@@ -2305,6 +2324,8 @@ object Http4sUKOBv401AccountInfo extends MdcLoggable {
     case req @ GET -> `ukV401Prefix` / "aisp" / "balances" =>
       EndpointHelpers.withUser(req) { (u, cc) =>
         for {
+          _ <- NewStyle.function.checkUKConsent(u, Some(cc))
+          _ <- passesPsd2Aisp(Some(cc))
           availablePrivateAccounts <- Views.views.vend.getPrivateBankAccountsFuture(u)
           (accounts, _) <- NewStyle.function.getBankAccounts(availablePrivateAccounts, Some(cc))
         } yield JSONFactory_UKOpenBanking_401.createBalancesJSON(accounts)
@@ -3446,6 +3467,8 @@ object Http4sUKOBv401AccountInfo extends MdcLoggable {
     case req @ GET -> `ukV401Prefix` / "aisp" / "transactions" =>
       EndpointHelpers.withUser(req) { (u, cc) =>
         for {
+          _ <- NewStyle.function.checkUKConsent(u, Some(cc))
+          _ <- passesPsd2Aisp(Some(cc))
           (bank, _) <- NewStyle.function.getBank(BankId(defaultBankId), Some(cc))
           availablePrivateAccounts <- Views.views.vend.getPrivateBankAccountsFuture(u)
           (accounts, _) <- NewStyle.function.getBankAccounts(availablePrivateAccounts, Some(cc))