From c5493f8a168cbe748ed670dcd054023fbb7d391d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Estev=C3=A3o?= Date: Mon, 1 Jun 2026 12:26:10 -0300 Subject: [PATCH 1/7] fix: don't sed dump --- scripts/db-entrypoint.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/db-entrypoint.sh b/scripts/db-entrypoint.sh index 3355b8891..012c09f93 100755 --- a/scripts/db-entrypoint.sh +++ b/scripts/db-entrypoint.sh @@ -10,7 +10,11 @@ sed_vars () { cd /home/mysql/raw_entrypoint || exit for file in *.sql; do - sed_vars "$file" + if [[ "$file" == *dump* ]]; then + cp "$file" /home/mysql/entrypoint/"$file" + else + sed_vars "$file" + fi done cd ../entrypoint/ || exit From ea376a042e233cbf222f37e9a42a2e12a4a9ab32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Estev=C3=A3o?= Date: Mon, 1 Jun 2026 14:47:14 -0300 Subject: [PATCH 2/7] chore: improve testing from dump --- Makefile | 12 ++++++++++++ docker-compose-from-dump.yml | 4 ++++ scripts/db-entrypoint.sh | 14 ++++++++++++-- scripts/db/Dockerfile | 2 +- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f6542b34f..3ea9680dc 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,18 @@ dev-from-dump: $(touch_local_env) docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump up --build -d +stop-dev-from-dump: + docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump down + +reset-dev-from-dump: + make stop-dev-from-dump && make dev-from-dump + +reset-dev-from-dump-keep-db: + docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump up --build -d --force-recreate --no-deps paybutton + +nuke-dev-from-dump: + docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump down -v + logs-dev: docker logs -f paybutton-dev diff --git a/docker-compose-from-dump.yml b/docker-compose-from-dump.yml index da676c508..ee778cf27 100644 --- a/docker-compose-from-dump.yml +++ b/docker-compose-from-dump.yml @@ -42,6 +42,7 @@ services: - ./scripts/db/init-supertokens-db.sql:/home/mysql/raw_entrypoint/3-init-supertokens-db.sql - ./scripts/db/prisma-shadow-db-fix.sql:/home/mysql/raw_entrypoint/4-prisma-shadow-db-fix.sql - ./dump.sql:/home/mysql/raw_entrypoint/5-dump.sql + - paybutton-dump-db-data:/var/lib/mysql users-service: container_name: paybutton-users-service @@ -69,3 +70,6 @@ services: - ./redis:/data/redis:z - ./scripts:/data/scripts command: sh -c "sed -i 's/\\r//g' ./scripts/redis-start.sh && sh ./scripts/redis-start.sh" + +volumes: + paybutton-dump-db-data: diff --git a/scripts/db-entrypoint.sh b/scripts/db-entrypoint.sh index 012c09f93..3b34e3e76 100755 --- a/scripts/db-entrypoint.sh +++ b/scripts/db-entrypoint.sh @@ -11,8 +11,10 @@ cd /home/mysql/raw_entrypoint || exit for file in *.sql; do if [[ "$file" == *dump* ]]; then - cp "$file" /home/mysql/entrypoint/"$file" + echo "Linking dump file: $file" + ln -s /home/mysql/raw_entrypoint/"$file" /home/mysql/entrypoint/"$file" else + echo "Copying file $file after seding vars" sed_vars "$file" fi done @@ -20,5 +22,13 @@ done cd ../entrypoint/ || exit for file in *.sql; do - mariadb -u root -p"$MYSQL_ROOT_PASSWORD" < "$file" + if [[ "$file" == *dump* ]]; then + filesize=$(stat -c%s "$file" 2>/dev/null || stat -f%z "$file" 2>/dev/null) + echo "Importing $file ($(numfmt --to=iec $filesize)) ..." + pv "$file" | mariadb -u root -p"$MYSQL_ROOT_PASSWORD" + else + echo "Importing $file ..." + mariadb -u root -p"$MYSQL_ROOT_PASSWORD" < "$file" + fi + echo "Done importing $file" done diff --git a/scripts/db/Dockerfile b/scripts/db/Dockerfile index 2675f8ae6..b31899ec2 100644 --- a/scripts/db/Dockerfile +++ b/scripts/db/Dockerfile @@ -3,7 +3,7 @@ FROM mariadb:latest RUN mkhomedir_helper mysql RUN apt-get update || echo && \ - apt-get install -y gettext || echo + apt-get install -y gettext pv || echo RUN mkdir -p /home/mysql/raw_entrypoint && \ mkdir -p /home/mysql/entrypoint From 424bfd1a1ae888f78dcea6bf43b64cc7d3fc6840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Estev=C3=A3o?= Date: Mon, 1 Jun 2026 15:47:03 -0300 Subject: [PATCH 3/7] chore: improve testing from dump 2 --- .env.from-dump | 1 + scripts/db-entrypoint.sh | 2 +- scripts/paybutton-server-start.sh | 10 ++++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.env.from-dump b/.env.from-dump index 4e21a4685..4d4c9f905 100644 --- a/.env.from-dump +++ b/.env.from-dump @@ -1,2 +1,3 @@ IGNORE_SEEDING=true DONT_EXECUTE_TRIGGERS=true +FROM_DUMP=true diff --git a/scripts/db-entrypoint.sh b/scripts/db-entrypoint.sh index 3b34e3e76..9d2138e88 100755 --- a/scripts/db-entrypoint.sh +++ b/scripts/db-entrypoint.sh @@ -23,7 +23,7 @@ cd ../entrypoint/ || exit for file in *.sql; do if [[ "$file" == *dump* ]]; then - filesize=$(stat -c%s "$file" 2>/dev/null || stat -f%z "$file" 2>/dev/null) + filesize=$(stat -Lc%s "$file" 2>/dev/null || stat -Lf%z "$file" 2>/dev/null) echo "Importing $file ($(numfmt --to=iec $filesize)) ..." pv "$file" | mariadb -u root -p"$MYSQL_ROOT_PASSWORD" else diff --git a/scripts/paybutton-server-start.sh b/scripts/paybutton-server-start.sh index 1028c14fe..0884f3ca0 100755 --- a/scripts/paybutton-server-start.sh +++ b/scripts/paybutton-server-start.sh @@ -17,12 +17,18 @@ logtime=$(date +%Y-%m-%d@%H:%M) [ -e logs/jobs.log ] && mv logs/jobs.log logs/history/jobs_"$logtime".log [ -e logs/ws-server.log ] && mv logs/ws-server.log logs/history/ws-server_"$logtime".log -if [ "$ENVIRONMENT" = "production" ]; then +if [ "$FROM_DUMP" = "true" ]; then + yarn prisma generate || exit 1 +elif [ "$ENVIRONMENT" = "production" ]; then yarn prisma migrate deploy || exit 1 yarn prisma generate || exit 1 pm2 start yarn --time --interpreter ash --name jobs --output logs/jobs.log --error logs/jobs.log -- initJobs || exit 1 pm2 start yarn --time --interpreter ash --name WSServer --output logs/ws-server.log --error logs/ws-server.log -- initWSServer || exit 1 - pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- prod || exit 1 + if [ "$ENVIRONMENT" = "production" ]; then + pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- prod || exit 1 + else + pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- dev || exit 1 + fi else yarn prisma migrate dev || exit 1 yarn prisma db seed || exit 1 From e4d26160e32f0bc740541ce6a1a887ade59e0198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Estev=C3=A3o?= Date: Mon, 1 Jun 2026 21:00:56 -0300 Subject: [PATCH 4/7] fix: dev-from-dump setup (named volume, prisma deploy, pv progress) --- scripts/paybutton-server-start.sh | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/scripts/paybutton-server-start.sh b/scripts/paybutton-server-start.sh index 0884f3ca0..64a909685 100755 --- a/scripts/paybutton-server-start.sh +++ b/scripts/paybutton-server-start.sh @@ -17,23 +17,19 @@ logtime=$(date +%Y-%m-%d@%H:%M) [ -e logs/jobs.log ] && mv logs/jobs.log logs/history/jobs_"$logtime".log [ -e logs/ws-server.log ] && mv logs/ws-server.log logs/history/ws-server_"$logtime".log -if [ "$FROM_DUMP" = "true" ]; then - yarn prisma generate || exit 1 -elif [ "$ENVIRONMENT" = "production" ]; then +if [ "$FROM_DUMP" = "true" ] || [ "$ENVIRONMENT" = "production" ]; then yarn prisma migrate deploy || exit 1 yarn prisma generate || exit 1 - pm2 start yarn --time --interpreter ash --name jobs --output logs/jobs.log --error logs/jobs.log -- initJobs || exit 1 - pm2 start yarn --time --interpreter ash --name WSServer --output logs/ws-server.log --error logs/ws-server.log -- initWSServer || exit 1 - if [ "$ENVIRONMENT" = "production" ]; then - pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- prod || exit 1 - else - pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- dev || exit 1 - fi else yarn prisma migrate dev || exit 1 yarn prisma db seed || exit 1 - pm2 start yarn --time --interpreter ash --name jobs --output logs/jobs.log --error logs/jobs.log -- initJobs || exit 1 - pm2 start yarn --time --interpreter ash --name WSServer --output logs/ws-server.log --error logs/ws-server.log -- initWSServer || exit 1 +fi + +pm2 start yarn --time --interpreter ash --name jobs --output logs/jobs.log --error logs/jobs.log -- initJobs || exit 1 +pm2 start yarn --time --interpreter ash --name WSServer --output logs/ws-server.log --error logs/ws-server.log -- initWSServer || exit 1 +if [ "$ENVIRONMENT" = "production" ]; then + pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- prod || exit 1 +else pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- dev || exit 1 fi pm2 logs next From c05b008a9dbd1ba35da4b0333e4d6f938debe210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Estev=C3=A3o?= Date: Tue, 2 Jun 2026 08:36:55 -0300 Subject: [PATCH 5/7] feat: SKIP_CACHE_REBUILD to prevent I/O saturation with prod data --- .env.from-dump | 1 + redis/paymentCache.ts | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.env.from-dump b/.env.from-dump index 4d4c9f905..8381294e4 100644 --- a/.env.from-dump +++ b/.env.from-dump @@ -1,3 +1,4 @@ IGNORE_SEEDING=true DONT_EXECUTE_TRIGGERS=true FROM_DUMP=true +SKIP_CACHE_REBUILD=true diff --git a/redis/paymentCache.ts b/redis/paymentCache.ts index ad18d9f6a..fb6939445 100755 --- a/redis/paymentCache.ts +++ b/redis/paymentCache.ts @@ -31,9 +31,11 @@ export async function * getUserUncachedAddresses (userId: string): AsyncGenerato } export const getPaymentList = async (userId: string): Promise => { - const uncachedAddressStream = getUserUncachedAddresses(userId) - for await (const address of uncachedAddressStream) { - void await CacheSet.addressCreation(address) + if (process.env.SKIP_CACHE_REBUILD === undefined) { + const uncachedAddressStream = getUserUncachedAddresses(userId) + for await (const address of uncachedAddressStream) { + void await CacheSet.addressCreation(address) + } } return await getCachedPaymentsForUser(userId) } @@ -304,6 +306,7 @@ export const clearPaymentCacheForAddress = async (addressString: string): Promis } export const initPaymentCache = async (address: Address): Promise => { + if (process.env.SKIP_CACHE_REBUILD !== undefined) return false const cachedKeys = await getCachedWeekKeysForAddress(address.address) if (cachedKeys.length === 0) { await CacheSet.addressCreation(address) @@ -313,10 +316,12 @@ export const initPaymentCache = async (address: Address): Promise => { } export async function * getPaymentStream (userId: string): AsyncGenerator { - const uncachedAddressStream = getUserUncachedAddresses(userId) - for await (const address of uncachedAddressStream) { - console.log('[CACHE]: Creating cache for address', address.address) - await CacheSet.addressCreation(address) + if (process.env.SKIP_CACHE_REBUILD === undefined) { + const uncachedAddressStream = getUserUncachedAddresses(userId) + for await (const address of uncachedAddressStream) { + console.log('[CACHE]: Creating cache for address', address.address) + await CacheSet.addressCreation(address) + } } const userButtonIds: string[] = (await fetchPaybuttonArrayByUserId(userId)) .map(p => p.id) From 85a2ba26507f4f3d073df64188726b20eb89162b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Estev=C3=A3o?= Date: Tue, 2 Jun 2026 08:42:20 -0300 Subject: [PATCH 6/7] chore: ignore .forever --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2e84540ba..2f11a0ed9 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ prisma-local/seeds/productionTxs.csv paybutton-config.json dump.sql* +.forever From d9ba1448978f2aa0cb2473d80296eda673f5c3b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Estev=C3=A3o?= Date: Mon, 8 Jun 2026 10:50:00 -0300 Subject: [PATCH 7/7] fix: echo -> true --- scripts/db/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/db/Dockerfile b/scripts/db/Dockerfile index b31899ec2..b2c4bf841 100644 --- a/scripts/db/Dockerfile +++ b/scripts/db/Dockerfile @@ -2,8 +2,8 @@ FROM mariadb:latest RUN mkhomedir_helper mysql -RUN apt-get update || echo && \ - apt-get install -y gettext pv || echo +RUN apt-get update || true && \ + apt-get install -y gettext pv || true RUN mkdir -p /home/mysql/raw_entrypoint && \ mkdir -p /home/mysql/entrypoint