Skip to content

Commit d2cb052

Browse files
levkklevkk
authored
chore: resharding integration test (#877)
Add an integration test that tests resharding and replication end-to-end, incl. that all rows are copied and replicated correctly.
1 parent 096e210 commit d2cb052

8 files changed

Lines changed: 417 additions & 0 deletions

File tree

‎.rwx/integration.yml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,11 @@ tasks:
103103
python3-venv \
104104
python3-virtualenv \
105105
ruby-full
106+
sudo install -d /usr/share/keyrings
107+
curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /usr/share/keyrings/postgresql.gpg
108+
echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "${VERSION_CODENAME}")-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null
109+
sudo apt-get update
110+
sudo apt-get install -y postgresql-client-18
106111
sudo apt-get remove -y cmake || true
107112
sudo pip3 install --break-system-packages cmake==3.31.6
108113
cmake --version
@@ -430,6 +435,33 @@ tasks:
430435
- key: lcov-copy-data
431436
path: copy-data.lcov
432437

438+
- key: integration-resharding
439+
use: integration-build-pgdog-cov
440+
docker: true
441+
timeout: 20m
442+
agent:
443+
cpus: 4
444+
memory: 16gb
445+
run: |
446+
export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/resharding-%p-%m.profraw"
447+
448+
cleanup() {
449+
(cd integration/resharding && docker-compose down >/dev/null 2>&1 || true)
450+
killall -TERM pgdog 2>/dev/null || true
451+
sleep 1
452+
killall -KILL pgdog 2>/dev/null || true
453+
}
454+
trap cleanup EXIT
455+
456+
timeout --signal=TERM --kill-after=90s 16m bash integration/resharding/dev.sh
457+
458+
cargo llvm-cov report --release --package pgdog --lcov --output-path resharding.lcov
459+
outputs:
460+
filesystem: false
461+
artifacts:
462+
- key: lcov-resharding
463+
path: resharding.lcov
464+
433465
- key: integration-python
434466
use: integration-build-pgdog-cov
435467
background-processes: *postgres-bg–processes
@@ -534,6 +566,7 @@ tasks:
534566
-a "$LCOV_TOXI" \
535567
-a "$LCOV_RUST" \
536568
-a "$LCOV_COPY_DATA" \
569+
-a "$LCOV_RESHARDING" \
537570
-a "$LCOV_PYTHON" \
538571
-a "$LCOV_LOAD_BALANCER" \
539572
-a "$LCOV_COMPLEX" \
@@ -551,6 +584,7 @@ tasks:
551584
LCOV_TOXI: ${{ tasks.integration-toxi.artifacts.lcov-toxi }}
552585
LCOV_RUST: ${{ tasks.integration-rust.artifacts.lcov-rust }}
553586
LCOV_COPY_DATA: ${{ tasks.integration-copy-data.artifacts.lcov-copy-data }}
587+
LCOV_RESHARDING: ${{ tasks.integration-resharding.artifacts.lcov-resharding }}
554588
LCOV_PYTHON: ${{ tasks.integration-python.artifacts.lcov-python }}
555589
LCOV_LOAD_BALANCER: ${{ tasks.integration-load-balancer.artifacts.lcov-load-balancer }}
556590
LCOV_COMPLEX: ${{ tasks.integration-complex.artifacts.lcov-complex }}

‎integration/resharding/dev.sh‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/bin/bash
2+
set -e
3+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
4+
DEFAULT_BIN="${SCRIPT_DIR}/../../target/debug/pgdog"
5+
PGDOG_BIN=${PGDOG_BIN:-$DEFAULT_BIN}
6+
7+
pushd ${SCRIPT_DIR}
8+
docker-compose down && docker-compose up -d
9+
10+
# Give it a second to boot up.
11+
# It restarts the DB during initialization.
12+
sleep 2
13+
14+
for port in 15432 15433 15434 15435; do
15+
echo "Waiting for database on port ${port}..."
16+
until PGPASSWORD=pgdog pg_isready -h 127.0.0.1 -p "${port}" -U pgdog -d postgres; do
17+
sleep 1
18+
done
19+
done
20+
21+
${PGDOG_BIN} &
22+
PGDOG_PID="$!"
23+
24+
export PGPASSWORD=pgdog
25+
export PGHOST=127.0.0.1
26+
export PGPORT=6432
27+
export PGUSER=pgdog
28+
29+
until psql source -c 'SELECT 1' 2> /dev/null; do
30+
sleep 1
31+
done
32+
33+
pgbench -f pgbench.sql -P 1 source -c 5 -t 1000000 &
34+
PGBENCH_PID="$!"
35+
36+
sleep 10
37+
38+
psql admin -c 'COPY_DATA source destination pgdog'
39+
40+
sleep 10
41+
42+
kill -TERM ${PGBENCH_PID}
43+
44+
replace_copy_with_replicate() {
45+
local table="$1"
46+
local column="$2"
47+
48+
psql source -c "UPDATE ${table} SET ${column} = regexp_replace(${column}, '-copy$', '-replicate') WHERE ${column} LIKE '%-copy';"
49+
}
50+
51+
replace_copy_with_replicate tenants name
52+
replace_copy_with_replicate accounts full_name
53+
replace_copy_with_replicate projects name
54+
replace_copy_with_replicate tasks title
55+
replace_copy_with_replicate task_comments body
56+
replace_copy_with_replicate settings name
57+
58+
wait_for_no_copy_rows() {
59+
local table="$1"
60+
local column="$2"
61+
62+
while true; do
63+
count=$(psql -d destination -tAc "SELECT COUNT(*) FROM ${table} WHERE ${column} LIKE '%-copy'")
64+
if [ "${count}" -eq 0 ]; then
65+
echo "${table}.${column}: replication caught up"
66+
break
67+
fi
68+
69+
echo "${table}.${column}: waiting for ${count} rows ending in -copy"
70+
sleep 1
71+
done
72+
}
73+
74+
wait_for_no_copy_rows tenants name
75+
wait_for_no_copy_rows accounts full_name
76+
wait_for_no_copy_rows projects name
77+
wait_for_no_copy_rows tasks title
78+
wait_for_no_copy_rows task_comments body
79+
wait_for_no_copy_rows settings name
80+
81+
check_row_count_matches() {
82+
local table="$1"
83+
local source_count
84+
local destination_count
85+
86+
source_count=$(psql -d source -tAc "SELECT COUNT(*) FROM ${table}")
87+
destination_count=$(psql -d destination -tAc "SELECT COUNT(*) FROM ${table}")
88+
89+
if [ "${source_count}" -ne "${destination_count}" ]; then
90+
echo "MISMATCH ${table}: source=${source_count} destination=${destination_count}"
91+
exit 1
92+
fi
93+
94+
echo "OK ${table}: ${source_count} rows"
95+
}
96+
97+
check_row_count_matches tenants
98+
check_row_count_matches accounts
99+
check_row_count_matches projects
100+
check_row_count_matches tasks
101+
check_row_count_matches task_comments
102+
check_row_count_matches settings
103+
104+
kill -TERM ${PGDOG_PID}
105+
docker-compose down

‎integration/resharding/docker-compose.yaml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
services:
2+
source_0:
3+
image: postgres:18
4+
command: postgres -c wal_level=logical
5+
environment:
6+
POSTGRES_USER: pgdog
7+
POSTGRES_PASSWORD: pgdog
8+
POSTGRES_DB: postgres
9+
volumes:
10+
- ./schema.sql:/docker-entrypoint-initdb.d/schema.sql
11+
ports:
12+
- 15432:5432
13+
networks:
14+
- postgres
15+
16+
source_1:
17+
image: postgres:18
18+
command: postgres -c wal_level=logical
19+
environment:
20+
POSTGRES_USER: pgdog
21+
POSTGRES_PASSWORD: pgdog
22+
POSTGRES_DB: postgres
23+
volumes:
24+
- ./schema.sql:/docker-entrypoint-initdb.d/schema.sql
25+
ports:
26+
- 15433:5432
27+
networks:
28+
- postgres
29+
30+
destination_0:
31+
image: postgres:18
32+
command: postgres -c wal_level=logical
33+
environment:
34+
POSTGRES_USER: pgdog
35+
POSTGRES_PASSWORD: pgdog
36+
POSTGRES_DB: postgres
37+
ports:
38+
- 15434:5432
39+
networks:
40+
- postgres
41+
42+
destination_1:
43+
image: postgres:18
44+
command: postgres -c wal_level=logical
45+
environment:
46+
POSTGRES_USER: pgdog
47+
POSTGRES_PASSWORD: pgdog
48+
POSTGRES_DB: postgres
49+
ports:
50+
- 15435:5432
51+
networks:
52+
- postgres
53+
54+
networks:
55+
postgres:

‎integration/resharding/pgbench.sql‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
\set id_seed random(1, 1000000000)
2+
3+
INSERT INTO tenants (id, name, billing_email)
4+
VALUES (
5+
:id_seed,
6+
'tenant-' || :id_seed || '-copy',
7+
'tenant-' || :id_seed || '-copy@example.test'
8+
)
9+
ON CONFLICT (id) DO NOTHING;
10+
11+
INSERT INTO accounts (id, tenant_id, email, full_name)
12+
VALUES (
13+
:id_seed,
14+
:id_seed,
15+
'account-' || :id_seed || '-copy@example.test',
16+
'account-' || :id_seed || '-copy'
17+
)
18+
ON CONFLICT (id) DO NOTHING;
19+
20+
INSERT INTO projects (id, tenant_id, owner_account_id, name, status)
21+
VALUES (
22+
:id_seed,
23+
:id_seed,
24+
:id_seed,
25+
'project-' || :id_seed || '-copy',
26+
'active'
27+
)
28+
ON CONFLICT (id) DO NOTHING;
29+
30+
INSERT INTO tasks (id, tenant_id, project_id, assignee_account_id, title, state)
31+
VALUES (
32+
:id_seed,
33+
:id_seed,
34+
:id_seed,
35+
:id_seed,
36+
'task-' || :id_seed || '-copy',
37+
'open'
38+
)
39+
ON CONFLICT (id) DO NOTHING;
40+
41+
INSERT INTO task_comments (id, tenant_id, task_id, author_account_id, body)
42+
VALUES (
43+
:id_seed,
44+
:id_seed,
45+
:id_seed,
46+
:id_seed,
47+
'comment-' || :id_seed || '-copy'
48+
)
49+
ON CONFLICT (id) DO NOTHING;
50+
51+
INSERT INTO settings (id, name, value)
52+
VALUES (
53+
:id_seed,
54+
'setting-' || :id_seed || '-copy',
55+
'value-' || :id_seed || '-copy'
56+
)
57+
ON CONFLICT (id) DO NOTHING;
58+
59+
UPDATE tenants
60+
SET name = 'tenant-' || :id_seed || '-replicate'
61+
WHERE id = :id_seed;
62+
63+
UPDATE accounts
64+
SET full_name = 'account-' || :id_seed || '-replicate'
65+
WHERE id = :id_seed;
66+
67+
UPDATE projects
68+
SET name = 'project-' || :id_seed || '-replicate'
69+
WHERE id = :id_seed;
70+
71+
UPDATE tasks
72+
SET title = 'task-' || :id_seed || '-replicate'
73+
WHERE id = :id_seed;
74+
75+
UPDATE task_comments
76+
SET body = 'comment-' || :id_seed || '-replicate'
77+
WHERE id = :id_seed;
78+
79+
UPDATE settings
80+
SET name = 'setting-' || :id_seed || '-replicate'
81+
WHERE id = :id_seed;

‎integration/resharding/pgdog.toml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
[[databases]]
2+
name = "source"
3+
host = "127.0.0.1"
4+
port = 15432
5+
shard = 0
6+
database_name = "postgres"
7+
8+
[[databases]]
9+
name = "source"
10+
host = "127.0.0.1"
11+
port = 15433
12+
shard = 1
13+
database_name = "postgres"
14+
15+
[[databases]]
16+
name = "destination"
17+
host = "127.0.0.1"
18+
port = 15434
19+
shard = 0
20+
database_name = "postgres"
21+
22+
[[databases]]
23+
name = "destination"
24+
host = "127.0.0.1"
25+
port = 15435
26+
shard = 1
27+
database_name = "postgres"
28+
29+
[[sharded_tables]]
30+
database = "source"
31+
name = "tenants"
32+
column = "id"
33+
data_type = "bigint"
34+
35+
[[sharded_tables]]
36+
database = "source"
37+
column = "tenant_id"
38+
data_type = "bigint"
39+
40+
[[sharded_tables]]
41+
database = "destination"
42+
name = "tenants"
43+
column = "id"
44+
data_type = "bigint"
45+
46+
[[sharded_tables]]
47+
database = "destination"
48+
column = "tenant_id"
49+
data_type = "bigint"
50+
51+
[admin]
52+
password = "pgdog"
53+
user = "pgdog"
54+
55+
[replication]
56+
pg_dump_path = "/usr/lib/postgresql/18/bin/pg_dump"

0 commit comments

Comments
 (0)