Skip to content

Fix: unstable ao vacuum icw test #376

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
my-ship-it merged 2 commits into from
Feb 2, 2024
Merged

Fix: unstable ao vacuum icw test #376

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5249ebd
Fix: unstable ao vacuum icw test
jiaqizho Jan 31, 2024
29bdf6d
fix bgwriter_checkpoint
jiaqizho Feb 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 20 additions & 13 deletions src/test/fsync/expected/bgwriter_checkpoint.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,19 @@ select gp_segment_id, pg_reload_conf() from gp_id union select gp_segment_id, pg
(4 rows)

begin;
create or replace function wait_until_dirty_buffer_flushed() returns text as $$
begin
for i in 1..60 loop
if ((select count(*) = 0 from dirty_buffers_on_qes() as (tablespace oid, database oid, relfilenode int8, block int)) AND
(select count(*) = 0 from dirty_buffers_on_qd() as (tablespace oid, database oid, relfilenode int8, block int))) then
return 'OK'; /* in func */
end if; /* in func */
perform pg_sleep(0.1); /* in func */
checkpoint; /* in func */
end loop; /* in func */
return 'Fail'; /* in func */
end; /* in func */
$$ language plpgsql;
create function num_dirty_on_qes(relid oid) returns setof bigint as
$$
declare
Expand Down Expand Up @@ -99,6 +112,13 @@ select gp_inject_fault_infinite('all', 'reset', dbid) from gp_segment_configurat

-- Start with a clean slate (no dirty buffers).
checkpoint;
-- Ensure no buffers are dirty before we start.
select wait_until_dirty_buffer_flushed();
wait_until_dirty_buffer_flushed
---------------------------------
OK
(1 row)

-- Skip checkpoints.
select gp_inject_fault_infinite('checkpoint', 'skip', dbid)
from gp_segment_configuration where role = 'p' and content > -1;
Expand All @@ -119,19 +139,6 @@ from gp_segment_configuration where role = 'p' and content > -1;
Success:
(3 rows)

-- Ensure no buffers are dirty before we start.
select * from dirty_buffers_on_qd()
as (tablespace oid, database oid, relfilenode int8, block int);
tablespace | database | relfilenode | block
------------+----------+-------------+-------
(0 rows)

select * from dirty_buffers_on_qes()
as (tablespace oid, database oid, relfilenode int8, block int);
tablespace | database | relfilenode | block
------------+----------+-------------+-------
(0 rows)

-- Make buffers dirty. At least two relfiles must be sync'ed during
-- next checkpoint.
insert into fsync_test1 select i, i from generate_series(1,1000)i;
Expand Down
23 changes: 17 additions & 6 deletions src/test/fsync/sql/bgwriter_checkpoint.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,20 @@ alter system set autovacuum = off;
select gp_segment_id, pg_reload_conf() from gp_id union select gp_segment_id, pg_reload_conf() from gp_dist_random('gp_id');

begin;
create or replace function wait_until_dirty_buffer_flushed() returns text as $$
begin
for i in 1..60 loop
if ((select count(*) = 0 from dirty_buffers_on_qes() as (tablespace oid, database oid, relfilenode int8, block int)) AND
(select count(*) = 0 from dirty_buffers_on_qd() as (tablespace oid, database oid, relfilenode int8, block int))) then
return 'OK'; /* in func */
end if; /* in func */
perform pg_sleep(0.1); /* in func */
checkpoint; /* in func */
end loop; /* in func */
return 'Fail'; /* in func */
end; /* in func */
$$ language plpgsql;

create function num_dirty_on_qes(relid oid) returns setof bigint as
$$
declare
Expand Down Expand Up @@ -86,6 +100,9 @@ select gp_inject_fault_infinite('all', 'reset', dbid) from gp_segment_configurat
-- Start with a clean slate (no dirty buffers).
checkpoint;

-- Ensure no buffers are dirty before we start.
select wait_until_dirty_buffer_flushed();

-- Skip checkpoints.
select gp_inject_fault_infinite('checkpoint', 'skip', dbid)
from gp_segment_configuration where role = 'p' and content > -1;
Expand All @@ -94,12 +111,6 @@ from gp_segment_configuration where role = 'p' and content > -1;
select gp_inject_fault_infinite('fault_in_background_writer_main', 'suspend', dbid)
from gp_segment_configuration where role = 'p' and content > -1;

-- Ensure no buffers are dirty before we start.
select * from dirty_buffers_on_qd()
as (tablespace oid, database oid, relfilenode int8, block int);
select * from dirty_buffers_on_qes()
as (tablespace oid, database oid, relfilenode int8, block int);

-- Make buffers dirty. At least two relfiles must be sync'ed during
-- next checkpoint.
insert into fsync_test1 select i, i from generate_series(1,1000)i;
Expand Down
86 changes: 58 additions & 28 deletions src/test/regress/expected/uao_catalog_tables.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,11 +220,14 @@ select count(*) from uao_table_tupcount_changes_after_delete;
(1 row)

vacuum full uao_table_tupcount_changes_after_delete;
select sum(tupcount) from gp_toolkit.__gp_aoseg('uao_table_tupcount_changes_after_delete');
sum
-----
9
(1 row)
select segno,hidden_tupcount from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_tupcount_changes_after_delete'::regclass);
segno | hidden_tupcount
-------+-----------------
1 | 0
1 | 0
1 | 0
2 | 0
(4 rows)

select count(*) from uao_table_tupcount_changes_after_delete;
count
Expand Down Expand Up @@ -262,11 +265,14 @@ select count(*) from uao_table_tupcount_changes_after_update;
(1 row)

vacuum full uao_table_tupcount_changes_after_update;
select sum(tupcount) from gp_toolkit.__gp_aoseg('uao_table_tupcount_changes_after_update');
sum
-----
10
(1 row)
select segno,hidden_tupcount from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_tupcount_changes_after_update'::regclass);
segno | hidden_tupcount
-------+-----------------
1 | 0
1 | 0
1 | 0
2 | 0
(4 rows)

select count(*) from uao_table_tupcount_changes_after_update;
count
Expand Down Expand Up @@ -295,17 +301,24 @@ select * from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_
(3 rows)

vacuum full uao_table_check_hidden_tup_count_after_delete;
select * from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_count_after_delete'::regclass);
segno | hidden_tupcount | total_tupcount
-------+-----------------+----------------
1 | 0 | 0
2 | 0 | 0
1 | 0 | 0
2 | 0 | 0
1 | 0 | 0
2 | 0 | 0
select segno,hidden_tupcount from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_count_after_delete'::regclass);
segno | hidden_tupcount
-------+-----------------
1 | 0
2 | 0
1 | 0
2 | 0
1 | 0
2 | 0
(6 rows)

select * from uao_table_check_hidden_tup_count_after_delete;
NOTICE: One or more columns in the following table(s) do not have statistics: uao_table_check_hidden_tup_count_after_delete
HINT: For non-partitioned tables, run analyze <table_name>(<column_list>). For partitioned tables, run analyze rootpartition <table_name>(<column_list>). See log for columns missing statistics.
i | j | k
---+---+---
(0 rows)

-- Verify the hidden tup_count using UDF gp_aovisimap_hidden_info(oid) for uao relation after update and vacuum
create table uao_table_check_hidden_tup_count_after_update(i int, j varchar(20), k int ) with (appendonly=true) DISTRIBUTED BY (i);
insert into uao_table_check_hidden_tup_count_after_update select i,'aa'||i,i+10 from generate_series(1,10) as i;
Expand All @@ -327,14 +340,31 @@ select * from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_
(3 rows)

vacuum full uao_table_check_hidden_tup_count_after_update;
select * from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_count_after_update'::regclass);
segno | hidden_tupcount | total_tupcount
-------+-----------------+----------------
1 | 0 | 0
2 | 0 | 5
1 | 0 | 0
2 | 0 | 4
1 | 0 | 0
2 | 0 | 1
select segno,hidden_tupcount from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_count_after_update'::regclass);
segno | hidden_tupcount
-------+-----------------
1 | 0
2 | 0
1 | 0
2 | 0
1 | 0
2 | 0
(6 rows)

select * from uao_table_check_hidden_tup_count_after_update;
NOTICE: One or more columns in the following table(s) do not have statistics: uao_table_check_hidden_tup_count_after_update
HINT: For non-partitioned tables, run analyze <table_name>(<column_list>). For partitioned tables, run analyze rootpartition <table_name>(<column_list>). See log for columns missing statistics.
i | j | k
----+--------+----
5 | test21 | 15
6 | test21 | 16
9 | test21 | 19
10 | test21 | 20
2 | test21 | 12
3 | test21 | 13
4 | test21 | 14
7 | test21 | 17
8 | test21 | 18
1 | test21 | 11
(10 rows)

65 changes: 49 additions & 16 deletions src/test/regress/expected/uaocs_catalog_tables.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,31 @@ select * from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tu
(3 rows)

vacuum full uaocs_table_check_hidden_tup_count_after_delete;
select * from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tup_count_after_delete'::regclass);
segno | hidden_tupcount | total_tupcount
-------+-----------------+----------------
1 | 0 | 5
1 | 0 | 0
2 | 0 | 0
1 | 0 | 4
select segno,hidden_tupcount from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tup_count_after_delete'::regclass);
segno | hidden_tupcount
-------+-----------------
1 | 0
2 | 0
1 | 0
1 | 0
(4 rows)

select * from uaocs_table_check_hidden_tup_count_after_delete;
NOTICE: One or more columns in the following table(s) do not have statistics: uaocs_table_check_hidden_tup_count_after_delete
HINT: For non-partitioned tables, run analyze <table_name>(<column_list>). For partitioned tables, run analyze rootpartition <table_name>(<column_list>). See log for columns missing statistics.
i | j | k
----+------+----
2 | aa2 | 12
3 | aa3 | 13
4 | aa4 | 14
7 | aa7 | 17
8 | aa8 | 18
5 | aa5 | 15
6 | aa6 | 16
9 | aa9 | 19
10 | aa10 | 20
(9 rows)

-- Verify the hidden tup_count using UDF gp_aovisimap_hidden_info(oid) for uaocs relation after update and vacuum
create table uaocs_table_check_hidden_tup_count_after_update(i int, j varchar(20), k int ) with (appendonly=true, orientation=column) DISTRIBUTED BY (i);
insert into uaocs_table_check_hidden_tup_count_after_update select i,'aa'||i,i+10 from generate_series(1,10) as i;
Expand All @@ -75,14 +91,31 @@ select * from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tu
(3 rows)

vacuum full uaocs_table_check_hidden_tup_count_after_update;
select * from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tup_count_after_update'::regclass);
segno | hidden_tupcount | total_tupcount
-------+-----------------+----------------
1 | 0 | 0
2 | 0 | 4
1 | 0 | 0
2 | 0 | 5
1 | 0 | 0
2 | 0 | 1
select segno,hidden_tupcount from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tup_count_after_update'::regclass);
segno | hidden_tupcount
-------+-----------------
1 | 0
2 | 0
1 | 0
2 | 0
1 | 0
2 | 0
(6 rows)

select * from uaocs_table_check_hidden_tup_count_after_update;
NOTICE: One or more columns in the following table(s) do not have statistics: uaocs_table_check_hidden_tup_count_after_update
HINT: For non-partitioned tables, run analyze <table_name>(<column_list>). For partitioned tables, run analyze rootpartition <table_name>(<column_list>). See log for columns missing statistics.
i | j | k
----+-------------+----
5 | test_update | 15
6 | test_update | 16
9 | test_update | 19
10 | test_update | 20
2 | test_update | 12
3 | test_update | 13
4 | test_update | 14
7 | test_update | 17
8 | test_update | 18
1 | test_update | 11
(10 rows)

3 changes: 3 additions & 0 deletions src/test/regress/greenplum_schedule
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,9 @@ test: disable_autovacuum
# Run uao[cs]_catalog_tables separately. They run VACUUM FULL on
# append-optimized tables and assume that no AWAITING_DROP segfiles exist at
# the end of VACUUM FULL.

# uao_catalog_tables/uaocs_catalog_tables/uao_compaction/threshold/uaocs_compaction/threshold
# should always be below disable_autovacuum, don't change the order.
test: uao_catalog_tables
test: uaocs_catalog_tables
test: uao_compaction/threshold
Expand Down
6 changes: 6 additions & 0 deletions src/test/regress/sql/disable_autovacuum.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,8 @@
alter system set autovacuum = off;
select gp_segment_id, pg_reload_conf() from gp_id union select gp_segment_id, pg_reload_conf() from gp_dist_random('gp_id');
-- start_ignore
-- The reason for restarting cbdb here is that if the subsequent
-- vacuum ao test(uao*_catalog_tables/threshold) encounters an
-- unfinished transaction, it will fail.
\!gpstop -ari
-- end_ignore
11 changes: 7 additions & 4 deletions src/test/regress/sql/uao_catalog_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ delete from uao_table_tupcount_changes_after_delete where i = 1;
select sum(tupcount) from gp_toolkit.__gp_aoseg('uao_table_tupcount_changes_after_delete');
select count(*) from uao_table_tupcount_changes_after_delete;
vacuum full uao_table_tupcount_changes_after_delete;
select sum(tupcount) from gp_toolkit.__gp_aoseg('uao_table_tupcount_changes_after_delete');
select segno,hidden_tupcount from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_tupcount_changes_after_delete'::regclass);
select count(*) from uao_table_tupcount_changes_after_delete;

-- Verify the tupcount changes in pg_aoseg when updating uao table
Expand All @@ -107,7 +107,7 @@ update uao_table_tupcount_changes_after_update set j=j||'test11' where i = 1;
select sum(tupcount) from gp_toolkit.__gp_aoseg('uao_table_tupcount_changes_after_update');
select count(*) from uao_table_tupcount_changes_after_update;
vacuum full uao_table_tupcount_changes_after_update;
select sum(tupcount) from gp_toolkit.__gp_aoseg('uao_table_tupcount_changes_after_update');
select segno,hidden_tupcount from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_tupcount_changes_after_update'::regclass);
select count(*) from uao_table_tupcount_changes_after_update;

-- Verify the hidden tup_count using UDF gp_aovisimap_hidden_info(oid) for uao relation after delete and vacuum
Expand All @@ -117,7 +117,8 @@ select * from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_
delete from uao_table_check_hidden_tup_count_after_delete;
select * from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_count_after_delete'::regclass);
vacuum full uao_table_check_hidden_tup_count_after_delete;
select * from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_count_after_delete'::regclass);
select segno,hidden_tupcount from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_count_after_delete'::regclass);
select * from uao_table_check_hidden_tup_count_after_delete;

-- Verify the hidden tup_count using UDF gp_aovisimap_hidden_info(oid) for uao relation after update and vacuum
create table uao_table_check_hidden_tup_count_after_update(i int, j varchar(20), k int ) with (appendonly=true) DISTRIBUTED BY (i);
Expand All @@ -126,4 +127,6 @@ select * from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_
update uao_table_check_hidden_tup_count_after_update set j = 'test21';
select * from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_count_after_update'::regclass);
vacuum full uao_table_check_hidden_tup_count_after_update;
select * from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_count_after_update'::regclass);
select segno,hidden_tupcount from gp_toolkit.__gp_aovisimap_hidden_info('uao_table_check_hidden_tup_count_after_update'::regclass);
select * from uao_table_check_hidden_tup_count_after_update;

6 changes: 4 additions & 2 deletions src/test/regress/sql/uaocs_catalog_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ select * from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tu
delete from uaocs_table_check_hidden_tup_count_after_delete where i = 1;
select * from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tup_count_after_delete'::regclass);
vacuum full uaocs_table_check_hidden_tup_count_after_delete;
select * from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tup_count_after_delete'::regclass);
select segno,hidden_tupcount from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tup_count_after_delete'::regclass);
select * from uaocs_table_check_hidden_tup_count_after_delete;

-- Verify the hidden tup_count using UDF gp_aovisimap_hidden_info(oid) for uaocs relation after update and vacuum
create table uaocs_table_check_hidden_tup_count_after_update(i int, j varchar(20), k int ) with (appendonly=true, orientation=column) DISTRIBUTED BY (i);
Expand All @@ -37,4 +38,5 @@ select * from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tu
update uaocs_table_check_hidden_tup_count_after_update set j = 'test_update';
select * from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tup_count_after_update'::regclass);
vacuum full uaocs_table_check_hidden_tup_count_after_update;
select * from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tup_count_after_update'::regclass);
select segno,hidden_tupcount from gp_toolkit.__gp_aovisimap_hidden_info('uaocs_table_check_hidden_tup_count_after_update'::regclass);
select * from uaocs_table_check_hidden_tup_count_after_update;