Skip to content

Add Iceberg lake-table DDL: CREATE ICEBERG TABLE, FOREIGN CATALOG, FOREIGN VOLUME (kernel scaffolding) #1842

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MisterRaindrop wants to merge 16 commits into
base: main
Choose a base branch
from
+2,641 −7
Open

Add Iceberg lake-table DDL: CREATE ICEBERG TABLE, FOREIGN CATALOG, FOREIGN VOLUME (kernel scaffolding) #1842

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
a8e5f53
catalog: add pg_foreign_catalog, pg_foreign_volume, pg_lake_table
MisterRaindrop Jul 2, 2026
c2bb7c0
nodes: add parse nodes for Iceberg lake-table DDL
MisterRaindrop Jul 2, 2026
b028fb8
parser: add grammar for CREATE ICEBERG TABLE / FOREIGN CATALOG / VOLUME
MisterRaindrop Jul 2, 2026
5a679b2
commands: implement CREATE/DROP FOREIGN CATALOG and FOREIGN VOLUME
MisterRaindrop Jul 2, 2026
b4704be
commands: implement CREATE ICEBERG TABLE
MisterRaindrop Jul 2, 2026
a81bd6b
bin: pg_dump skip and psql tab completion for lake-table DDL
MisterRaindrop Jul 2, 2026
73172e8
tests: add lake_table regression test, refresh catalog expecteds
MisterRaindrop Jul 2, 2026
5be5f9f
Use ASF license header for new lake-table source files
MisterRaindrop Jul 6, 2026
272d910
Move ASF license header to top of file header block
MisterRaindrop Jul 6, 2026
c70fbf9
foreigncmds: flatten IF NOT EXISTS duplicate handling
MisterRaindrop Jul 7, 2026
5105d02
parser/commands: unify lake-table DROP syntax with CREATE
MisterRaindrop Jul 10, 2026
f21763c
commands: make foreign catalog type a required first-class column
MisterRaindrop Jul 10, 2026
7e6e7ca
Update src/backend/foreign/foreign.c
MisterRaindrop Jul 10, 2026
70b451a
Update src/include/commands/laketablecmds.h
MisterRaindrop Jul 10, 2026
32bda14
psql: complete only iceberg tables for DROP ICEBERG TABLE
MisterRaindrop Jul 10, 2026
f6d3286
commands: finish ValidateLakeTableOptions -> ValidateLakeTableStmt re…
MisterRaindrop Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions contrib/pax_storage/src/test/regress/expected/oidjoins.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,10 @@ NOTICE: checking pg_foreign_data_wrapper {fdwhandler} => pg_proc {oid}
NOTICE: checking pg_foreign_data_wrapper {fdwvalidator} => pg_proc {oid}
NOTICE: checking pg_foreign_server {srvowner} => pg_authid {oid}
NOTICE: checking pg_foreign_server {srvfdw} => pg_foreign_data_wrapper {oid}
NOTICE: checking pg_foreign_catalog {fcowner} => pg_authid {oid}
NOTICE: checking pg_foreign_catalog {fcserver} => pg_foreign_server {oid}
NOTICE: checking pg_foreign_volume {fvowner} => pg_authid {oid}
NOTICE: checking pg_foreign_volume {fvserver} => pg_foreign_server {oid}
Comment thread
MisterRaindrop marked this conversation as resolved.
NOTICE: checking pg_user_mapping {umuser} => pg_authid {oid}
NOTICE: checking pg_user_mapping {umserver} => pg_foreign_server {oid}
NOTICE: checking pg_compression {compconstructor} => pg_proc {oid}
Expand All @@ -247,6 +251,9 @@ NOTICE: checking pg_foreign_table {ftrelid} => pg_class {oid}
NOTICE: checking pg_foreign_table {ftserver} => pg_foreign_server {oid}
NOTICE: checking pg_foreign_table_seg {ftsrelid} => pg_class {oid}
NOTICE: checking pg_foreign_table_seg {ftsserver} => pg_foreign_server {oid}
NOTICE: checking pg_lake_table {ltrelid} => pg_class {oid}
NOTICE: checking pg_lake_table {ltforeign_catalog} => pg_foreign_catalog {oid}
NOTICE: checking pg_lake_table {ltforeign_volume} => pg_foreign_volume {oid}
NOTICE: checking pg_policy {polrelid} => pg_class {oid}
NOTICE: checking pg_policy {polroles} => pg_authid {oid}
NOTICE: checking pg_default_acl {defaclrole} => pg_authid {oid}
Expand Down
3 changes: 3 additions & 0 deletions contrib/pax_storage/src/test/regress/expected/sanity_check.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,13 +112,16 @@ pg_enum|t
pg_event_trigger|t
pg_extension|t
pg_extprotocol|t
pg_foreign_catalog|t
pg_foreign_data_wrapper|t
pg_foreign_server|t
pg_foreign_table|t
pg_foreign_table_seg|t
pg_foreign_volume|t
pg_index|t
pg_inherits|t
pg_init_privs|t
pg_lake_table|t
pg_language|t
pg_largeobject|t
pg_largeobject_metadata|t
Expand Down
1 change: 1 addition & 0 deletions src/backend/catalog/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ CATALOG_HEADERS := \
pg_subscription_rel.h gp_partition_template.h pg_task.h pg_task_run_history.h \
pg_profile.h pg_password_history.h pg_directory_table.h gp_storage_server.h \
gp_storage_user_mapping.h pg_tag.h pg_tag_description.h \
pg_foreign_catalog.h pg_foreign_volume.h pg_lake_table.h \
gp_matview_aux.h \
gp_matview_tables.h

Expand Down
12 changes: 12 additions & 0 deletions src/backend/catalog/aclchk.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3050,6 +3050,12 @@ aclcheck_error(AclResult aclerr, ObjectType objtype,
case OBJECT_FOREIGN_SERVER:
msg = gettext_noop("permission denied for foreign server %s");
break;
case OBJECT_FOREIGN_CATALOG:
msg = gettext_noop("permission denied for foreign catalog %s");
break;
case OBJECT_FOREIGN_VOLUME:
msg = gettext_noop("permission denied for foreign volume %s");
break;
case OBJECT_FOREIGN_TABLE:
msg = gettext_noop("permission denied for foreign table %s");
break;
Expand Down Expand Up @@ -3198,6 +3204,12 @@ aclcheck_error(AclResult aclerr, ObjectType objtype,
case OBJECT_FOREIGN_SERVER:
msg = gettext_noop("must be owner of foreign server %s");
break;
case OBJECT_FOREIGN_CATALOG:
msg = gettext_noop("must be owner of foreign catalog %s");
break;
case OBJECT_FOREIGN_VOLUME:
msg = gettext_noop("must be owner of foreign volume %s");
break;
case OBJECT_FOREIGN_TABLE:
msg = gettext_noop("must be owner of foreign table %s");
break;
Expand Down
12 changes: 12 additions & 0 deletions src/backend/catalog/dependency.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,10 @@
#include "catalog/pg_directory_table.h"
#include "catalog/pg_event_trigger.h"
#include "catalog/pg_extension.h"
#include "catalog/pg_foreign_catalog.h"
#include "catalog/pg_foreign_data_wrapper.h"
#include "catalog/pg_foreign_server.h"
#include "catalog/pg_foreign_volume.h"
#include "catalog/pg_init_privs.h"
#include "catalog/pg_language.h"
#include "catalog/pg_largeobject.h"
Expand Down Expand Up @@ -226,6 +228,8 @@ static const Oid object_classes[] = {
ExtprotocolRelationId, /* OCLASS_EXTPROTOCOL */
GpMatviewAuxId, /* OCLASS_MATVIEW_AUX */
TaskRelationId, /* OCLASS_TASK */
ForeignCatalogRelationId, /* OCLASS_FOREIGN_CATALOG */
ForeignVolumeRelationId, /* OCLASS_FOREIGN_VOLUME */
};

/*
Expand Down Expand Up @@ -1629,6 +1633,8 @@ doDeletion(const ObjectAddress *object, int flags)
case OCLASS_TSTEMPLATE:
case OCLASS_FDW:
case OCLASS_FOREIGN_SERVER:
case OCLASS_FOREIGN_CATALOG:
case OCLASS_FOREIGN_VOLUME:
case OCLASS_USER_MAPPING:
case OCLASS_DEFACL:
case OCLASS_EVENT_TRIGGER:
Expand Down Expand Up @@ -3141,6 +3147,12 @@ getObjectClass(const ObjectAddress *object)
case TagDescriptionRelationId:
return OCLASS_TAG_DESCRIPTION;

case ForeignCatalogRelationId:
return OCLASS_FOREIGN_CATALOG;

case ForeignVolumeRelationId:
return OCLASS_FOREIGN_VOLUME;

default:
{
struct CustomObjectClass *coc;
Expand Down
5 changes: 5 additions & 0 deletions src/backend/catalog/heap.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
#include "catalog/storage.h"
#include "catalog/storage_directory_table.h"
#include "catalog/storage_xlog.h"
#include "commands/laketablecmds.h"
#include "commands/tablecmds.h"
#include "commands/typecmds.h"
#include "miscadmin.h"
Expand Down Expand Up @@ -2322,6 +2323,10 @@ heap_drop_with_catalog(Oid relid)
*/
CheckTableForSerializableConflictIn(rel);

/* If this is a lake table, remove its pg_lake_table entry */
if (RelationIsIcebergTable(rel))
RemoveLakeTableEntry(relid);

@yjhjstz yjhjstz Jul 9, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can RemoveLakeTableEntry put into extension via object_access_hook ?

@MisterRaindrop MisterRaindrop Jul 10, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can migrate this logic to object_access_hook, but the issue is that the AM layer hasn’t been implemented yet on my end, which blocks the migration.
So do I need to implement a default plugin for the AM layer first? Then replace this default AM plugin later once the official datalake plugin is fully rolled out?

@yjhjstz yjhjstz Jul 10, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if you can provide a draft version, that's good.


/*
* Delete pg_foreign_table tuple first.
*/
Expand Down
154 changes: 154 additions & 0 deletions src/backend/catalog/objectaddress.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,10 @@
#include "catalog/pg_event_trigger.h"
#include "catalog/pg_extension.h"
#include "catalog/pg_extprotocol.h"
#include "catalog/pg_foreign_catalog.h"
#include "catalog/pg_foreign_data_wrapper.h"
#include "catalog/pg_foreign_server.h"
#include "catalog/pg_foreign_volume.h"
#include "catalog/pg_language.h"
#include "catalog/pg_largeobject.h"
#include "catalog/pg_largeobject_metadata.h"
Expand Down Expand Up @@ -305,6 +307,34 @@ static const ObjectPropertyType ObjectProperty[] =
OBJECT_FOREIGN_SERVER,
true
},
{
"foreign catalog",
ForeignCatalogRelationId,
ForeignCatalogOidIndexId,
FOREIGNCATALOGOID,
FOREIGNCATALOGNAME,
Anum_pg_foreign_catalog_oid,
Anum_pg_foreign_catalog_fcname,
InvalidAttrNumber,
Anum_pg_foreign_catalog_fcowner,
InvalidAttrNumber,
OBJECT_FOREIGN_CATALOG,
true
},
{
"foreign volume",
ForeignVolumeRelationId,
ForeignVolumeOidIndexId,
FOREIGNVOLUMEOID,
FOREIGNVOLUMENAME,
Anum_pg_foreign_volume_oid,
Anum_pg_foreign_volume_fvname,
InvalidAttrNumber,
Anum_pg_foreign_volume_fvowner,
InvalidAttrNumber,
OBJECT_FOREIGN_VOLUME,
true
},
{
"storage server",
StorageServerRelationId,
Expand Down Expand Up @@ -995,6 +1025,14 @@ static const struct object_type_map
/* OCLASS_TAG */
{
"tag", OBJECT_TAG
},
/* OCLASS_FOREIGN_CATALOG */
{
"catalog", OBJECT_FOREIGN_CATALOG
},
/* OCLASS_FOREIGN_VOLUME */
{
"volume", OBJECT_FOREIGN_VOLUME
}
};

Expand Down Expand Up @@ -1165,6 +1203,8 @@ get_object_address(ObjectType objtype, Node *object,
case OBJECT_LANGUAGE:
case OBJECT_FDW:
case OBJECT_FOREIGN_SERVER:
case OBJECT_FOREIGN_CATALOG:
case OBJECT_FOREIGN_VOLUME:
case OBJECT_EVENT_TRIGGER:
case OBJECT_EXTPROTOCOL:
case OBJECT_PARAMETER_ACL:
Expand Down Expand Up @@ -1472,6 +1512,16 @@ get_object_address_unqualified(ObjectType objtype,
address.objectId = get_foreign_server_oid(name, missing_ok);
address.objectSubId = 0;
break;
case OBJECT_FOREIGN_CATALOG:
address.classId = ForeignCatalogRelationId;
address.objectId = get_foreign_catalog_oid(name, missing_ok);
address.objectSubId = 0;
break;
case OBJECT_FOREIGN_VOLUME:
address.classId = ForeignVolumeRelationId;
address.objectId = get_foreign_volume_oid(name, missing_ok);
address.objectSubId = 0;
break;
case OBJECT_EVENT_TRIGGER:
address.classId = EventTriggerRelationId;
address.objectId = get_event_trigger_oid(name, missing_ok);
Expand Down Expand Up @@ -2506,6 +2556,8 @@ pg_get_object_address(PG_FUNCTION_ARGS)
case OBJECT_EXTENSION:
case OBJECT_FDW:
case OBJECT_FOREIGN_SERVER:
case OBJECT_FOREIGN_CATALOG:
case OBJECT_FOREIGN_VOLUME:
case OBJECT_STORAGE_SERVER:
case OBJECT_LANGUAGE:
case OBJECT_PARAMETER_ACL:
Expand Down Expand Up @@ -2663,6 +2715,8 @@ check_object_ownership(Oid roleid, ObjectType objtype, ObjectAddress address,
case OBJECT_EXTENSION:
case OBJECT_FDW:
case OBJECT_FOREIGN_SERVER:
case OBJECT_FOREIGN_CATALOG:
case OBJECT_FOREIGN_VOLUME:
case OBJECT_LANGUAGE:
case OBJECT_PUBLICATION:
case OBJECT_SCHEMA:
Expand Down Expand Up @@ -3955,6 +4009,50 @@ getObjectDescription(const ObjectAddress *object, bool missing_ok)
break;
}

case OCLASS_FOREIGN_CATALOG:
{
HeapTuple catTup;
Form_pg_foreign_catalog catForm;

catTup = SearchSysCache1(FOREIGNCATALOGOID,
ObjectIdGetDatum(object->objectId));
if (!HeapTupleIsValid(catTup))
{
if (!missing_ok)
elog(ERROR, "cache lookup failed for foreign catalog %u",
object->objectId);
break;
}

catForm = (Form_pg_foreign_catalog) GETSTRUCT(catTup);
appendStringInfo(&buffer, _("catalog %s"),
NameStr(catForm->fcname));
ReleaseSysCache(catTup);
break;
}

case OCLASS_FOREIGN_VOLUME:
{
HeapTuple volTup;
Form_pg_foreign_volume volForm;

volTup = SearchSysCache1(FOREIGNVOLUMEOID,
ObjectIdGetDatum(object->objectId));
if (!HeapTupleIsValid(volTup))
{
if (!missing_ok)
elog(ERROR, "cache lookup failed for foreign volume %u",
object->objectId);
break;
}

volForm = (Form_pg_foreign_volume) GETSTRUCT(volTup);
appendStringInfo(&buffer, _("volume %s"),
NameStr(volForm->fvname));
ReleaseSysCache(volTup);
break;
}

case OCLASS_USER_MAPPING:
{
HeapTuple tup;
Expand Down Expand Up @@ -4949,6 +5047,14 @@ getObjectTypeDescription(const ObjectAddress *object, bool missing_ok)
appendStringInfoString(&buffer, "server");
break;

case OCLASS_FOREIGN_CATALOG:
appendStringInfoString(&buffer, "catalog");
break;

case OCLASS_FOREIGN_VOLUME:
appendStringInfoString(&buffer, "volume");
break;

case OCLASS_USER_MAPPING:
appendStringInfoString(&buffer, "user mapping");
break;
Expand Down Expand Up @@ -6043,6 +6149,54 @@ getObjectIdentityParts(const ObjectAddress *object,
break;
}

case OCLASS_FOREIGN_CATALOG:
{
HeapTuple catTup;
Form_pg_foreign_catalog catForm;

catTup = SearchSysCache1(FOREIGNCATALOGOID,
ObjectIdGetDatum(object->objectId));
if (!HeapTupleIsValid(catTup))
{
if (!missing_ok)
elog(ERROR, "cache lookup failed for foreign catalog %u",
object->objectId);
break;
}

catForm = (Form_pg_foreign_catalog) GETSTRUCT(catTup);
appendStringInfoString(&buffer,
quote_identifier(NameStr(catForm->fcname)));
if (objname)
*objname = list_make1(pstrdup(NameStr(catForm->fcname)));
ReleaseSysCache(catTup);
break;
}

case OCLASS_FOREIGN_VOLUME:
{
HeapTuple volTup;
Form_pg_foreign_volume volForm;

volTup = SearchSysCache1(FOREIGNVOLUMEOID,
ObjectIdGetDatum(object->objectId));
if (!HeapTupleIsValid(volTup))
{
if (!missing_ok)
elog(ERROR, "cache lookup failed for foreign volume %u",
object->objectId);
break;
}

volForm = (Form_pg_foreign_volume) GETSTRUCT(volTup);
appendStringInfoString(&buffer,
quote_identifier(NameStr(volForm->fvname)));
if (objname)
*objname = list_make1(pstrdup(NameStr(volForm->fvname)));
ReleaseSysCache(volTup);
break;
}

case OCLASS_STORAGE_SERVER:
{
StorageServer *srv;
Expand Down
Loading
Loading